diff --git a/Documentation/devicetree/bindings/interrupt-controller/apple,pcie-msi.yaml b/Documentation/devicetree/bindings/interrupt-controller/apple,pcie-msi.yaml new file mode 100644 index 0000000000000..e08ca3bf7108a --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/apple,pcie-msi.yaml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/interrupt-controller/apple,pcie-msi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple PCIe port MSI controller + +maintainers: + - Thomas Glanzmann + +description: | + The MSI map of one Apple PCIe root port, for a port that the bootloader + trained and that Linux drives as a generic ECAM host. Endpoint MSI writes + to the doorbell are routed by this map to a contiguous range of AIC + interrupts. + +allOf: + - $ref: /schemas/interrupt-controller/msi-controller.yaml# + +properties: + compatible: + const: apple,t8142-pcie-msi + + reg: + maxItems: 1 + description: The port's MSI map registers. + + msi-controller: true + + msi-ranges: + maxItems: 1 + +required: + - compatible + - reg + - msi-controller + - msi-ranges + +unevaluatedProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + msi-controller@488028000 { + compatible = "apple,t8142-pcie-msi"; + reg = <0x4 0x88028000 0x0 0x8000>; + msi-controller; + msi-ranges = <&aic AIC_IRQ 1741 IRQ_TYPE_EDGE_RISING 32>; + }; + }; diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 8fb92757fe7e5..cc67f2ec1fdf4 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -45,6 +45,7 @@ properties: - apple,t8132-dart - apple,t8140-dart - apple,t8142-dart + - apple,t8152-dart - const: apple,t8110-dart apple,managed-sids: diff --git a/Documentation/devicetree/bindings/iommu/apple,sart.yaml b/Documentation/devicetree/bindings/iommu/apple,sart.yaml index a75b4c8f05d5b..47fb67a6906df 100644 --- a/Documentation/devicetree/bindings/iommu/apple,sart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,sart.yaml @@ -37,6 +37,7 @@ properties: - apple,t8122-sart - apple,t8132-sart - apple,t8140-sart + - apple,t8142-sart - const: apple,t6000-sart - enum: - apple,t6000-sart diff --git a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml index f060194de4b9d..289c29ba752fb 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -21,6 +21,7 @@ properties: enum: - apple,t6040-ascwrap-v6 - apple,t8140-ascwrap-v6 + - apple,t8142-ascwrap-v6 - apple,t6050-apif-ascwrap-v6 - apple,t6050-ascwrap-v6 - apple,t6050-x1n1-ascwrap-v6 diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 071ba0f3a6fed..2f281baae5dca 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -22,6 +22,11 @@ description: properties: compatible: oneOf: + - description: + T8152 SMC post-office mailbox with counted FIFOs and interrupt + masking provided by the interrupt controller. + const: apple,t8152-smc-mailbox + - description: ASC mailboxes are the most common variant found on the M1 used for example for the display controller, the system management @@ -126,6 +131,21 @@ properties: maxItems: 2 allOf: + - if: + properties: + compatible: + contains: + const: apple,t8152-smc-mailbox + then: + properties: + interrupts: + minItems: 2 + maxItems: 2 + interrupt-names: + items: + - const: send-empty + - const: recv-not-empty + - if: properties: compatible: @@ -174,6 +194,7 @@ allOf: - apple,t6021-ane-mailbox - apple,t8112-ane-mailbox - apple,t8140-aop-setup-mailbox + - apple,t8152-smc-mailbox then: properties: interrupts: diff --git a/Documentation/devicetree/bindings/media/apple,avd.yaml b/Documentation/devicetree/bindings/media/apple,avd.yaml index de4101dfb50f7..1187255b0f474 100644 --- a/Documentation/devicetree/bindings/media/apple,avd.yaml +++ b/Documentation/devicetree/bindings/media/apple,avd.yaml @@ -35,6 +35,7 @@ properties: - apple,t8122-avd - apple,t8132-avd - apple,t8140-avd + - apple,t8142-avd reg: items: diff --git a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml index 59c6c91fb19d7..1671fa01bec96 100644 --- a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml +++ b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml @@ -19,14 +19,18 @@ select: properties: compatible: contains: - const: apple,t8140-isp + enum: + - apple,t8140-isp + - apple,t8142-isp required: - compatible properties: compatible: items: - - const: apple,t8140-isp + - enum: + - apple,t8140-isp + - apple,t8142-isp - const: apple,isp reg: diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index c90df16e9b974..1fd6820d86748 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -16,6 +16,8 @@ description: properties: compatible: oneOf: + - description: T8152 SMC with read-only hardware monitoring. + const: apple,t8152-smc - items: - enum: - apple,t6020-smc diff --git a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml index 85f0fd2b6fa1e..93388178e330e 100644 --- a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml +++ b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml @@ -26,6 +26,9 @@ properties: - items: - const: apple,t8140-nvme-ans2 - const: apple,t8132-nvme-ans2 + - items: + - const: apple,t8142-nvme-ans2 + - const: apple,t8140-nvme-ans2 - items: - enum: # Do not add additional SoC to this list. @@ -146,6 +149,7 @@ allOf: enum: - apple,t6041-nvme-ans2 - apple,t8132-nvme-ans2 + - apple,t8140-nvme-ans2 then: properties: reg: diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 3a499f802b8f6..d5510bafbef11 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -62,6 +62,7 @@ properties: - const: apple,t6020-pcie - const: apple,t6020-pcie - const: apple,t8140-pcie + - const: apple,t8152-pcie - const: apple,t8132-pcie - const: apple,t8103-pciec - items: @@ -319,20 +320,38 @@ allOf: interrupts: maxItems: 3 else: - properties: - reg-names: - minItems: 3 - items: - - const: config - - const: rc - - const: port0 - - const: port1 - - const: port2 - - const: port3 - - const: phy0 - - const: phy1 - - const: phy2 - - const: phy3 + if: + properties: + compatible: + contains: + const: apple,t8152-pcie + then: + properties: + reg: + minItems: 4 + maxItems: 8 + reg-names: + minItems: 4 + maxItems: 8 + items: + enum: [config, rc, port0, phy0, port1, phy1, port2, phy2] + interrupts: + maxItems: 3 + else: + properties: + reg-names: + minItems: 3 + items: + - const: config + - const: rc + - const: port0 + - const: port1 + - const: port2 + - const: port3 + - const: phy0 + - const: phy1 + - const: phy2 + - const: phy3 patternProperties: "^pci@": properties: diff --git a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml index edae16e33adbd..1c74f8ccc37a2 100644 --- a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml @@ -26,6 +26,7 @@ properties: - apple,t8122-pinctrl - apple,t8132-pinctrl - apple,t8140-pinctrl + - apple,t8152-pinctrl - const: apple,t8103-pinctrl - items: # Do not add additional SoC to this list. diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index a3728ed3effee..825d68ce51998 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -28,6 +28,8 @@ properties: - apple,t6041-pmc - apple,t8132-pmc - apple,t8140-pmc + - apple,t8142-pmc + - apple,t8152-pmc reg: minItems: 2 @@ -228,6 +230,8 @@ patternProperties: - apple,t6041-clpc - apple,t8132-clpc - apple,t8140-clpc + - apple,t8142-clpc + - apple,t8152-clpc reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/spi/apple,qspi.yaml b/Documentation/devicetree/bindings/spi/apple,qspi.yaml index 0b98af1ad68d2..40d08a8273712 100644 --- a/Documentation/devicetree/bindings/spi/apple,qspi.yaml +++ b/Documentation/devicetree/bindings/spi/apple,qspi.yaml @@ -19,11 +19,15 @@ allOf: properties: compatible: - enum: - - apple,t8122-qspi - - apple,t8132-qspi - - apple,t6041-qspi - - apple,t8140-qspi + oneOf: + - enum: + - apple,t8122-qspi + - apple,t8132-qspi + - apple,t6041-qspi + - apple,t8140-qspi + - items: + - const: apple,t8152-qspi + - const: apple,t8140-qspi reg: maxItems: 1 diff --git a/arch/arm64/boot/dts/apple/t8142-j813-audio.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-audio.dtsi new file mode 100644 index 0000000000000..892155da27ee0 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-audio.dtsi @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (13-inch, M5) J813: sound through the AOP firmware. + * + * As on J873 (T8152), the AOP's AOPAudioService owns the audio fabric and + * runs the MCA serializers: "cout"/"cin" (CS42L84 headphone jack) on + * admac-base-ns TX2/RX2 and "spkr" (four SN012776) on TX0, their V/I sense + * on RX0 (ADT iop-audio-controller; RX1 is the firmware's speaker tap). + * The AOP itself is laid out like T8140's (ascwrap-v6, asc-mailbox-v4), + * including the setup port. + * Addresses are from the J813 ADT (arm-io offset 0x210000000). + */ + +#include + +&pmgr { + ps_i2c1: power-controller@350 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c1"; + }; + + ps_i2c3: power-controller@360 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c3"; + }; +}; + +&{/soc} { + pinctrl_ap: pinctrl@39a000000 { + compatible = "apple,t8142-pinctrl", "apple,t8140-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x9a000000 0x0 0x100000>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 224>; + apple,npins = <224>; + /* + * Not every pin is the AP's: reading pin 78's register raises + * an SError. Expose only the pins sound uses: 100 (jack IRQ), + * 150 (amp IRQ) and 151 (amp shutdown). + */ + gpio-reserved-ranges = <0 100>, <101 49>, <152 72>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pinctrl_nub: pinctrl@3881f0000 { + compatible = "apple,t8142-pinctrl", "apple,t8140-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x881f0000 0x0 0x4000>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_nub 0 0 40>; + apple,npins = <40>; + /* Only pin 18 (jack codec reset), as on pinctrl_ap. */ + gpio-reserved-ranges = <0 18>, <19 21>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pmgr_aop: power-management@388284000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x3 0x88284000 0x0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + + /* + * Only the MCA serializer leaves (ADT AUDIO_MCA0_M..AUDIO_MCA5_M) + * are the AP's, as on J873. Their parents AUDIO_A, AUDIO_FR, + * AUDIO_P and AUDIO_BASE_NS_A (0x80, 0xb8, 0xd8, 0xf8) are the AOP + * firmware's: iBoot leaves them off, the firmware's audio init + * switches them off and polls for that (an AP-held auto-enable + * made it time out with 0xe00002d6, so "aop-audio" never + * registered and the firmware panicked in systemReady), and a + * service's pw0 request raises them again. + * + * Likewise the microphone array's LEAP leaves (AUDIO_LEAP_MCA, + * AUDIO_LEAP_S_A, AUDIO_LEAP_RX0) without their parents + * AUDIO_LEAP_C and AUDIO_LEAP_A (0x138, 0xe0); hpai's pw0 raises + * those. + */ + ps_audio_leap_mca: power-controller@100 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_mca"; + }; + + ps_audio_mca0_m: power-controller@108 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca0_m"; + }; + + ps_audio_mca1_m: power-controller@110 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca1_m"; + }; + + ps_audio_mca2_m: power-controller@118 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca2_m"; + }; + + ps_audio_mca3_m: power-controller@120 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca3_m"; + }; + + ps_audio_mca4_m: power-controller@128 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca4_m"; + }; + + ps_audio_mca5_m: power-controller@130 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca5_m"; + }; + + ps_audio_leap_s_a: power-controller@140 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_s_a"; + }; + + ps_audio_leap_rx0: power-controller@150 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_rx0"; + }; + }; + + aop_mbox: mailbox@390608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x90608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + /* + * The ADT's aop-exclave-mailbox, flagged exclave-assigned as on T8140, + * is the AOP setup port the host drives. Its ADT reg (0x3d067c000) is + * not behind the arm-io translation: AOP ASC + 0x4007c000, as on + * T8140. ADT interrupts [497, 494], T8140's +100. + */ + aop_setup_mbox: mailbox@3d067c000 { + compatible = "apple,t8142-aop-setup-mailbox", "apple,t8140-aop-setup-mailbox"; + reg = <0x3 0xd067c000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + /* dart-aop stream 1: ADT vm-base-1 0x20000000000, as on T8140. */ + iommus = <&aop_dart 1>; + apple,dma-range = <0x200 0x0 0x10 0x0>; + #mbox-cells = <0>; + }; + + /* + * dart-aop: streams 0 (AOP), 3, 8 (admac-base-ns), 10 and 16 are the + * AP's; 1, 9 and 11 are exclave streams, of which the host drives 1 + * (setup port) and 11 (admac-leap-s) on the raw boot. As on T8140, the + * firmware's own mappings own the first slot, host buffers go above it. + */ + aop_dart: iommu@390fc0000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x3 0x90fc0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + admac_base_ns: dma-controller@393800000 { + compatible = "apple,t8140-admac"; + reg = <0x3 0x93800000 0x0 0x34000>; + dma-channels = <13>; + interrupts-extended = <&aic AIC_IRQ 549 IRQ_TYPE_LEVEL_HIGH>, + <0>, <0>, <0>; + #dma-cells = <1>; + iommus = <&aop_dart 8>; + apple,dma-range = <0x110 0x00000000 0x0 0x10000000>; + }; + + /* + * admac-leap-s, for the microphone array (ADT audio-hp-mic -> + * leap-s-adma-rx-0), through its admac-leap-s-rx-0-proxy aperture as + * on T8140: the direct window (0x393200000) faults on per-channel + * registers. dart-aop stream 11 with the ADT's vm-base-11/vm-size-11. + * No power domain: the channel registers answer only with the LEAP + * leaves up, which the audio driver raises (after hpai's pw0) before + * it requests the channel; the controller touches no register at + * probe. No resets: the firmware owns the LEAP. + */ + admac_leap_s: dma-controller@3d3200000 { + compatible = "apple,t8140-admac"; + reg = <0x3 0xd3200000 0x0 0x34000>; + dma-channels = <9>; + interrupts-extended = <&aic AIC_IRQ 551 IRQ_TYPE_LEVEL_HIGH>, + <0>, <0>, <0>; + #dma-cells = <1>; + iommus = <&aop_dart 11>; + apple,dma-range = <0x100 0x40000000 0x0 0xffff0000>; + }; + + i2c1: i2c@3a5014000 { + compatible = "apple,t8142-i2c", "apple,t8103-i2c"; + reg = <0x3 0xa5014000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c1>; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 18 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 100 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + sound-name-prefix = "Jack"; + }; + }; + + /* + * Four SN012776: left woofer (0x38, carries the shared shutdown and + * interrupt lines), left tweeter, right woofer, right tweeter. The ADT + * speaker-config lists them as 0x38, then its speaker1..3 (right + * woofer, left tweeter, right tweeter): playback slots 0-3 and V/I + * sense in 8-bit slots 0/2, 4/6, 8/10, 12/14 in that order. The sense + * returns on two lines, the left pair and the right pair; the woofers + * drive the other pair's slots to zero (ADT amp-tx-zd-config, see the + * speaker dai-link). The firmware's speaker frame is 250 BCLKs of + * 12 MHz (ADT audio-speaker stream), which the amplifiers' ratio + * detection rejects. + */ + i2c3: i2c@3a501c000 { + compatible = "apple,t8142-i2c", "apple,t8103-i2c"; + reg = <0x3 0xa501c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c3>; + + speaker_left_woofer: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 150 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer"; + ti,imon-slot-no = <2>; + ti,vmon-slot-no = <0>; + ti,bclk-ratio = <250>; + }; + + speaker_left_tweeter: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + ti,imon-slot-no = <10>; + ti,vmon-slot-no = <8>; + ti,bclk-ratio = <250>; + }; + + speaker_right_woofer: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer"; + ti,imon-slot-no = <6>; + ti,vmon-slot-no = <4>; + ti,bclk-ratio = <250>; + }; + + speaker_right_tweeter: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + ti,imon-slot-no = <14>; + ti,vmon-slot-no = <12>; + ti,bclk-ratio = <250>; + }; + }; + + aop: aop@390c00000 { + compatible = "apple,t8142-aop", "apple,t8152-aop"; + reg = <0x3 0x90c00000 0x0 0x1e0000>, + <0x3 0x90600000 0x0 0x4000>; + reg-names = "mmio", "asc"; + mboxes = <&aop_mbox>, <&aop_setup_mbox>; + mbox-names = "mbox", "setup"; + iommus = <&aop_dart 0>; + apple,dma-range = <0x110 0x0 0x10 0x0>; + + aop_audio: audio { + compatible = "apple,t8142-aop-audio", "apple,t8152-aop-audio"; + /* The microphone card's name: AppleJ813HPAI. */ + apple,chassis-name = "J813"; + apple,machine-kind = "MacBook Air"; + /* hpai: leap-s RX0 (Linux channel 1, odd = DEV_TO_MEM). */ + dmas = <&admac_base_ns 4>, <&admac_base_ns 5>, + <&admac_base_ns 0>, <&admac_base_ns 1>, + <&admac_leap_s 1>; + dma-names = "cout", "cin", "spkr", "sense", "hpai"; + power-domains = <&ps_audio_mca0_m>, <&ps_audio_mca1_m>, + <&ps_audio_mca2_m>, <&ps_audio_mca3_m>, + <&ps_audio_mca4_m>, <&ps_audio_mca5_m>, + <&ps_audio_leap_s_a>, <&ps_audio_leap_rx0>, + <&ps_audio_leap_mca>; + power-domain-names = "mca0", "mca1", "mca2", + "mca3", "mca4", "mca5", + "leap_s_a", "leap_rx0", "leap_mca"; + #sound-dai-cells = <1>; + }; + }; +}; + +/ { + sound { + compatible = "apple,j813-macaudio", "apple,macaudio"; + model = "MacBook Air J813"; + #address-cells = <1>; + #size-cells = <0>; + + dai-link@0 { + reg = <0>; + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&aop_audio 0>; + }; + + codec { + sound-dai = <&jack_codec>; + }; + }; + + dai-link@1 { + reg = <1>; + link-name = "Speaker"; + /* + * Codecs in ADT speaker-config order, so that macaudio + * gives them playback slots 0-3 as macOS does (left + * channels in slots 0 and 2, right in 1 and 3). + */ + dai-tdm-idle-mode-0 = "zero"; + dai-tdm-slot-tx-idle-mask-0 = <0xfff0f0f0>; + dai-tdm-idle-mode-1 = "zero"; + dai-tdm-slot-tx-idle-mask-1 = <0xff0f0f0f>; + + cpu { + sound-dai = <&aop_audio 3>; + }; + + codec { + sound-dai = <&speaker_left_woofer>, <&speaker_right_woofer>, + <&speaker_left_tweeter>, <&speaker_right_tweeter>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi new file mode 100644 index 0000000000000..5672ebb885a51 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * M5 (T8142) AVD video decoder: a "Hibiscus" AVD (macOS CAvdMcpuHibiscus, + * avd-version 4, tunables = avd-fw v5 tier 2). From the J813 ADT: avd reg[0] + * 0x2f0000000 + 0x1404000, here at the arm-io offset 0x500000000; 0x14000 each + * of CM3 code and SRAM, mailbox at +0x10a8000 (CAvdApCommHibiscus); + * IRQs 1077/1078; dart-avd at +0x1020000, IRQ 1084, SIDs 0-2; power: PMGR + * device AVD-SYS-V -> AVD_SYS (+0x310). + */ + +&pmgr { + ps_avd_sys: power-controller@310 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "avd_sys"; + }; +}; + +&{/soc} { + avd: avd@501080000 { + compatible = "apple,t8142-avd"; + interrupt-parent = <&aic>; + interrupts = , + ; + power-domains = <&ps_avd_sys>; + resets = <&ps_avd_sys>; + reg = <0x5 0x01080000 0x0 0x14000>, + <0x5 0x01094000 0x0 0x14000>, + <0x5 0x010a8000 0x0 0x4000>, + <0x5 0x01100000 0x0 0x10000>; + reg-names = "code", "sram", "mbox", "ctrl"; + iommus = <&avd_dart 0>, <&avd_dart 1>, <&avd_dart 2>; + }; + + avd_dart: iommu@501020000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x5 0x01020000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_avd_sys>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi index 9bd549a5784cc..d9e890af7fd44 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi @@ -91,5 +91,18 @@ iommus = <&disp0_dart 16>; phandle = <&disp0_piodma>; }; + + /* + * 13.6" 2560x1664 (1600 below the notch); 500 nits peak (ADT + * backlight known-nits). Makes the connector eDP and gives the + * DCP a backlight to drive. + */ + panel: panel { + compatible = "apple,panel-j813", "apple,panel"; + width-mm = <290>; + height-mm = <188>; + adj-height-mm = <181>; + apple,max-brightness = <500>; + }; }; }; diff --git a/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi new file mode 100644 index 0000000000000..d1b8b540538bd --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * M5 (T8142) GPU, laid out like the A18 Pro's (t8140.dtsi). Addresses and + * interrupts are from the J813 ADT (sgx, gfx-asc, gfx1-asc, pmgr reg[48]). + * The bootloader fills the uat-* regions from iBoot's raw-boot handoff, the + * gfx-data regions from the firmware DATA segments iBoot loaded, and the + * OPPs from sgx perf-states. + */ + +/ { + aliases { + gpu = &gpu; + }; + + reserved-memory { + /* gfx-asc and gfx1-asc __DATA (ADT sgx gfx-data, gfx1-data) */ + gfx_data: gfx-data { + reg = <0 0 0 0>; + }; + + gfx1_data: gfx1-data { + reg = <0 0 0 0>; + }; + + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + }; + + uat_l2_pagetables: uat-l2-pagetables { + reg = <0 0 0 0>; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + }; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + }; + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* 13 entries in perf-state order, filled in by the bootloader */ + opp00 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp01 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp02 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp03 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp04 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp05 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp06 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp07 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp08 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp09 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp10 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp11 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp12 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + }; +}; + +&{/soc} { + pmgr_gpu: power-management@580e80000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <5 0x80e80000 0 0x30000>; + #address-cells = <1>; + #size-cells = <1>; + + ps_gpx: power-controller@0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gpx"; + apple,always-on; /* critical */ + }; + + ps_gfx: power-controller@110 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + power-domains = <&ps_gpx>; + }; + }; + + gpu: gpu@582600000 { + compatible = "apple,agx-t8142"; + reg = <0x5 0x82600000 0x0 0x88000>, + <0x5 0x80000000 0x0 0x4000000>; + reg-names = "asc", "sgx"; + mboxes = <&gfx_mbox>, <&gfx1_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, + <&uat_l2_pagetables>, <&gfx_data>, <&gfx1_data>; + memory-region-names = "ttbs", "pagetables", "handoff", + "shared-l2", "gfx-data", "gfx1-data"; + operating-points-v2 = <&gpu_opp>; + /* Holds the DCS and fabric floor while GPU work is in flight. */ + apple,clpc = <&clpc>; + }; + + gfx_mbox: mailbox@582600000 { + compatible = "apple,t8142-ascwrap-v6"; + reg = <0x5 0x82600000 0x0 0x88000>, + <0x5 0x82050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + }; + + gfx1_mbox: mailbox@582e00000 { + compatible = "apple,t8142-ascwrap-v6"; + reg = <0x5 0x82e00000 0x0 0x88000>, + <0x5 0x82850000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX1"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi new file mode 100644 index 0000000000000..f3850c4161925 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * M5 (T8142) ISP: the Neo's (T8140) ISP17a at another base. From the J813 + * ADT (arm-io offset 0x210000000): isp reg 0x4a0000000 + 0x4000000, IRQs + * [425 426 427 428 438] (425 carries the IPC mailbox), sensor-type 0x71; + * dart-isp at +0x28c0000/+0x28e0000/+0x2900000, IRQ 443, vm-size 0xe0000000, + * remap streams 1 2 3 6 7 10 11 -> 0; EIC mmio-wd-isp 0x4e3b10000 + 0x8000. + * PMGR (0x380700000): ISP_SYS +0x2f8 (ADT id 75), the ISP17a slots + * +0x4000..+0x4060 (ids 162-174, the Neo's offsets), ISPSENS0/1 +0x190/+0x198. + * aura carves the firmware heap and adds it as "isp-heap"; without it, aura + * disables the node. + */ + +/ { + aliases { + isp = &isp; + }; +}; + +&pmgr { + ps_isp_sys: power-controller@2f8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sys"; + }; + + ps_ispsens0: power-controller@190 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens0"; + }; + + ps_ispsens1: power-controller@198 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens1"; + }; + + ps_isp_cpu: power-controller@4000 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cpu"; + power-domains = <&ps_isp_sys>; + }; + + ps_isp_cpu_core0: power-controller@4008 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cpu_core0"; + power-domains = <&ps_isp_cpu>; + }; + + /* + * Not here: ISP_CPU_CORE1 (+0x4010), ISP_SECURE (+0x4028), ISP_STS1 + * (+0x4038), ISP_PEARL (+0x4040), ISP_CVD (+0x4058) and ISP_CVM + * (+0x4060). On J813 their target fields ignore writes from the AP and + * read back 0. The dart-isp DAPF passes this block to the ISP. + */ + + ps_isp_fe: power-controller@4018 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + power-domains = <&ps_isp_sys>; + }; + + /* Unnamed in the ADT, as on the Neo: the firmware's ISP_DPRX. */ + ps_isp_dprx: power-controller@4020 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_dprx"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_sts0: power-controller@4030 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sts0"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_be: power-controller@4048 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_be"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_clr: power-controller@4050 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_clr"; + power-domains = <&ps_isp_be>; + }; + +}; + +&{/soc} { + isp_dart0: iommu@4a28c0000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x4 0xa28c0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + /* As on the Neo: 3-level tables, the ISP fetches at vm-base + * 0x10000000000 and the walk ignores the high bits. */ + apple,dma-range = <0x0 0x0 0x8 0x0>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + }; + + isp_dart1: iommu@4a28e0000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x4 0xa28e0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,dma-range = <0x0 0x0 0x8 0x0>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + }; + + isp_dart2: iommu@4a2900000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x4 0xa2900000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,dma-range = <0x0 0x0 0x8 0x0>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + }; + + /* + * As on the Neo: "coproc" stops short of the secure route table at + * +0x1804000 (a read takes a sticky SError) and of the IRQ block + * (+0x2920000, "mbox") and the GPIO words (+0x2921000). + */ + isp: isp@4a0000000 { + compatible = "apple,t8142-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "wdt"; + reg = <0x4 0xa0000000 0x0 0x1801000>, + <0x4 0xa2920000 0x0 0x1000>, + <0x4 0xa2921000 0x0 0x100>, + <0x4 0xe3b10000 0x0 0x8000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, + <&ps_isp_cpu_core0>, <&ps_isp_fe>, + <&ps_isp_dprx>, <&ps_isp_sts0>, + <&ps_isp_be>, <&ps_isp_clr>; + apple,dart-vm-size = <0x0 0xe0000000>; + apple,platform-id = <0x71>; + apple,temporal-filter = <1>; + }; +}; + +/* + * The J813 IMX958 (sensor version 0x958) as its firmware lists it: 13 configs, + * among them 3040x2880 (config 7) and a binned 1520x1440 (config 10); half + * of them report doubled heights and are left out. Its setfile is + * apple/isp_2224_01XX.dat (from macOS appleh16camerad). + */ +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <1>; + + /* + * All presets scale from the full-resolution sensor mode (config 7). + * In the binned mode (config 10, 1520x1440) the firmware's AWB keeps + * the setfile's default white point and the picture is magenta; in + * config 7 it converges. + */ + sensor-presets { + /* 1280x720 */ + preset0 { + apple,config-index = <7>; + apple,input-size = <3040 2880>; + apple,output-size = <1280 720>; + apple,crop = <0 584 3040 1710>; + }; + /* 1440x1080 (4:3) */ + preset1 { + apple,config-index = <7>; + apple,input-size = <3040 2880>; + apple,output-size = <1440 1080>; + apple,crop = <80 360 2880 2160>; + }; + /* 1080x1080 (1:1) */ + preset2 { + apple,config-index = <7>; + apple,input-size = <3040 2880>; + apple,output-size = <1080 1080>; + apple,crop = <440 360 2160 2160>; + }; + /* 1920x1080 */ + preset3 { + apple,config-index = <7>; + apple,input-size = <3040 2880>; + apple,output-size = <1920 1080>; + apple,crop = <0 584 3040 1710>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi new file mode 100644 index 0000000000000..43a87da373a91 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * N1 Wi-Fi/Bluetooth on apcie port 0. The bootloader trains the link (AUS5 + * PHY, refclk buffer, N1 reset, port; see the m5air FINDINGS); Linux uses the + * root port as a generic ECAM host and owns the DART and the endpoint. + * Ranges and the DART IRQ from the J813 ADT, as in t8142-j813-wifi.dts. + */ +&{/} { + pcie0: pcie@1cb0000000 { + compatible = "pci-host-ecam-generic"; + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + bus-range = <0 1>; + reg = <0x1c 0xb0000000 0 0x200000>; + ranges = <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0 0x20000000>, + <0x02000000 0 0x80000000 0xb 0x80000000 0 0x40000000>; + dma-coherent; + msi-parent = <&pcie0_msi>; + /* N1's three ADT mapper nodes use stream zero. */ + iommu-map = <0x100 &dart_apcie0 0 1>, + <0x101 &dart_apcie0 0 1>, + <0x102 &dart_apcie0 0 1>; + }; +}; + +&{/soc} { + /* + * Port 0's MSI map (ADT apcie reg[8]). ADT msi-address 0xfffff000, + * msi-vector-offset 1741, #msi-vectors 96 for three ports: port 0 + * owns the first 32. + */ + pcie0_msi: msi-controller@488028000 { + compatible = "apple,t8142-pcie-msi"; + reg = <4 0x88028000 0 0x8000>; + msi-controller; + msi-ranges = <&aic AIC_IRQ 1741 IRQ_TYPE_EDGE_RISING 32>; + }; + + dart_apcie0: iommu@488000000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x88000000 0 0x20000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813-nvme.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-nvme.dtsi new file mode 100644 index 0000000000000..82f2bce059d17 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-nvme.dtsi @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (13-inch, M5) J813 with the internal ANS2 NVMe SSD. + * + * Addresses from the J813 ADT (/arm-io/ans, /arm-io/sart-ans, arm-io offset + * 0x210000000). ANS is an ascwrap-v6 ASC with its mailbox at +0x8000, like the + * SMC; the SART is coastguard sart-version 3. As on T8140, the NVMe registers + * are used through the non-secure alias (ADT reg[9]) and the NVMMU through + * the 0x60000 window (ADT reg[3]). iBoot leaves ANS running with the + * controller disabled, so the driver wakes it with a fresh RTKit handshake. + */ + + +&pmgr { + ps_apcie_gp: power-controller@2f0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_gp"; + }; + + ps_ans: power-controller@300 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ans"; + apple,always-on; /* handed over running, cannot be reset */ + }; + + ps_apcie_sys_gp: power-controller@408 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_gp"; + power-domains = <&ps_apcie_gp>; + }; + + ps_apcie_st: power-controller@410 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x410 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_st"; + power-domains = <&ps_ans>; + }; + + ps_apcie_sys_st: power-controller@520 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x520 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_st"; + power-domains = <&ps_ans>, <&ps_apcie_st>; + }; + + ps_apcie_phy_sw: power-controller@560 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x560 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_phy_sw"; + power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + }; +}; + +&{/soc} { + ans_mbox: mailbox@481608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x81608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_ans>; + #mbox-cells = <0>; + }; + + sart: iommu@485c50000 { + compatible = "apple,t8142-sart", "apple,t6000-sart"; + reg = <0x4 0x85c50000 0x0 0xc000>; + power-domains = <&ps_ans>; + }; + + nvme: nvme@4c5cc0000 { + compatible = "apple,t8142-nvme-ans2", "apple,t8140-nvme-ans2"; + reg = <0x4 0xc5cc0000 0x0 0x30000>, + <0x4 0x81600000 0x0 0x4000>, + <0x4 0x85cc0000 0x0 0x60000>; + reg-names = "nvme", "ans", "nvmmu"; + interrupt-parent = <&aic>; + interrupts = ; + mboxes = <&ans_mbox>; + apple,sart = <&sart>; + power-domains = <&ps_ans>, <&ps_apcie_sys_st>, + <&ps_apcie_phy_sw>; + power-domain-names = "ans", "apcie0", "apcie_phy"; + resets = <&ps_ans>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi index a6708b1c972df..5ca6fed62568a 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi @@ -1,10 +1,20 @@ // SPDX-License-Identifier: GPL-2.0+ OR MIT +#include #include / { aliases { mtp = &mtp; keyboard = &keyboard; + nvram = &nvram; + }; + + /* J813 ADT /arm-io/qspi clock-ids [396]; rate as the T8140 QSPI. */ + clk_qspi: clock-qspi { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_qspi"; }; led-controller { @@ -24,7 +34,8 @@ &{/soc} { pmgr: power-management@380700000 { compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; - reg = <3 0x80700000 0 0x4000>; + /* As on T8140: up to the ISP17a slots at +0x4000..+0x4060. */ + reg = <3 0x80700000 0 0x14000>; #address-cells = <1>; #size-cells = <1>; @@ -35,6 +46,122 @@ #reset-cells = <0>; label = "fpwm0"; }; + + /* ADT pmgr devices: SPI_P id 94 (+0x390), QSPI id 120 (+0x460). */ + ps_spi_p: power-controller@390 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + + /* ADT pmgr devices: PMP id 159 (+0x598), PMS_SRAM id 160 (+0x5a0). */ + ps_pmp: power-controller@598 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x598 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmp"; + /* The PMP's SRAM, which the driver reads first. */ + power-domains = <&ps_pms_sram>; + }; + + ps_pms_sram: power-controller@5a0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_sram"; + /* T8140 holds it through its always-on PMP report node. */ + apple,always-on; + }; + + ps_qspi: power-controller@460 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x460 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + }; + }; + + /* + * PMP, laid out as on T8140 (ADT /arm-io/pmp, dart-pmp): it runs the + * DRAM controller (DCS) and fabric DVFS. Without it the memory stays at + * the bootloader's state (memcpy ~20 GB/s against ~120 GB/s on macOS). + */ + pmp_dart: iommu@380300000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x3 0x80300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000000000 0xbff00000>; + power-domains = <&ps_pmp>; + }; + + pmp: pmp@380500000 { + compatible = "apple,t8142-pmp-v2", "apple,t8140-pmp-v2"; + reg = <0x3 0x80500000 0x0 0xd0000>, + <0x3 0x80e00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + /* ADT /chosen board-id, dram-vendor-id */ + apple,board-id = <36>; + apple,dram-vendor-id = <1>; + /* ADT /arm-io/pmgr pmc, pmc-pmgr (registry-v2 boot arguments) */ + apple,pmc = <1>; + apple,pmc-pmgr = <31>; + }; + + pmp_mbox: mailbox@380e08000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x80e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>; + }; + + /* ADT /arm-io/qspi: 0x195118000 and its FIFO window at +0x800, IRQ 1245. */ + qspi: spi@3a5118000 { + compatible = "apple,t8142-qspi", "apple,t8140-qspi"; + reg = <0x3 0xa5118000 0x0 0x4800>; + clocks = <&clk_qspi>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_qspi>; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + /* Conservative until the input clock is confirmed. */ + spi-max-frequency = <10000000>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + /* ADT spinor/anvram reg: 0x100000 bytes at 0x700000. */ + nvram: partition@700000 { + label = "nvram"; + reg = <0x700000 0x100000>; + }; + }; + }; }; fpwm0: pwm@3a5040000 { @@ -66,6 +193,12 @@ compatible = "apple,smc-hwmon"; apple,discover-temperatures; }; + + smc_gpio: gpio { + compatible = "apple,smc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; }; mtp: mtp@394600000 { @@ -121,13 +254,20 @@ interrupt-names = "tx", "rx"; apple,fifo-size = <0x800>; apple,helper-cpu = <&mtp>; + /* + * ADT dockchannel-mtp/mtp-transport/multi-touch: + * function-afe-reset = smc-pmu pKW4('gP1c'), SMC GPIO 28. + * The MTP firmware cannot reach the AFE on its own ("No + * HINT_L"): after macOS left it running the trackpad + * worked, after a Linux warm reboot it stayed in reset. + */ + apple,afe-reset-gpios = <&smc_gpio 28 GPIO_ACTIVE_LOW>; stm { }; multi-touch { firmware-name = "apple/tpmtfw-j813.bin"; apple,power-method = <2>; - apple,firmware-managed-reset; }; /* Layout is filled by the loader using this machine's own data. */ @@ -135,3 +275,4 @@ }; }; }; +#include "t8142-j813-pmp-tunables.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8142-j813-pmp-tunables.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-pmp-tunables.dtsi new file mode 100644 index 0000000000000..fa34d4d80fe7d --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-pmp-tunables.dtsi @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J813 PMP firmware properties: the ADT /arm-io/pmp/iop-pmp-nub values the + * firmware queries by name at boot (pmp.rs serves apple,tunable-). + * Generated from the J813 ADT by tools/adtnub.py; raw ADT bytes. + */ + +&pmp { + apple,tunable-agx-dcs-bw-constraint = [00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00]; + apple,tunable-mc_quota_req_ptd_count = [08 00 00 00]; + apple,tunable-mc_cmd_ptd_count = [02 00 00 00]; + apple,tunable-ane-dcs-bw-dvfs-filter = [ff ff 1f 00]; + apple,tunable-sochot-sensors = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 15 00 00 00 16 00 00 00 17 00 00 00 18 00 00 00]; + apple,tunable-soc-ni3-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-agx-af-rd-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 35 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 34 00 00 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 00 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 00 00 00 00 00 00 3a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 39 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 2d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 2c 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 11 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3c 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 33 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 7a 00 00 00 00 00 00 00 32 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 79 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 2c 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 8a 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 89 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3e 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 6f 00 00 00 00 00 00 00 3d 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 6e 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 34 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 33 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2c 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 41 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 40 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3c 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 37 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 36 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00]; + apple,tunable-ane-af-bw-dvfs-filter = [ff ff 1f 00]; + apple,tunable-ans-af-bw-threshold = [70 3d 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 70 3d 0b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 cc 0c 1c 00 00 00 00 00]; + apple,tunable-ptd-range = [01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 50 54 44 2d 4e 55 4c 4c 00 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 01 00 00 00 0b 00 00 00 50 4d 50 2d 53 54 41 54 55 53 00 00 00 00 00 00 03 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 44 56 46 53 2d 53 54 41 54 45 00 00 00 00 00 00 04 00 00 00 06 00 00 00 04 00 00 00 00 00 00 00 50 4d 50 54 4f 4f 4c 00 00 00 00 00 00 00 00 00 05 00 00 00 98 00 00 00 08 00 00 00 00 00 00 00 50 4d 53 2d 50 4d 47 52 2d 50 4b 54 00 00 00 00 06 00 00 00 a0 00 00 00 08 00 00 00 00 00 00 00 50 4d 53 2d 44 50 45 2d 50 4b 54 00 00 00 00 00 07 00 00 00 28 02 00 00 18 00 00 00 00 00 00 00 50 4d 53 2d 44 45 56 43 2d 50 4b 54 00 00 00 00 09 00 00 00 a8 00 00 00 80 01 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 4b 54 00 00 00 00 00 0a 00 00 00 27 00 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 52 45 51 00 00 0b 00 00 00 2f 00 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 41 43 4b 00 00 0c 00 00 00 37 00 00 00 10 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 44 56 46 53 00 00 00 00 0d 00 00 00 47 00 00 00 0a 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 42 57 52 00 00 00 00 00 0e 00 00 00 13 00 00 00 0a 00 00 00 00 00 00 00 4d 43 2d 44 52 56 2d 54 4f 2d 43 54 52 4c 00 00 0f 00 00 00 51 00 00 00 01 00 00 00 00 00 00 00 44 56 46 53 2d 43 45 49 4c 00 00 00 00 00 00 00 14 00 00 00 0a 00 00 00 01 00 00 00 00 00 00 00 41 56 44 2d 4e 52 47 00 00 00 00 00 00 00 00 00 15 00 00 00 0b 00 00 00 01 00 00 00 00 00 00 00 53 4f 43 2d 41 4f 4e 2d 4e 52 47 00 00 00 00 00 16 00 00 00 0c 00 00 00 01 00 00 00 00 00 00 00 53 4f 43 2d 52 45 53 54 2d 4e 52 47 00 00 00 00 1e 00 00 00 1d 00 00 00 09 00 00 00 00 00 00 00 4d 43 2d 43 54 52 4c 2d 54 4f 2d 44 52 56 00 00 1f 00 00 00 0f 00 00 00 02 00 00 00 0c 00 00 00 46 41 53 54 2d 44 49 45 2d 43 45 00 00 00 00 00 20 00 00 00 52 00 00 00 07 00 00 00 0d 00 00 00 53 4d 43 2d 54 4f 2d 50 50 4d 00 00 00 00 00 00 21 00 00 00 0d 00 00 00 01 00 00 00 0a 00 00 00 4c 54 53 2d 56 4f 4c 54 2d 43 41 50 00 00 00 00 40 00 00 00 59 00 00 00 05 00 00 00 0e 00 00 00 50 50 4d 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 41 00 00 00 5e 00 00 00 05 00 00 00 0f 00 00 00 43 4c 50 43 2d 54 4f 2d 53 4d 43 00 00 00 00 00 43 00 00 00 11 00 00 00 02 00 00 00 00 00 00 00 46 41 53 54 2d 44 49 45 2d 54 4f 2d 53 4d 43 00 44 00 00 00 26 00 00 00 01 00 00 00 00 00 00 00 4d 45 4d 43 2d 54 4f 2d 53 4d 43 00 00 00 00 00 52 00 00 00 0e 00 00 00 01 00 00 00 00 00 00 00 4c 54 53 2d 54 4f 2d 47 46 58 00 00 00 00 00 00]; + apple,tunable-sram-index = [01 00 00 00]; + apple,tunable-no-shutdown = [01 00 00 00]; + apple,tunable-dvfs-domain = [01 00 00 00 08 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 53 4f 43 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 06 00 00 00 00 00 00 00 44 49 53 50 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-dvfs-disp-min-state = [01 00 00 00]; + apple,tunable-fast-die-ctrl-ce-map = [01 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 45 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 5f 41 4e 45 00 00 00 00 00 00 00 00]; + apple,tunable-ane-dcs-bw-constraint = [00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00]; + apple,tunable-soc-ni2-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-agx-af-wr-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00]; + apple,tunable-user-power-managed = [01 00 00 00]; + apple,tunable-ane-af-bw-threshold = [00 00 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 1c 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 09 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 06 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 16 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 41 00 00 00 00 00 00 00 15 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 40 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3f 00 00 00 00 00 00 00 12 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 3e 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3a 00 00 00 00 00 00 00 0c 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 39 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3a 00 00 00 00 00 00 00 0c 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 39 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 31 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 1e 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 0e 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 08 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 02 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 02 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00]; + apple,tunable-lts-voltage = [01 00 00 00 03 00 00 00 03 00 00 00 01 00 00 00 45 41 43 43 00 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 02 00 00 00 50 41 43 43 00 00 00 00 03 00 00 00 01 00 00 00 06 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 04 00 00 00 03 00 00 00 05 00 00 00 04 00 00 00 41 47 58 00 00 00 00 00]; + apple,tunable-ane-dcs-bw-threshold = [a3 f0 13 00 01 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 02 00 00 00 a3 f0 12 00 00 00 00 00 1e 05 3a 00 03 00 00 00 66 e6 26 00 00 00 00 00 0a 57 4a 00 04 00 00 00 1e 05 39 00 00 00 00 00 3d 0a 57 00 05 00 00 00 0a 57 49 00 00 00 00 00 3d 8a 65 00 06 00 00 00 3d 0a 56 00 00 00 00 00 5c 0f 74 00 07 00 00 00 3d 8a 64 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 00 00 00 00 a3 f0 13 00 01 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 02 00 00 00 a3 f0 12 00 00 00 00 00 1e 05 3a 00 03 00 00 00 66 e6 26 00 00 00 00 00 0a 57 4a 00 04 00 00 00 1e 05 39 00 00 00 00 00 3d 0a 57 00 05 00 00 00 0a 57 49 00 00 00 00 00 3d 8a 65 00 06 00 00 00 3d 0a 56 00 00 00 00 00 5c 0f 74 00 07 00 00 00 3d 8a 64 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 00 00 00 00 a3 f0 13 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 a3 f0 12 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 00 00 0e 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 00 00 09 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 00 00 09 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 1e 05 3a 00 00 00 00 00 00 00 0e 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 1e 05 3a 00 00 00 00 00 00 00 0e 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 04 00 06 00 00 00 0a 57 4a 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 57 4a 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 15 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 57 4a 00 00 00 00 00 00 00 14 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 00 00 06 00 00 00 3d 0a 57 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 00 00 06 00 00 00 3d 0a 57 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00]; + apple,tunable-bwr-catch-up-factor = [00 00 01 00]; + apple,tunable-controller = [01 00 00 00 1f 00 00 00 43 00 00 00 00 00 00 00 00 00 00 00 44 49 45 2d 54 45 4d 50 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 46 49 2d 54 4f 50 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 2d 52 44 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 2d 57 52 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 10 00 00 00 0e 00 00 00 1e 00 00 00 00 00 00 00 00 00 00 00 4d 45 4d 2d 43 41 43 48 45 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 11 00 00 00 52 00 00 00 21 00 00 00 00 00 00 00 00 00 00 00 4c 49 46 45 54 49 4d 45 2d 53 45 52 56 4f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-soc-device-ps-group = [00 fc ff 1f 00 00 00 00]; + apple,tunable-agx-dcs-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 27 00 00 00 00 00 00 00 12 00 00 00 00 00 00 00 39 00 00 00 00 00 00 00 26 00 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 38 00 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 11 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 27 00 00 00 00 00 00 00 10 00 06 00 00 00 00 00 39 00 00 00 00 00 00 00 26 00 06 00 00 00 00 00 4a 00 00 00 00 00 00 00 38 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 49 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 27 00 00 00 00 00 00 00 0b 00 06 00 00 00 00 00 39 00 00 00 00 00 00 00 26 00 06 00 00 00 00 00 47 00 00 00 00 00 00 00 38 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 46 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 37 00 00 00 00 00 00 00 21 00 06 00 00 00 00 00 47 00 00 00 00 00 00 00 36 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 46 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 1f 00 00 00 00 00 00 00 14 00 06 00 00 00 00 00 47 00 00 00 00 00 00 00 1e 00 06 00 00 00 00 00 57 00 00 00 00 00 00 00 46 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 56 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 07 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 57 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 65 00 00 00 00 00 00 00 56 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 64 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 17 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 43 00 00 00 00 00 00 00 16 00 06 00 00 00 00 00 65 00 00 00 00 00 00 00 42 00 06 00 00 00 00 00 74 00 00 00 00 00 00 00 64 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2d 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 50 00 00 00 00 00 00 00 2c 00 06 00 00 00 00 00 74 00 00 00 00 00 00 00 4f 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 10 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 3c 00 00 00 00 00 00 00 0f 00 06 00 00 00 00 00 74 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 1e 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00]; + apple,tunable-soc-ni1-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-mc_quota_rsp_ptd_count = [08 00 00 00]; + apple,tunable-lts-ctrl-ts = [00 00 05 00]; + apple,tunable-temp-sensor = [01 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 80 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 30 69 00 00 00 00 00 00 00 00 00 00 02 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 33 b3 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 31 69 00 00 00 00 00 00 00 00 00 00 03 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 66 e6 03 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 32 69 00 00 00 00 00 00 00 00 00 00 04 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 80 05 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 35 69 00 00 00 00 00 00 00 00 00 00 05 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 66 e6 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 36 69 00 00 00 00 00 00 00 00 00 00 06 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 cc 4c 05 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 37 69 00 00 00 00 00 00 00 00 00 00 07 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 30 69 00 00 00 00 00 00 00 00 00 00 08 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 31 69 00 00 00 00 00 00 00 00 00 00 09 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 33 69 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 34 69 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 01 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6c 69 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 00 00 01 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6d 69 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 05 00 00 00 00 00 00 00 33 33 03 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6e 69 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 06 00 00 00 00 00 00 00 66 e6 02 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6f 69 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 07 00 00 00 00 00 00 00 99 99 02 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 70 69 00 00 00 00 00 00 00 00 00 00 10 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 09 00 00 00 00 00 00 00 33 33 02 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 72 69 00 00 00 00 00 00 00 00 00 00 11 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 39 69 00 00 00 00 00 00 00 00 00 00 12 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 61 69 00 00 00 00 00 00 00 00 00 00 13 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 36 69 00 00 00 00 00 00 00 00 00 00 14 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 37 69 00 00 00 00 00 00 00 00 00 00 15 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 66 66 08 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 37 69 00 00 00 00 00 00 00 00 00 00 16 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 80 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 31 69 00 00 00 00 00 00 00 00 00 00 17 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 99 99 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 32 69 00 00 00 00 00 00 00 00 00 00 18 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 33 b3 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 33 69 00 00 00 00 00 00 00 00 00 00 19 00 00 00 01 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6d 00 00 00 6d 00 00 00 4d 54 52 5f 54 4f 50 5f 41 47 58 00 00 00 00 00]; + apple,tunable-agx-af-wr-bw-dvfs-filter = [fe 7f 00 00]; + apple,tunable-dcs-bw-threshold = [a3 f0 13 00 01 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 02 00 00 00 a3 f0 12 00 00 00 00 00 1e 05 3a 00 03 00 00 00 66 e6 26 00 00 00 00 00 0a 57 4a 00 04 00 00 00 1e 05 39 00 00 00 00 00 3d 0a 57 00 05 00 00 00 0a 57 49 00 00 00 00 00 3d 8a 65 00 06 00 00 00 3d 0a 56 00 00 00 00 00 5c 0f 74 00 07 00 00 00 3d 8a 64 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 00 00 00 00]; + apple,tunable-mc_cmd_ack_ptd_count = [01 00 00 00]; + apple,tunable-soc-ni0-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-fast-die-ctrl-loop = [01 00 00 00 00 00 00 00 00 00 00 00 66 06 00 00 47 41 00 00 00 00 6e 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 01 00 16 00 00 00 15 00 00 00 16 00 00 00 17 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 45 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 66 06 00 00 47 41 00 00 00 00 6e 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 01 00 16 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 02 00 00 00 66 06 00 00 47 41 00 00 00 00 6e 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 01 00 16 00 00 00 01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-agx-dcs-bw-dvfs-filter = [fe 7f 00 00]; + apple,tunable-agx-af-rd-bw-dvfs-filter = [fe 7f 00 00]; + apple,tunable-firmware-name = [74 38 31 34 32 70 6d 70 00]; + apple,tunable-lts-clpc-voltage = [01 00 00 00 98 03 00 00 00 00 69 00 33 33 69 00 00 00 00 00 50 41 43 43 00 00 00 00]; + apple,tunable-soc-ni5-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-energy-counter = [01 00 00 00 01 00 00 00 05 00 00 00 01 00 00 00 41 47 58 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 05 00 00 00 02 00 00 00 41 47 58 2d 53 52 41 4d 00 00 00 00 00 00 00 00 03 00 00 00 01 00 00 00 06 00 00 00 01 00 00 00 41 4e 45 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 14 00 00 00 00 00 00 00 56 44 45 43 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 53 4f 43 5f 41 4f 4e 00 00 00 00 00 00 00 00 00]; + apple,tunable-lts-config = [01 00 00 00 3b 01 00 00 a6 72 0f 00 00 00 00 00 b5 d5 0f 00 6e 00 00 00 69 00 00 00 00 00 11 00 33 b3 00 00 b0 32 47 01 02 00 00 00 47 03 00 00 3b 61 00 00 00 00 00 00 f7 dd 0f 00 6e 00 00 00 69 00 00 00 00 00 11 00 33 b3 00 00 d7 43 ef 02]; + apple,tunable-agx-af-rd-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00]; + apple,tunable-ane-af-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00]; + apple,tunable-mcache-stream = [0e 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0d 00 00 00 38 00 00 00 38 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 11 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3c 00 00 00 3c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1b 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3d 00 00 00 3d 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 13 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 40 00 00 00 40 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 24 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 41 00 00 00 41 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1c 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 42 00 00 00 42 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1e 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2b 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0e 00 00 00 0e 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 05 00 00 00 01 00 00 00 01 00 00 00 07 00 00 00 07 00 00 00 20 00 00 00 20 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 06 00 00 00 02 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 24 00 00 00 24 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0b 00 00 00 05 00 00 00 01 00 00 00 0b 00 00 00 0b 00 00 00 30 00 00 00 30 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 01 00 00 00 11 00 00 00 11 00 00 00 48 00 00 00 48 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 13 00 00 00 13 00 00 00 50 00 00 00 50 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 15 00 00 00 15 00 00 00 58 00 00 00 5a 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 10 00 00 00 04 00 00 00 01 00 00 00 1f 00 00 00 1f 00 00 00 80 00 00 00 bf 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 04 00 00 00 02 00 00 00 10 00 00 00 10 00 00 00 44 00 00 00 44 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 20 00 00 00 03 00 00 00 02 00 00 00 03 00 00 00 06 00 00 00 10 00 00 00 1f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 1f 00 00 00 03 00 00 00 02 00 00 00 1e 00 00 00 1e 00 00 00 7c 00 00 00 7c 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 12 00 00 00 05 00 00 00 02 00 00 00 09 00 00 00 09 00 00 00 28 00 00 00 28 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0a 00 00 00 05 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 34 00 00 00 34 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 15 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5d 00 00 00 5d 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 36 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5e 00 00 00 5e 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 37 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5f 00 00 00 5f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 19 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 05 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 29 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 06 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 07 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0c 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 09 00 00 00 09 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 14 00 00 00 04 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 0a 00 00 00 0a 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 27 00 00 00 04 00 00 00 03 00 00 00 1d 00 00 00 1d 00 00 00 78 00 00 00 78 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2c 00 00 00 03 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 64 00 00 00 64 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2f 00 00 00 02 00 00 00 03 00 00 00 1a 00 00 00 1a 00 00 00 6c 00 00 00 6c 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 26 00 00 00 05 00 00 00 03 00 00 00 1c 00 00 00 1c 00 00 00 74 00 00 00 74 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 23 00 00 00 00 00 00 00 03 00 00 00 14 00 00 00 14 00 00 00 54 00 00 00 54 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 28 00 00 00 00 00 00 00 03 00 00 00 1b 00 00 00 1b 00 00 00 70 00 00 00 70 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 31 00 00 00 0f 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 60 00 00 00 60 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 30 00 00 00 0e 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 61 00 00 00 61 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 33 00 00 00 11 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 62 00 00 00 62 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 34 00 00 00 12 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 63 00 00 00 63 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 35 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 68 00 00 00 68 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 1d 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 69 00 00 00 6b 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2d 00 00 00 0c 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 65 00 00 00 65 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 16 00 00 00 0b 00 00 00 03 00 00 00 ff ff 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00]; + apple,tunable-ptd-driver-version = [01 00 00 00]; + apple,tunable-afi-top-bw-threshold = [99 19 3a 00 00 00 00 00 00 00 00 00 00 00 00 00 51 78 4d 00 01 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 51 78 4c 00 00 00 00 00 00 00 00 00 00 00 00 00 33 33 73 00 00 00 00 00 00 00 00 00 00 00 00 00 b8 1e 7c 00 00 00 00 00]; + apple,tunable-region-size = [00 00 0d 00 00 00 00 00]; + apple,tunable-coredump-enable = [40 00 00 00]; + apple,tunable-soc-ni4-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-pm-ptd-ranges = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 00 00 00 00 0f 00 00 00]; + apple,tunable-fast-die-ctrl-ts-avg-temp = [00 00 64 00]; + apple,tunable-lts-ctrl-loop = [02 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 02 00 00 00 01 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 47 50 55 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-soc-device = [01 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 02 00 00 00 45 4e 4f 4e 01 00 00 00 16 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4d 43 43 00 00 00 00 17 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 00 18 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 00 19 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 00 1a 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 00 1b 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 00 1c 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 00 10 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 41 4f 4e 00 11 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 52 45 53 54 03 00 00 00 45 4e 4f 4e 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 45 41 43 43 30 00 00 00 04 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 30 00 00 00 05 00 00 00 5f 58 46 47 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 00 00 00 00 00 06 00 00 00 45 4e 4f 4e 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 30 00 00 00 00 07 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 49 53 50 00 00 00 00 00 08 00 00 00 45 4e 4f 4e 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 00 00 00 00 09 00 00 00 45 4e 4f 4e 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 30 0a 00 00 00 45 4e 4f 4e 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 31 0b 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 00 00 00 00 00 0c 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 14 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 44 00 00 00 00 00 0d 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 53 52 00 00 00 00 00 0e 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4a 50 45 47 00 00 00 00 0f 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 43 4f 44 45 43 00 00 1d 00 00 00 5f 52 54 4c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4c 54 52 00 00 00 00 00 1e 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 30 00 00 00 00 1f 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 31 00 00 00 00 20 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 32 00 00 00 00 21 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 33 00 00 00 00 22 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 00 00 00 00 00 23 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 43 49 45 00 00 00 00 24 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 42 52 00 00 00 00 00 25 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 45 50 00 00 00 00 00 26 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 49 4f 00 00 00 00 00]; + apple,tunable-fast-die-ctrl-ts = [00 00 05 00]; + apple,tunable-region-base = [00 00 50 80 03 00 00 00]; + apple,tunable-agx-af-wr-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 35 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 34 00 00 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 00 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 00 00 00 00 00 00 3a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 39 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 2d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 2c 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 11 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3c 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 33 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 7a 00 00 00 00 00 00 00 32 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 79 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 2c 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 8a 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 89 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3e 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 6f 00 00 00 00 00 00 00 3d 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 6e 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 34 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 33 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2c 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 41 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 40 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3c 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 37 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 36 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00]; + apple,tunable-uuid = [30 41 37 44 41 39 43 33 2d 45 37 46 36 2d 33 31 35 31 2d 41 43 44 45 2d 44 41 30 31 37 42 45 42 45 36 34 35 00]; + apple,tunable-segment-names = [5f 5f 54 45 58 54 3b 5f 5f 44 41 54 41 00]; + apple,tunable-segment-ranges = [00 00 50 80 03 00 00 00 00 00 00 01 00 00 00 00 00 00 50 80 03 00 00 00 00 c0 03 00 03 00 00 00 00 c0 53 80 03 00 00 00 00 c0 03 01 00 00 00 00 00 c0 53 80 03 00 00 00 00 e0 06 00 06 00 00 00]; + apple,tunable-pre-loaded = [01 00 00 00]; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi new file mode 100644 index 0000000000000..fa080be75e8fa --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (13-inch, M5) J813: SMC power off / restart, the PMU on + * SPMI and the watchdog. + */ + +&{/soc} { + /* ADT /arm-io/wdt (wdt,t8142): restart (and the SMC's fallback). */ + wdt: watchdog@3882b0000 { + compatible = "apple,t8142-wdt", "apple,t8103-wdt"; + reg = <0x3 0x882b0000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; +}; + +&{/soc} { + /* + * ADT /arm-io/nub-spmi0 (gen 4) and its primary PMU, pmu-main + * ("abbey", slave 0xe). The boot flags live in the legacy scratchpad + * (info-leg_scrpad = 0xf800), laid out as on T8103 and J873. + */ + nub_spmi0: spmi@388714000 { + compatible = "apple,t8142-spmi"; + reg = <0x3 0x88714000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + + pmic@e { + compatible = "apple,abbey-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + boot_stage: boot-stage@f801 { + reg = <0xf801 0x1>; + }; + + boot_error_count: boot-error-count@f802,0 { + reg = <0xf802 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@f802,4 { + reg = <0xf802 0x1>; + bits = <4 4>; + }; + + /* Set before off1, or the machine powers back on. */ + shutdown_flag: shutdown-flag@f80f,3 { + reg = <0xf80f 0x1>; + bits = <3 1>; + }; + }; + }; + }; +}; + +&smc { + /* Power off (MBSE = off1) and restart (MBSE = phra). */ + reboot { + compatible = "apple,smc-reboot"; + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813-psci-idle.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-psci-idle.dtsi new file mode 100644 index 0000000000000..9c9cf88a0267b --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-psci-idle.dtsi @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * CPU idle through PSCI CPU_SUSPEND when aura runs the OS at EL1 under its + * monitor (aura's /psci node, "method = smc", enable-method = "psci"). The + * monitor's CPU_SUSPEND is a WFI at EL2 that preserves the registers WFI + * wipes on these cores, so this one retention state replaces idle=nop. + * Without a /psci node (raw aura, spin-table) cpuidle-psci does not probe. + */ + +&{/cpus} { + idle-states { + entry-method = "psci"; + + cpu_retention: cpu-retention { + compatible = "arm,idle-state"; + arm,psci-suspend-param = <0x0>; + entry-latency-us = <10>; + exit-latency-us = <10>; + min-residency-us = <100>; + }; + }; +}; + +&cpu0 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu1 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu2 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu3 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu4 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu5 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu6 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu7 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu8 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu9 { + cpu-idle-states = <&cpu_retention>; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 576c4637f37df..dd1526e6caad1 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -180,6 +180,36 @@ opp-hz = /bits/ 64 <972000000>; opp-level = <2>; }; + + opp-1152000000 { + opp-hz = /bits/ 64 <1152000000>; + opp-level = <3>; + }; + + opp-1584000000 { + opp-hz = /bits/ 64 <1584000000>; + opp-level = <4>; + }; + + opp-1992000000 { + opp-hz = /bits/ 64 <1992000000>; + opp-level = <5>; + }; + + opp-2352000000 { + opp-hz = /bits/ 64 <2352000000>; + opp-level = <6>; + }; + + opp-2700000000 { + opp-hz = /bits/ 64 <2700000000>; + opp-level = <7>; + }; + + opp-2964000000 { + opp-hz = /bits/ 64 <2964000000>; + opp-level = <8>; + }; }; pcluster_opp: opp-table-1 { @@ -235,6 +265,53 @@ opp-hz = /bits/ 64 <3516000000>; opp-level = <10>; }; + + /* ADT levels 11-20, measured on J813 (level 19, 4416 MHz, is below + * level 18 and left out). The SoC throttles them when hot. */ + opp-3720000000 { + opp-hz = /bits/ 64 <3720000000>; + opp-level = <11>; + }; + + opp-3876000000 { + opp-hz = /bits/ 64 <3876000000>; + opp-level = <12>; + }; + + opp-4020000000 { + opp-hz = /bits/ 64 <4020000000>; + opp-level = <13>; + }; + + opp-4176000000 { + opp-hz = /bits/ 64 <4176000000>; + opp-level = <14>; + }; + + opp-4248000000 { + opp-hz = /bits/ 64 <4248000000>; + opp-level = <15>; + }; + + opp-4272000000 { + opp-hz = /bits/ 64 <4272000000>; + opp-level = <16>; + }; + + opp-4332000000 { + opp-hz = /bits/ 64 <4332000000>; + opp-level = <17>; + }; + + opp-4428000000 { + opp-hz = /bits/ 64 <4428000000>; + opp-level = <18>; + }; + + opp-4464000000 { + opp-hz = /bits/ 64 <4464000000>; + opp-level = <20>; + }; }; clkref: clock-ref { @@ -262,6 +339,30 @@ #performance-domain-cells = <0>; }; + /* + * PMC voter interface (AppleT8142PMC, ADT pmc-pmgr 31 + 0x700000): + * floors at 0x382700000 + agent * 0x4000, per-agent states at + * +0x106000, rail enables at +0x108000. Agents (ADT voter-agent): + * PMP 0, CLPC 1, GFX 2, AP 3. iBoot leaves every enable at 0, so + * no vote is honoured and DRAM and fabric stay at their lowest + * states (CPU DRAM latency 171 ns instead of 70 ns). + */ + pmc: power-management@382700000 { + compatible = "apple,t8142-pmc"; + reg = <0x3 0x82700000 0x0 0x10000>, + <0x3 0x82806000 0x0 0x3000>; + reg-names = "voters", "rails"; + #address-cells = <1>; + #size-cells = <0>; + /* PMP, CLPC and AP agents */ + apple,static-voters = <0 1 3>; + + clpc: performance-controller@1 { + compatible = "apple,t8142-clpc"; + reg = <1>; + }; + }; + aic: interrupt-controller@381000000 { compatible = "apple,t8142-aic3", "apple,t8122-aic3"; interrupt-controller; @@ -293,5 +394,13 @@ }; #include "t8142-j813-display.dtsi" +#include "t8142-j813-gpu.dtsi" #include "t8142-j813-peripherals.dtsi" #include "t8142-j813-usb.dtsi" +#include "t8142-j813-avd.dtsi" +#include "t8142-j813-isp.dtsi" +#include "t8142-j813-psci-idle.dtsi" +#include "t8142-j813-nvme.dtsi" +#include "t8142-j813-n1.dtsi" +#include "t8142-j813-power.dtsi" +#include "t8142-j813-audio.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8152-j873-gpu.dtsi b/arch/arm64/boot/dts/apple/t8152-j873-gpu.dtsi new file mode 100644 index 0000000000000..2166be04d4111 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8152-j873-gpu.dtsi @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple Mac mini (M6), J873: G18G GPU resources of the T8152. + * + * Addresses and sizes follow the 26A428 ADT /arm-io/sgx, gfx-asc and + * gfx1-asc nodes. This only describes the hardware; it does not start the + * firmware or change the GPU's power state. + * + * Unless a loader prepares the boot, nothing here starts the GPU: + * - the driver's G18 backend binds "apple,agx-t8152", but refuses with + * ENODEV before any firmware access unless the loader's /chosen + * aurora,g18g-gpu-firmware-profile receipt matches, and before any MMIO + * with aurora.g18_initdata_test=0; + * - the eight reserved regions are boot-local. They stay disabled with a + * zero reg until the loader copies this boot's /arm-io/sgx base and size + * values into them and sets status = "okay" (as m1n1 dt_set_resvmem() + * does). The loader finds them by these node names; + * - the four firmware segment carveouts (GFX/GFX1 __DATA and + * __DATA_SHARED_RO) are no-map, so they never enter the linear map. The + * driver only takes their resources, and maps __DATA read-only to scan + * for a crash log. The four UAT regions stay mapped: the driver uses + * their pages as System RAM; + * - the GFX and GFX1 ASCWrap v8 mailboxes stay disabled. The loader enables + * both together with the /chosen aurora,g18g-gpu-firmware-profile + * property, and only after it has checked the loaded firmware images. + * Even then the provider only maps its windows: ASC CPU control is + * refused unless apple_mailbox.ascwrap_v8_cpu_control=1 is given, and the + * receive side is polled, with no interrupt enabled, unless + * apple_mailbox.ascwrap_v8_rx_irq=1 is given. + * + * No power-domain is attached. The GPU is powered by PMGR2 PSD services 3 + * (sgx), 81 (gfx-asc) and 82 (gfx1-asc). Their 0x100-byte service blocks + * are listed for state reads (state +0x80, ack +0xbc). + */ + +/* Loaders locate the node through this alias (m1n1 dt_set_gpu()). */ +&{/aliases} { + gpu = &gpu; +}; + +&{/reserved-memory} { + /* sgx gpu-region */ + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + status = "disabled"; + }; + + /* sgx gfx-shared-region (0x80000) */ + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + status = "disabled"; + }; + + /* sgx gfx-handoff */ + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + status = "disabled"; + }; + + /* sgx gfx-shared-l2-region */ + uat_l2_pagetables: uat-l2-pagetables { + reg = <0 0 0 0>; + status = "disabled"; + }; + + /* GFX __DATA (0xbc000 on 26A428) */ + gfx_data: gfx-data { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + /* GFX1 __DATA (0xb4000 on 26A428) */ + gfx1_data: gfx1-data { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + /* GFX __DATA_SHARED_RO (0x8000, firmware VA +0x10c000), never host-writable */ + gfx_data_shared_ro: gfx-data-shared-ro { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + /* GFX1 __DATA_SHARED_RO (0x8000, firmware VA +0x108000), never host-writable */ + gfx1_data_shared_ro: gfx1-data-shared-ro { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; +}; + +&{/soc} { + gpu: gpu@580000000 { + compatible = "apple,agx-t8152"; + /* + * ADT sgx reg[0]. ADT reg[1] (0x580d00000/0x180000) lies inside + * it and is reached through this window. The PSD service blocks + * follow. + */ + reg = <5 0x80000000 0 0x4000000>, + <3 0x03a10300 0 0x100>, + <3 0x03a15100 0 0x100>, + <3 0x03a15200 0 0x100>; + reg-names = "sgx", "psd-sgx", "psd-gfx-asc", "psd-gfx1-asc"; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, + <&uat_l2_pagetables>, <&gfx_data>, <&gfx1_data>, + <&gfx_data_shared_ro>, <&gfx1_data_shared_ro>; + memory-region-names = "ttbs", "pagetables", "handoff", "shared-l2", + "gfx-data", "gfx1-data", + "gfx-data-shared-ro", "gfx1-data-shared-ro"; + mboxes = <&gfx_asc>, <&gfx1_asc>; + }; + + /* + * ADT gfx-asc: iop,ascwrap-v8, msgbox-mailbox-num = 1, alias, + * postoffice. Only this role's 0x80-byte post-office window + * (0x316020000 + (2 * 1 + 0) * 0x80) is claimed; the rest of the + * 0x316020000 bank belongs to PMP, DCP, DCPEXT and SCALER0. + * Interrupt index 0 is send-empty and index 1 recv-not-empty, as in + * the macOS ASCWrap v8 driver. The provider polls by default. + */ + gfx_asc: mailbox@582600000 { + compatible = "apple,t8152-gfx-ascwrap-v8"; + reg = <5 0x82600000 0 0x74000>, <5 0x82050000 0 8>, + <3 0x16020100 0 0x80>, <3 0x03a15100 0 0x100>; + reg-names = "wrapper", "iop-vbar", "mailbox", "psd"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + #mbox-cells = <0>; + status = "disabled"; + }; + + /* ADT gfx1-asc: msgbox-mailbox-num = 2, window 0x316020200. */ + gfx1_asc: mailbox@582e00000 { + compatible = "apple,t8152-gfx-ascwrap-v8"; + reg = <5 0x82e00000 0 0x74000>, <5 0x82850000 0 8>, + <3 0x16020200 0 0x80>, <3 0x03a15200 0 0x100>; + reg-names = "wrapper", "iop-vbar", "mailbox", "psd"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX1"; + #mbox-cells = <0>; + status = "disabled"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8152-j873-n1.dtsi b/arch/arm64/boot/dts/apple/t8152-j873-n1.dtsi new file mode 100644 index 0000000000000..ffa33863f782b --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8152-j873-n1.dtsi @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J873g N1 on APCIE port 0. Aura releases the SMC-controlled N1 reset, + * initializes the common PHY and applies the inactive DART's tunables. + * Linux owns port training and the three function-specific DMA streams. + * Resources follow the J873g ADT and the public J873 PCIe bring-up tree. + */ +#include + +/ { + soc { + pinctrl_ap: pinctrl@31a000000 { + compatible = "apple,t8152-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x1a000000 0x0 0x100000>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 224>; + apple,npins = <224>; + interrupt-controller; + #interrupt-cells = <2>; + interrupts = , + , + , + , + , + , + ; + }; + + pcie0_dart: iommu@388000000 { + compatible = "apple,t8152-dart", "apple,t8110-dart"; + reg = <0x3 0x88000000 0x0 0xc000>; + #iommu-cells = <1>; + interrupts = ; + }; + + pcie: pcie@1cb0000000 { + compatible = "apple,t8152-pcie"; + device_type = "pci"; + reg = <0x1c 0xb0000000 0x0 0x1000000>, + <0x3 0x8c000000 0x0 0x4000>, + <0x3 0x88028000 0x0 0x4000>, + <0x3 0x8f010000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "phy0"; + interrupts = ; + msi-controller; + msi-parent = <&pcie>; + msi-ranges = <&aic AIC_IRQ 1879 IRQ_TYPE_EDGE_RISING 32>; + iommu-map = <0x100 &pcie0_dart 1 1>, + <0x101 &pcie0_dart 2 1>, + <0x102 &pcie0_dart 3 1>; + iommu-map-mask = <0xffff>; + bus-range = <0 1>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0x0 0x20000000>, + <0x02000000 0x0 0x80000000 0xb 0x80000000 0x0 0x40000000>; + dma-coherent; + + port00: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 3 GPIO_ACTIVE_LOW>; + bus-range = <1 1>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8152-j873-pmc.dtsi b/arch/arm64/boot/dts/apple/t8152-j873-pmc.dtsi new file mode 100644 index 0000000000000..3b809b363b84e --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8152-j873-pmc.dtsi @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT + +/ { + soc { + /* + * Perf-state floors of voter agents 0-3 (PMP, CLPC, GFX, AP) and, + * at 0x1c6000, the honoured states and per-rail voter enables + * (rails DCS, SOC, DISP) in the PMC window, pmgr reg 46. + */ + pmc: power-management@302800000 { + compatible = "apple,t8152-pmc"; + reg = <0x3 0x02800000 0x0 0x10000>, + <0x3 0x029c6000 0x0 0x3000>; + reg-names = "voters", "rails"; + #address-cells = <1>; + #size-cells = <0>; + #cooling-cells = <2>; + /* PMP, CLPC and AP agents */ + apple,static-voters = <0 1 3>; + + clpc: performance-controller@1 { + compatible = "apple,t8152-clpc"; + reg = <1>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8152-j873-qspi.dtsi b/arch/arm64/boot/dts/apple/t8152-j873-qspi.dtsi new file mode 100644 index 0000000000000..c03fbf04b24e4 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8152-j873-qspi.dtsi @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT + +/* + * The boot NOR behind the QSPIMC controller (ADT /arm-io/qspi/spinor: firmware, + * anvram, syscfg), for asahi-nvram and asahi-bless. The controller's power + * states (PMGR devices SPI_P and QSPI) are off once iBoot is done with it; + * reading it without them takes an SError. + */ +/ { + aliases { + nvram = &nvram; + }; + + /* QSPIMC input clock, as on T8140 (ADT clock-ids 412) */ + clk_200m: clock-200m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_200m"; + }; + + soc { + pmgr: power-management@300700000 { + compatible = "apple,t8152-pmgr", "apple,t8103-pmgr", + "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x3 0x700000 0x0 0x14000>; + }; + + qspi: spi@331118000 { + compatible = "apple,t8152-qspi", "apple,t8140-qspi"; + /* One aperture; the command block starts at offset 0x800. */ + reg = <0x3 0x31118000 0x0 0x4800>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_200m>; + power-domains = <&ps_qspi>; + #address-cells = <1>; + #size-cells = <0>; + + spinor: flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <10000000>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + /* + * 64 KiB NVRAM v3 banks ("3VVN") from 0x24000 + * to 0x1f4000, as on T8140 J700. + */ + nvram: partition@24000 { + label = "nvram"; + reg = <0x24000 0x1d0000>; + }; + }; + }; + }; + }; +}; + +&pmgr { + ps_spi_p: power-controller@d40 { + compatible = "apple,t8152-pmgr-pwrstate", + "apple,t8103-pmgr-pwrstate"; + reg = <0xd40 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + + ps_qspi: power-controller@1140 { + compatible = "apple,t8152-pmgr-pwrstate", + "apple,t8103-pmgr-pwrstate"; + reg = <0x1140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts index 0a0818d85d085..1513a2abbc83f 100644 --- a/arch/arm64/boot/dts/apple/t8152-j873g.dts +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -20,7 +20,7 @@ #size-cells = <2>; interrupt-parent = <&aic>; - /* Request the m1n1 RAM-only handoff, without PCIe/DAPF setup. */ + /* Retain the RAM handoff and secondary CPU park contract. */ aurora,minimal-boot; aliases { @@ -45,7 +45,10 @@ #address-cells = <2>; #size-cells = <2>; ranges; - /* Filled by the loader. */ + smc-firmware@30de00000 { + reg = <3 0x0de00000 0 0x120000>; + no-map; + }; }; memory@10000000000 { @@ -238,6 +241,35 @@ #performance-domain-cells = <0>; }; + smc_mbox: mailbox@30c710080 { + compatible = "apple,t8152-smc-mailbox"; + reg = <3 0x0c710080 0 0x80>; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@30c600000 { + compatible = "apple,t8152-smc"; + reg = <3 0x0c600000 0 0x74000>, + <3 0x0de00000 0 0x120000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + hwmon { + compatible = "apple,smc-hwmon"; + apple,discover-temperatures; + + fan-F0Ac { + apple,key-id = "F0Ac"; + apple,fan-minimum = "F0Mn"; + apple,fan-maximum = "F0Mx"; + label = "Fan"; + }; + }; + }; + dockchannel: serial@30812c000 { compatible = "apple,dockchannel-uart"; reg = <3 0x0812c000 0 0x4000>, @@ -248,3 +280,9 @@ }; }; }; + +#include "t8152-j873-gpu.dtsi" +#include "t8152-j873-n1.dtsi" + +#include "t8152-j873-pmc.dtsi" +#include "t8152-j873-qspi.dtsi" diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 1d12b124fd7dc..bd53d897b765a 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2625,6 +2625,31 @@ test_has_gicv5_legacy(const struct arm64_cpu_capabilities *entry, int scope) return !!(read_sysreg_s(SYS_ICC_IDR0_EL1) & ICC_IDR0_EL1_GCIE_LEGACY); } +/* + * Apple T8152 (M6) cores clear architectural state across WFIT, the FP/SIMD + * registers included. __delay() would execute it from any context, including + * the cpufreq transition polling that load-driven governors run on a CPU + * whose current task still has live FP/SIMD state: on J873 (M6) 25-56% of + * SHA-256 runs gave wrong results under schedutil. Treat WFxT as absent + * there, as arm64.nowfxt does, so __delay() uses WFE. T8142 (M5) was not + * measured; it is included because apple_m5_needs_wait_save() groups its + * cores with T8152 for this state loss. + */ +static bool has_wfxt(const struct arm64_cpu_capabilities *entry, int scope) +{ + static const struct midr_range apple_wfxt_loses_state[] = { + MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x062)), + MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x063)), + MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x072)), + MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x073)), + {} + }; + + if (is_midr_in_range_list(apple_wfxt_loses_state)) + return false; + return has_cpuid_feature(entry, scope); +} + static const struct arm64_cpu_capabilities arm64_features[] = { #ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL { @@ -3129,7 +3154,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .desc = "WFx with timeout", .capability = ARM64_HAS_WFXT, .type = ARM64_CPUCAP_SYSTEM_FEATURE, - .matches = has_cpuid_feature, + .matches = has_wfxt, ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, WFxT, IMP) }, { diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index f4ad1c3e95968..267677adc220a 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -53,10 +53,22 @@ static bool apple_wfi_loses_regs __ro_after_init; void __init apple_wfi_quirk_init(void) { + u32 model = read_cpuid_id() & MIDR_CPU_MODEL_MASK; + if (midr_is_apple_m4_or_later(read_cpuid_id())) { apple_wfi_loses_regs = true; pr_info("Apple: preserving registers across WFI\n"); } + /* + * aura passes idle=nop on M5 for kernels without this quirk. Spinning + * idle keeps all ten cores busy (cpuidle's WFI state spins too), and + * the SoC sits near 90 C and throttles; WFI is safe here. + */ + if (idle == ARM64_IDLE_NOP && + (model == MIDR_APPLE_T8142_E || model == MIDR_APPLE_T8142_P)) { + idle = ARM64_IDLE_WFI; + pr_info("Apple: T8142 idles in WFI, ignoring idle=nop\n"); + } } static __always_inline bool apple_wfi_needs_save(void) diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index b35637d575a34..832e2d3019ecc 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -506,4 +506,18 @@ config BT_INTEL_PCIE Say Y here to compiler support for Intel Bluetooth PCIe device into the kernel or say M to compile it as module (btintel_pcie) + +config BT_CENTAURI + tristate "Apple N1 Beta Bluetooth transport (experimental)" + depends on PCI && (ARCH_APPLE || COMPILE_TEST) + help + Experimental Bluetooth transport for Apple N1 on J813. + The Centauri Control driver must boot the shared firmware before this + driver binds to Beta. Controller initialization, discovery, BR/EDR ACL, + pairing and encrypted AVDTP discovery have been tested on hardware. + ACL delivery and native startup/connection handling remain opt-in; + audio playback and LE data are unqualified. SCO/ISO is not implemented. + + Say M to build the btcentauri module for bring-up testing. + endmenu diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile index e0f1cea5b6092..74c3834c2c4a3 100644 --- a/drivers/bluetooth/Makefile +++ b/drivers/bluetooth/Makefile @@ -7,8 +7,12 @@ obj-$(CONFIG_BT_HCIVHCI) += hci_vhci.o obj-$(CONFIG_BT_HCIUART) += hci_uart.o obj-$(CONFIG_BT_HCIBCM203X) += bcm203x.o obj-$(CONFIG_BT_HCIBCM4377) += hci_bcm4377.o +obj-$(CONFIG_BT_HCIMT79XX_PCIE) += btmtk_pcie.o obj-$(CONFIG_BT_HCIBPA10X) += bpa10x.o obj-$(CONFIG_BT_HCIBFUSB) += bfusb.o +obj-$(CONFIG_BT_HCIDTL1) += dtl1_cs.o +obj-$(CONFIG_BT_HCIBT3C) += bt3c_cs.o +obj-$(CONFIG_BT_HCIBLUECARD) += bluecard_cs.o obj-$(CONFIG_BT_HCIBTUSB) += btusb.o obj-$(CONFIG_BT_HCIBTSDIO) += btsdio.o @@ -50,7 +54,4 @@ hci_uart-$(CONFIG_BT_HCIUART_AG6XX) += hci_ag6xx.o hci_uart-$(CONFIG_BT_HCIUART_MRVL) += hci_mrvl.o hci_uart-$(CONFIG_BT_HCIUART_AML) += hci_aml.o hci_uart-objs := $(hci_uart-y) - -obj-$(CONFIG_BT_MTK7932_PCIE) += mt7932_bt_pcie.o - -CONTEXT_ANALYSIS := y +obj-$(CONFIG_BT_CENTAURI) += btcentauri.o diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c new file mode 100644 index 0000000000000..5830c8b2285e4 --- /dev/null +++ b/drivers/bluetooth/btcentauri.c @@ -0,0 +1,1041 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Apple N1 Beta command/event transport for J813 bring-up. + * + * The Centauri Control driver must have booted the shared firmware first. + * This driver owns only Beta. It never resets the port or the Wi-Fi function. + * BR/EDR discovery, ACL/SDP, pairing and encrypted AVDTP discovery are qualified. + * ACL remains opt-in; LE data, persistent bonds and audio are unqualified. + * SCO/ISO delivery is not implemented. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CEN_ARENA_SIZE SZ_1M +#define CEN_MTR 0x2000 +#define CEN_MCR 0x2200 +#define CEN_MESSAGES 0x4000 +#define CEN_RX 0x8000 +#define CEN_TX 0xa000 +#define CEN_FOOTER 260 +#define CEN_STRIDE (16 + CEN_FOOTER) +#define CEN_RX_SLOTS 17 +#define CEN_TX_SLOTS 5 +#define CEN_LE_RX 0xc000 +#define CEN_LE_TX 0xe000 +#define CEN_LE_FOOTER 256 +#define CEN_LE_STRIDE (16 + CEN_LE_FOOTER) +#define CEN_LE_SLOTS 17 +#define CEN_MESSAGE_SLOTS 16 +#define CEN_AUX_BASE 0x20000 +#define CEN_CAPTURE_SIZE SZ_1M + +static bool arm_le_data_rings = true; +module_param(arm_le_data_rings, bool, 0444); +MODULE_PARM_DESC(arm_le_data_rings, "Arm native Beta LE rings for radio bring-up"); + +static bool arm_auxiliary_rings; +module_param(arm_auxiliary_rings, bool, 0444); +MODULE_PARM_DESC(arm_auxiliary_rings, "Diagnostic: arm remaining native Beta resources and capture received frames privately"); + +static bool native_startup; +module_param(native_startup, bool, 0444); +MODULE_PARM_DESC(native_startup, "Diagnostic: program a temporary address, InitDone and random seed before the first HCI reset"); + +static bool enable_acl; +module_param(enable_acl, bool, 0444); +MODULE_PARM_DESC(enable_acl, "Experimental ACL data delivery; requires LE and auxiliary rings"); + +static bool native_hrb; +module_param(native_hrb, bool, 0444); +MODULE_PARM_DESC(native_hrb, "Experimental native Proxima Create Connection; requires native startup, ACL and firmware capability bit 9"); + +#define CEN_OP_READ_PROP_FEATURES 0xffa3 +#define CEN_OP_HRB_CREATE_CONN 0xff7b +#define CEN_PROP_HRB_CREATE BIT(9) + +struct cen_aux_spec { + u16 id, slots, footer; + bool receive; +}; + +/* Host native Beta inventory; each resource is opt-in until target-qualified. + * HCI and LE rings remain in their existing, separately qualified arenas. + */ +static const struct cen_aux_spec cen_aux_specs[] = { + { 2, 9, 2592, false }, /* ACL TX */ + { 3, 9, 724, false }, /* SCO TX */ + { 5, 9, 260, false }, /* LEA TX */ + { 6, 5, 1024, false }, /* LLA TX */ + { 8, 9, 2592, true }, /* ACL RX */ + { 9, 9, 724, true }, /* SCO RX */ + { 11, 9, 268, true }, /* LEA RX */ + { 12, 128, 1024, true }, /* Firmware debug RX */ + { 13, 17, 260, true }, /* LMP RX */ + { 14, 9, 1304, false }, /* TSI TX */ + { 15, 9, 1304, true }, /* TSI RX */ +}; + +struct cen_aux_state { + u32 offset, packets; + u16 head, post; +}; + +struct cen_bt { + struct pci_dev *pdev; + struct hci_dev *hdev; + struct dentry *debug; + void __iomem *bar; + u8 *arena; + dma_addr_t dma; + struct mutex lock; + struct delayed_work poll; + struct sk_buff_head txq; + struct sk_buff_head acl_txq, le_txq; + u16 acl_tx_head, le_tx_head; + u16 message_head, message_tag, tx_head, rx_head, rx_post; + u16 le_rx_head, le_rx_post; + u32 le_rx_packets; + struct cen_aux_state aux[ARRAY_SIZE(cen_aux_specs)]; + u8 *capture; + u32 capture_used, capture_records, capture_dropped; + bool ipc_requested, running; + u8 native_startup_steps; + bool hrb_supported, hrb_status_pending; + u32 proprietary_features, hrb_commands, hrb_status_events; + u32 hrb_connection_events; + u8 hrb_last_link_type; +}; + +static u16 cen_tail(struct cen_bt *bt, u16 ring) +{ + return le16_to_cpu(READ_ONCE(*(__le16 *)(bt->arena + 0x7a + ring * 2))); +} + +static void cen_post(struct cen_bt *bt, u16 ring, u16 value) +{ + put_unaligned_le16(value, bt->arena + 0xa8 + ring * 2); + dma_wmb(); + writel(value, bt->bar + 0x1000 + ring * 4); +} + +static int cen_message(struct cen_bt *bt, const void *data, size_t length) +{ + u16 head = bt->message_head, tail, next = (head + 1) % CEN_MESSAGE_SLOTS; + u16 tag = bt->message_tag + 1; + u8 *descriptor = bt->arena + CEN_MTR + head * 16; + u8 *completion = bt->arena + CEN_MCR + head * 32; + int ret; + + if (head >= CEN_MESSAGE_SLOTS || !length || length > 512) + return -EINVAL; + memset(descriptor, 0, 16); + memset(completion, 0, 32); + memcpy(bt->arena + CEN_MESSAGES + head * 512, data, length); + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(bt->dma + CEN_MESSAGES + head * 512, descriptor + 4); + put_unaligned_le16(tag, descriptor + 12); + cen_post(bt, 0, next); + ret = read_poll_timeout(cen_tail, tail, tail == next, + 100, 1500000, false, bt, 0); + if (ret) + return ret; + /* The device publishes the message completion before advancing its head. */ + ret = read_poll_timeout(le16_to_cpu, tail, tail == next, + 100, 1500000, false, + READ_ONCE(*(__le16 *)(bt->arena + 0x78))); + if (ret) + return ret; + dma_rmb(); + if (get_unaligned_le16(completion + 4) != tag || + get_unaligned_le16(completion + 6) != length || + get_unaligned_le32(completion + 8)) + return -EPROTO; + bt->message_head = next; + bt->message_tag = tag; + put_unaligned_le16(bt->message_head, bt->arena + 0xa6); + dma_wmb(); + return 0; +} + +static int cen_open_ring(struct cen_bt *bt, u16 id, u16 slots, u32 offset, u16 footer) +{ + u8 message[0x34] = { 1 }; + u32 words = footer / 4, shift = 0; + + if (!words || footer % 4) + return -EINVAL; + if (words > 255) + shift = min_t(u32, __ffs(words), 4); + if ((words >> shift) > 255) + return -EINVAL; + message[2] = words >> shift; + message[3] = shift << 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le64(bt->dma + offset, message + 8); + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(slots, message + 0x18); + put_unaligned_le16(0xffff, message + 0x1a); + put_unaligned_le16(id, message + 0x1c); + put_unaligned_le16(0x40, message + 0x1e); + return cen_message(bt, message, sizeof(message)); +} + +static void cen_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_RX + slot * CEN_STRIDE; + + memset(descriptor, 0, CEN_STRIDE); + put_unaligned_le32(1 | (CEN_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static void cen_le_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_LE_RX + slot * CEN_LE_STRIDE; + + memset(descriptor, 0, CEN_LE_STRIDE); + put_unaligned_le32(1 | (CEN_LE_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static void cen_capture_frame(struct cen_bt *bt, u16 ring, const u8 *data, u16 length) +{ + u8 *record; + + if (!bt->capture) + return; + if (8U + length > CEN_CAPTURE_SIZE - bt->capture_used) { + bt->capture_dropped++; + return; + } + record = bt->capture + bt->capture_used; + put_unaligned_le16(ring, record); + put_unaligned_le16(length, record + 2); + put_unaligned_le32(bt->capture_records++, record + 4); + memcpy(record + 8, data, length); + bt->capture_used += 8 + length; +} + +static int cen_receive_acl(struct cen_bt *bt, const u8 *data, u32 length) +{ + struct sk_buff *skb; + u32 packet_length, pos; + + if (length < HCI_ACL_HDR_SIZE) + return -EPROTO; + packet_length = HCI_ACL_HDR_SIZE + get_unaligned_le16(data + 2); + if (packet_length > length) + return -EPROTO; + /* Airship RX attributes follow, rather than belong to, the HCI packet. + * Only the native parser's known record sizes are accepted. Their values + * remain opaque and are retained only in the private diagnostic capture. + */ + for (pos = packet_length; pos < length;) { + if (!data[pos]) + break; + if (data[pos] == 1) + pos++; + else if (data[pos] == 2 && length - pos >= 9) + pos += 9; + else + return -EPROTO; + } + skb = bt_skb_alloc(packet_length, GFP_KERNEL); + if (!skb) { + bt->hdev->stat.err_rx++; + return 0; + } + skb_put_data(skb, data, packet_length); + hci_skb_pkt_type(skb) = HCI_ACLDATA_PKT; + bt->hdev->stat.byte_rx += packet_length; + hci_recv_frame(bt->hdev, skb); + return 0; +} + +static void cen_aux_rx_buffer(struct cen_bt *bt, unsigned int index, u16 slot) +{ + const struct cen_aux_spec *spec = &cen_aux_specs[index]; + u8 *descriptor = bt->arena + bt->aux[index].offset + slot * (16 + spec->footer); + + memset(descriptor, 0, 16 + spec->footer); + put_unaligned_le32(1 | (spec->footer << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static int cen_aux_start(struct cen_bt *bt) +{ + u32 offset = CEN_AUX_BASE; + unsigned int i, slot; + int ret; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u32 length = spec->slots * (16U + spec->footer); + + if (offset > CEN_ARENA_SIZE || length > CEN_ARENA_SIZE - offset) + return -ENOSPC; + state->offset = offset; + offset = ALIGN(offset + length, SZ_4K); + ret = cen_open_ring(bt, spec->id, spec->slots, state->offset, spec->footer); + if (ret) { + dev_err(&bt->pdev->dev, "Auxiliary ring %u open failed: %d\n", spec->id, ret); + return ret; + } + if (!spec->receive) + continue; + for (slot = 0; slot < spec->slots - 1; slot++) + cen_aux_rx_buffer(bt, i, slot); + state->post = spec->slots - 1; + cen_post(bt, spec->id, state->post); + } + return 0; +} + +static int cen_aux_drain(struct cen_bt *bt, bool repost) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u16 tail; + int budget = spec->slots; + + if (!spec->receive) + continue; + tail = cen_tail(bt, spec->id); + if (tail >= spec->slots) + return -EPROTO; + while (state->head != tail && budget--) { + u8 *descriptor = bt->arena + state->offset + state->head * (16 + spec->footer); + u32 length; + + dma_rmb(); + length = get_unaligned_le32(descriptor) >> 8; + if (length > spec->footer) + return -EPROTO; + cen_capture_frame(bt, spec->id, descriptor + 16, length); + if (repost && enable_acl && spec->id == 8 && + cen_receive_acl(bt, descriptor + 16, length)) + return -EPROTO; + state->packets++; + state->head = (state->head + 1) % spec->slots; + if (repost) { + cen_aux_rx_buffer(bt, i, state->post); + state->post = (state->post + 1) % spec->slots; + cen_post(bt, spec->id, state->post); + } + } + } + return 0; +} + +static int cen_start(struct cen_bt *bt) +{ + static const u16 indices[] = { 0x78, 0x7a, 0xa6, 0xa8 }; + u8 config[4] = { 9, 0, 0xff, 0xff }; + u32 status; + int i, ret; + + put_unaligned_le16(0x300, bt->arena); + put_unaligned_le16(0x68, bt->arena + 2); + put_unaligned_le64(bt->dma + 0x68, bt->arena + 8); + for (i = 0; i < ARRAY_SIZE(indices); i++) + put_unaligned_le64(bt->dma + indices[i], bt->arena + 0x10 + i * 8); + put_unaligned_le16(1, bt->arena + 0x30); + put_unaligned_le16(22, bt->arena + 0x32); + put_unaligned_le64(bt->dma + CEN_MCR, bt->arena + 0x34); + put_unaligned_le64(bt->dma + CEN_MTR, bt->arena + 0x3c); + put_unaligned_le16(16, bt->arena + 0x44); + put_unaligned_le16(16, bt->arena + 0x46); + /* The 25G76 native context encodes the MCR footer in four-byte words. */ + bt->arena[0x53] = 4; + put_unaligned_le64(bt->dma + 0x10000, bt->arena + 0x58); + pci_set_master(bt->pdev); + writel(lower_32_bits(bt->dma), bt->bar + 0xc); + writel(upper_32_bits(bt->dma), bt->bar + 0x10); + writel(CEN_ARENA_SIZE, bt->bar + 0x14); + bt->ipc_requested = true; + writel(1, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 1, 100, 2000000); + if (ret) + return ret; + dma_wmb(); + writel(lower_32_bits(bt->dma), bt->bar + 4); + writel(upper_32_bits(bt->dma), bt->bar + 8); + writel(2, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 2, 100, 2000000); + if (ret) + return ret; + ret = cen_message(bt, config, sizeof(config)); + if (!ret) + ret = cen_open_ring(bt, 7, CEN_RX_SLOTS, CEN_RX, CEN_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 1, CEN_TX_SLOTS, CEN_TX, CEN_FOOTER); + if (!ret && arm_le_data_rings) + ret = cen_open_ring(bt, 10, CEN_LE_SLOTS, CEN_LE_RX, CEN_LE_FOOTER); + if (!ret && arm_le_data_rings) + ret = cen_open_ring(bt, 4, CEN_LE_SLOTS, CEN_LE_TX, CEN_LE_FOOTER); + if (ret) + return ret; + for (i = 0; i < CEN_RX_SLOTS - 1; i++) + cen_rx_buffer(bt, i); + bt->rx_post = CEN_RX_SLOTS - 1; + cen_post(bt, 7, bt->rx_post); + if (arm_le_data_rings) { + for (i = 0; i < CEN_LE_SLOTS - 1; i++) + cen_le_rx_buffer(bt, i); + bt->le_rx_post = CEN_LE_SLOTS - 1; + cen_post(bt, 10, bt->le_rx_post); + } + if (arm_auxiliary_rings) + return cen_aux_start(bt); + return 0; +} + +static int cen_transmit(struct cen_bt *bt, struct sk_buff_head *queue, + u16 ring, u16 slots, u32 offset, u16 footer, u16 *head) +{ + struct sk_buff *skb; + u16 tail, next; + u8 *descriptor; + u32 wire_length; + + while (!skb_queue_empty(queue)) { + tail = cen_tail(bt, ring); + if (tail >= slots) + return -EPROTO; + next = (*head + 1) % slots; + if (next == tail) + break; + dma_rmb(); + skb = skb_dequeue(queue); + if (!skb) + break; + descriptor = bt->arena + offset + *head * (16 + footer); + memset(descriptor, 0, 16 + footer); + put_unaligned_le16(*head + 1, descriptor + 12); + memcpy(descriptor + 16, skb->data, skb->len); + wire_length = skb->len; + if (ring == 1 && READ_ONCE(bt->hrb_supported) && + get_unaligned_le16(skb->data) == HCI_OP_CREATE_CONN) { + u8 *command = descriptor + 16; + + /* Native Proxima uses the ordinary 13-byte parameter layout + * followed by two zero HRB fields (no forced band). It sends + * tagging=false and role_switch=0. Address byte order matches + * standard HCI. Keep the core's sent skb/opcode unchanged. + */ + put_unaligned_le16(CEN_OP_HRB_CREATE_CONN, command); + command[2] = 15; + command[HCI_COMMAND_HDR_SIZE + 9] = 0; + command[HCI_COMMAND_HDR_SIZE + 12] = 0; + wire_length = HCI_COMMAND_HDR_SIZE + 15; + bt->hrb_status_pending = true; + bt->hrb_commands++; + cen_capture_frame(bt, 1, command, wire_length); + } + put_unaligned_le32(1 | (wire_length << 8), descriptor); + *head = next; + cen_post(bt, ring, next); + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT) + bt->hdev->stat.cmd_tx++; + else + bt->hdev->stat.acl_tx++; + bt->hdev->stat.byte_tx += wire_length; + kfree_skb(skb); + } + return 0; +} + +/* The N1 reports legacy scan responses in LE Extended Advertising Reports with an event type whose low + * byte is 0x18 (legacy PDU and scan response, neither connectable nor scannable) and other bits in the + * high byte (seen: 0x2518, 0x2618, 0x2718). Linux accepts only the Core values 0x1a and 0x1b for a + * legacy scan response and drops anything else as "Unknown advertising packet type", so report the + * low byte 0x18 as 0x1a and keep the high byte. + */ +static void cen_normalize_ext_adv(struct sk_buff *skb) +{ + unsigned int offset = HCI_EVENT_HDR_SIZE + 2, reports; + + if (skb->len < offset + 1 || skb->data[0] != HCI_EV_LE_META || + skb->data[2] != HCI_EV_LE_EXT_ADV_REPORT) + return; + reports = skb->data[3]; + while (reports--) { + struct hci_ev_le_ext_adv_info *info; + + if (offset + sizeof(*info) > skb->len) + return; + info = (struct hci_ev_le_ext_adv_info *)(skb->data + offset); + if ((le16_to_cpu(info->type) & 0x00ff) == 0x0018) + info->type = cpu_to_le16((le16_to_cpu(info->type) & 0xff00) | 0x001a); + offset += sizeof(*info) + info->length; + } +} + +static void cen_poll(struct work_struct *work) +{ + struct cen_bt *bt = container_of(to_delayed_work(work), struct cen_bt, poll); + struct hci_dev *hdev = bt->hdev; + struct sk_buff *skb; + u16 tail; + u8 *descriptor; + u32 length; + int budget = CEN_RX_SLOTS; + + mutex_lock(&bt->lock); + if (!bt->running) + goto unlock; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 2) { + /* Preserve already completed diagnostics after abort, without posting + * any more buffers or ringing doorbells to the failed firmware. + */ + if (arm_auxiliary_rings && readl(bt->bar) == 2) + cen_aux_drain(bt, false); + goto protocol_error; + } + if (arm_auxiliary_rings && cen_aux_drain(bt, true)) + goto protocol_error; + if (arm_le_data_rings) { + tail = cen_tail(bt, 10); + if (tail >= CEN_LE_SLOTS) + goto protocol_error; + while (bt->le_rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_LE_RX + bt->le_rx_head * CEN_LE_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length > CEN_LE_FOOTER) + goto protocol_error; + bt->le_rx_packets++; + cen_capture_frame(bt, 10, descriptor + 16, length); + if (enable_acl && cen_receive_acl(bt, descriptor + 16, length)) + goto protocol_error; + bt->le_rx_head = (bt->le_rx_head + 1) % CEN_LE_SLOTS; + cen_le_rx_buffer(bt, bt->le_rx_post); + bt->le_rx_post = (bt->le_rx_post + 1) % CEN_LE_SLOTS; + cen_post(bt, 10, bt->le_rx_post); + } + budget = CEN_RX_SLOTS; + } + tail = cen_tail(bt, 7); + if (tail >= CEN_RX_SLOTS) + goto protocol_error; + while (bt->rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_RX + bt->rx_head * CEN_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length < HCI_EVENT_HDR_SIZE || length > CEN_FOOTER || + length != descriptor[17] + HCI_EVENT_HDR_SIZE) + goto protocol_error; + skb = bt_skb_alloc(length, GFP_KERNEL); + if (skb) { + skb_put_data(skb, descriptor + 16, length); + /* The firmware's status belongs to the standard connection + * request tracked by Linux. Capture the native response before + * restoring only its opcode; preserve status and command credit. + */ + if (bt->hrb_status_pending && length == 6 && + skb->data[0] == HCI_EV_CMD_STATUS && + get_unaligned_le16(skb->data + 4) == CEN_OP_HRB_CREATE_CONN) { + cen_capture_frame(bt, 7, skb->data, length); + put_unaligned_le16(HCI_OP_CREATE_CONN, skb->data + 4); + bt->hrb_status_pending = false; + bt->hrb_status_events++; + } + /* Native HCI flow treats 0xf0/0xf1 Connection Complete as + * ACL, with additional proprietary capability flags. Linux + * otherwise cannot find its pending ACL connection by type. + * Preserve the original event privately before normalizing it. + */ + if (READ_ONCE(bt->hrb_supported) && + length == HCI_EVENT_HDR_SIZE + sizeof(struct hci_ev_conn_complete) && + skb->data[0] == HCI_EV_CONN_COMPLETE && + (skb->data[11] & 0xfe) == 0xf0) { + cen_capture_frame(bt, 7, skb->data, length); + bt->hrb_last_link_type = skb->data[11]; + skb->data[11] = ACL_LINK; + bt->hrb_connection_events++; + } + cen_normalize_ext_adv(skb); + hci_skb_pkt_type(skb) = HCI_EVENT_PKT; + hdev->stat.byte_rx += length; + hci_recv_frame(hdev, skb); + } else { + hdev->stat.err_rx++; + } + bt->rx_head = (bt->rx_head + 1) % CEN_RX_SLOTS; + cen_rx_buffer(bt, bt->rx_post); + bt->rx_post = (bt->rx_post + 1) % CEN_RX_SLOTS; + cen_post(bt, 7, bt->rx_post); + } + if (cen_transmit(bt, &bt->txq, 1, CEN_TX_SLOTS, CEN_TX, + CEN_FOOTER, &bt->tx_head)) + goto protocol_error; + /* cen_aux_specs[0] is the native BR/EDR ACL TX resource (ring 2). */ + if (enable_acl && + (cen_transmit(bt, &bt->acl_txq, 2, cen_aux_specs[0].slots, + bt->aux[0].offset, cen_aux_specs[0].footer, + &bt->acl_tx_head) || + cen_transmit(bt, &bt->le_txq, 4, CEN_LE_SLOTS, CEN_LE_TX, + CEN_LE_FOOTER, &bt->le_tx_head))) + goto protocol_error; + schedule_delayed_work(&bt->poll, msecs_to_jiffies(1)); + goto unlock; +protocol_error: + bt->running = false; + hdev->stat.err_rx++; + dev_err(&bt->pdev->dev, "Beta transport stopped (exec=%u ipc=%u); polling disabled\n", + readl(bt->bar + 0x18), readl(bt->bar)); +unlock: + mutex_unlock(&bt->lock); +} + +static int cen_hci_open(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = true; + schedule_delayed_work(&bt->poll, 0); + mutex_unlock(&bt->lock); + return 0; +} + +static int cen_hci_close(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = false; + bt->hrb_status_pending = false; + mutex_unlock(&bt->lock); + cancel_delayed_work_sync(&bt->poll); + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_vendor_command(struct hci_dev *hdev, u16 opcode, + const void *data, u8 length) +{ + struct sk_buff *skb; + int ret; + + skb = __hci_cmd_sync(hdev, opcode, length, data, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len < 1 ? -EPROTO : -bt_to_errno(skb->data[0]); + kfree_skb(skb); + return ret; +} + +static int cen_hci_post_init(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff *skb; + u32 features; + int ret; + + if (!native_hrb) + return 0; + WRITE_ONCE(bt->hrb_supported, false); + skb = __hci_cmd_sync(hdev, CEN_OP_READ_PROP_FEATURES, 0, NULL, + HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len == 5 ? -bt_to_errno(skb->data[0]) : -EPROTO; + features = skb->len == 5 ? get_unaligned_le32(skb->data + 1) : 0; + kfree_skb(skb); + WRITE_ONCE(bt->proprietary_features, features); + if (ret) + return ret; + if (!(features & CEN_PROP_HRB_CREATE)) + return -EOPNOTSUPP; + WRITE_ONCE(bt->hrb_supported, true); + bt_dev_info(hdev, "Native Proxima Create Connection capability confirmed"); + return 0; +} + +static int cen_hci_setup(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + u8 address[6], init_done[16], seed[24]; + int ret; + + if (!native_startup) + return 0; + + /* Native PROXIMA initialization starts with these three commands, then + * HCI Reset. The core calls setup before its ordinary reset/init sequence. + * Address provisioning is still diagnostic: use a temporary locally + * administered address, in native WriteBdAddr display/network byte order. + */ + ret = get_random_bytes_wait(address, sizeof(address)); + if (ret) + return ret; + address[0] = (address[0] & 0xfc) | 0x02; + get_random_bytes(seed, sizeof(seed)); + memset(init_done, 0xff, sizeof(init_done)); + ret = cen_vendor_command(hdev, 0xfc06, address, sizeof(address)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 1); + ret = cen_vendor_command(hdev, 0xff0b, init_done, sizeof(init_done)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 2); + ret = cen_vendor_command(hdev, 0xfe44, seed, sizeof(seed)); + if (!ret) + WRITE_ONCE(bt->native_startup_steps, 3); +out: + memzero_explicit(seed, sizeof(seed)); + if (ret) + bt_dev_err(hdev, "Native startup stopped after %u commands: %d", + bt->native_startup_steps, ret); + else + bt_dev_info(hdev, "Native startup accepted; temporary diagnostic address in use"); + return ret; +} + +static int cen_hci_flush(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_hci_send(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff_head *queue = &bt->txq; + struct hci_conn *conn; + u16 handle; + u8 link_type; + int ret = 0; + + if (hci_skb_pkt_type(skb) == HCI_ACLDATA_PKT && enable_acl) { + if (skb->len < HCI_ACL_HDR_SIZE || + skb->len != HCI_ACL_HDR_SIZE + get_unaligned_le16(skb->data + 2)) { + ret = -EMSGSIZE; + goto out; + } + handle = get_unaligned_le16(skb->data); + rcu_read_lock(); + conn = hci_conn_hash_lookup_handle(hdev, hci_handle(handle)); + link_type = conn ? conn->type : 0xff; + rcu_read_unlock(); + if (link_type == LE_LINK && (handle & 0x3000) != 0x3000) { + queue = &bt->le_txq; + if (skb->len > CEN_LE_FOOTER) + ret = -EMSGSIZE; + } else if (link_type == ACL_LINK) { + queue = &bt->acl_txq; + if (skb->len > cen_aux_specs[0].footer) + ret = -EMSGSIZE; + } else { + ret = -EOPNOTSUPP; + } + if (ret) + goto out; + } else if (hci_skb_pkt_type(skb) != HCI_COMMAND_PKT) { + ret = -EOPNOTSUPP; + goto out; + } else if (skb->len < HCI_COMMAND_HDR_SIZE || skb->len > CEN_FOOTER || + skb->len != HCI_COMMAND_HDR_SIZE + skb->data[2]) { + ret = -EMSGSIZE; + goto out; + } + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT && native_hrb) { + u16 opcode = get_unaligned_le16(skb->data); + + /* Do not mix unmanaged native commands with translated requests. */ + if (opcode == CEN_OP_HRB_CREATE_CONN) { + ret = -EOPNOTSUPP; + goto out; + } + if (opcode == HCI_OP_CREATE_CONN && + skb->len != HCI_COMMAND_HDR_SIZE + sizeof(struct hci_cp_create_conn)) { + ret = -EMSGSIZE; + goto out; + } + } + mutex_lock(&bt->lock); + if (!bt->running) + ret = -ENETDOWN; + else if (skb_queue_len(queue) >= 64) + ret = -ENOBUFS; + else { + skb_queue_tail(queue, skb); + mod_delayed_work(system_wq, &bt->poll, 0); + } + mutex_unlock(&bt->lock); + if (!ret) + return 0; +out: + /* hci_send_frame() frees rejected packets. Ownership transfers to this + * driver only when the skb is queued and send returns success. + */ + return ret; +} + +static void cen_stop(struct cen_bt *bt) +{ + pci_clear_master(bt->pdev); + kvfree(bt->capture); + bt->capture = NULL; + /* IPC control 0 caused a firmware abort in the initial hardware probe. + * Until coordinated firmware teardown is understood, retain any published + * arena after detaching. A fresh firmware boot is required before rebinding. + */ + if (bt->ipc_requested || !pci_wait_for_pending_transaction(bt->pdev)) + dev_warn(&bt->pdev->dev, "Beta detach requires fresh firmware boot; retaining DMA arena\n"); + else + dma_free_coherent(&bt->pdev->dev, CEN_ARENA_SIZE, bt->arena, bt->dma); +} + +static int cen_status_show(struct seq_file *s, void *unused) +{ + struct cen_bt *bt = s->private; + unsigned int i; + + mutex_lock(&bt->lock); + seq_printf(s, "exec=%u ipc=%u running=%u tx_head=%u tx_consumed=%u rx_head=%u rx_consumed=%u queued=%u\n", + readl(bt->bar + 0x18), readl(bt->bar), bt->running, + bt->tx_head, cen_tail(bt, 1), bt->rx_head, cen_tail(bt, 7), + skb_queue_len(&bt->txq)); + seq_printf(s, "commands=%u events=%u tx_bytes=%u rx_bytes=%u tx_errors=%u rx_errors=%u\n", + bt->hdev->stat.cmd_tx, bt->hdev->stat.evt_rx, + bt->hdev->stat.byte_tx, bt->hdev->stat.byte_rx, + bt->hdev->stat.err_tx, bt->hdev->stat.err_rx); + seq_printf(s, "le_rings=%u le_rx_packets=%u\n", arm_le_data_rings, bt->le_rx_packets); + seq_printf(s, "native_startup=%u native_startup_steps=%u\n", + native_startup, READ_ONCE(bt->native_startup_steps)); + seq_printf(s, "native_hrb=%u hrb_supported=%u proprietary_features=0x%08x hrb_commands=%u hrb_status_events=%u hrb_pending=%u\n", + native_hrb, READ_ONCE(bt->hrb_supported), + READ_ONCE(bt->proprietary_features), bt->hrb_commands, + bt->hrb_status_events, bt->hrb_status_pending); + seq_printf(s, "hrb_connection_events=%u hrb_last_link_type=0x%02x\n", + bt->hrb_connection_events, bt->hrb_last_link_type); + seq_printf(s, "acl_enabled=%u acl_tx=%u acl_rx=%u acl_queued=%u le_queued=%u\n", + enable_acl, bt->hdev->stat.acl_tx, bt->hdev->stat.acl_rx, + skb_queue_len(&bt->acl_txq), skb_queue_len(&bt->le_txq)); + seq_printf(s, "auxiliary_rings=%u messages=%u message_head=%u capture_bytes=%u capture_records=%u capture_dropped=%u\n", + arm_auxiliary_rings, bt->message_tag, bt->message_head, + bt->capture_used, bt->capture_records, bt->capture_dropped); + if (arm_auxiliary_rings) + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) + if (cen_aux_specs[i].receive) + seq_printf(s, "aux_rx ring=%u packets=%u consumed=%u device_tail=%u\n", + cen_aux_specs[i].id, bt->aux[i].packets, + bt->aux[i].head, cen_tail(bt, cen_aux_specs[i].id)); + mutex_unlock(&bt->lock); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(cen_status); + +static ssize_t cen_capture_read(struct file *file, char __user *buffer, + size_t count, loff_t *pos) +{ + struct cen_bt *bt = file->private_data; + ssize_t ret; + + mutex_lock(&bt->lock); + ret = simple_read_from_buffer(buffer, count, pos, bt->capture, bt->capture_used); + mutex_unlock(&bt->lock); + return ret; +} + +static const struct file_operations cen_capture_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = cen_capture_read, + .llseek = default_llseek, +}; + +static ssize_t cen_capture_reset(struct file *file, const char __user *buffer, + size_t count, loff_t *pos) +{ + struct cen_bt *bt = file->private_data; + unsigned int value; + int ret = kstrtouint_from_user(buffer, count, 0, &value); + + if (ret || value != 1) + return ret ?: -EINVAL; + mutex_lock(&bt->lock); + bt->capture_used = 0; + bt->capture_records = 0; + bt->capture_dropped = 0; + mutex_unlock(&bt->lock); + return count; +} + +static const struct file_operations cen_capture_reset_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = cen_capture_reset, +}; + +static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct cen_bt *bt; + struct hci_dev *hdev; + int ret; + + if (!(of_machine_is_compatible("apple,j813") || of_machine_is_compatible("apple,j873g")) || + PCI_FUNC(pdev->devfn) != 2) + return -ENODEV; + bt = devm_kzalloc(&pdev->dev, sizeof(*bt), GFP_KERNEL); + if (!bt) + return -ENOMEM; + bt->pdev = pdev; + mutex_init(&bt->lock); + skb_queue_head_init(&bt->txq); + skb_queue_head_init(&bt->acl_txq); + skb_queue_head_init(&bt->le_txq); + if (enable_acl && (!arm_le_data_rings || !arm_auxiliary_rings)) + return -EINVAL; + if (native_hrb && (!native_startup || !enable_acl)) + return -EINVAL; + INIT_DELAYED_WORK(&bt->poll, cen_poll); + ret = pcim_enable_device(pdev); + if (ret) + return ret; + if (pci_resource_len(pdev, 0) != SZ_64K) + return -ENODEV; + ret = pcim_iomap_regions(pdev, BIT(0), "btcentauri"); + if (ret) + return ret; + bt->bar = pcim_iomap_table(pdev)[0]; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 0) + return dev_err_probe(&pdev->dev, -EBUSY, + "Beta requires running firmware and unowned IPC (exec=%u ipc=%u)\n", + readl(bt->bar + 0x18), readl(bt->bar)); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; + bt->arena = dma_alloc_coherent(&pdev->dev, CEN_ARENA_SIZE, &bt->dma, GFP_KERNEL); + if (!bt->arena) + return -ENOMEM; + memset(bt->arena, 0, CEN_ARENA_SIZE); + if (arm_auxiliary_rings) { + bt->capture = kvzalloc(CEN_CAPTURE_SIZE, GFP_KERNEL); + if (!bt->capture) { + ret = -ENOMEM; + goto stop; + } + } + ret = cen_start(bt); + if (ret) { + dev_err(&pdev->dev, "Beta IPC initialization failed: %d\n", ret); + goto stop; + } + hdev = hci_alloc_dev(); + if (!hdev) { + ret = -ENOMEM; + goto stop; + } + bt->hdev = hdev; + hdev->bus = HCI_PCI; + hdev->open = cen_hci_open; + hdev->close = cen_hci_close; + hdev->flush = cen_hci_flush; + hdev->send = cen_hci_send; + hdev->setup = cen_hci_setup; + hdev->post_init = cen_hci_post_init; + SET_HCIDEV_DEV(hdev, &pdev->dev); + hci_set_drvdata(hdev, bt); + pci_set_drvdata(pdev, bt); + ret = hci_register_dev(hdev); + if (ret) { + hci_free_dev(hdev); + goto stop; + } + bt->debug = debugfs_create_dir("btcentauri", NULL); + debugfs_create_file("status", 0400, bt->debug, bt, &cen_status_fops); + if (bt->capture) { + debugfs_create_file("auxiliary_capture", 0400, bt->debug, bt, &cen_capture_fops); + debugfs_create_file("capture_reset", 0200, bt->debug, bt, &cen_capture_reset_fops); + } + dev_info(&pdev->dev, "Beta HCI transport registered; ACL enabled=%u, SCO/ISO unsupported\n", + enable_acl); + return 0; +stop: + cen_stop(bt); + return ret; +} + +static void cen_remove(struct pci_dev *pdev) +{ + struct cen_bt *bt = pci_get_drvdata(pdev); + + debugfs_remove_recursive(bt->debug); + hci_unregister_dev(bt->hdev); + cen_hci_close(bt->hdev); + hci_free_dev(bt->hdev); + cen_stop(bt); +} + +/* + * As for the Wi-Fi functions (centauri_pci.c): the firmware keeps running, so + * keep the function enabled and in D0 (the PCI core's default suspend turns + * bus mastering off, and the next poll stopped the transport for good) and + * the PCIe host powered. Polling pauses across the sleep. + */ +static int cen_suspend(struct device *dev) +{ + struct cen_bt *bt = pci_get_drvdata(to_pci_dev(dev)); + + cancel_delayed_work_sync(&bt->poll); + device_set_awake_path(dev); + return pci_save_state(to_pci_dev(dev)); +} + +static int cen_resume(struct device *dev) +{ + struct cen_bt *bt = pci_get_drvdata(to_pci_dev(dev)); + + mutex_lock(&bt->lock); + if (bt->running) + schedule_delayed_work(&bt->poll, 0); + mutex_unlock(&bt->lock); + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(cen_pm_ops, cen_suspend, cen_resume); + +static const struct pci_device_id cen_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903) }, + { } +}; +MODULE_DEVICE_TABLE(pci, cen_ids); + +static struct pci_driver cen_driver = { + .name = "btcentauri", + .id_table = cen_ids, + .probe = cen_probe, + .remove = cen_remove, + .driver.pm = pm_sleep_ptr(&cen_pm_ops), +}; +module_pci_driver(cen_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 Beta HCI command/event transport for J813 and J873 bring-up"); diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index f4452c1005834..c988e566c5344 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1491,13 +1491,23 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, } ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit, ep->dcp->fw_compat == DCP_FIRMWARE_H17P); - if (ret) { + if (ret == -ENOENT && + ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) { + /* + * M5 (H17G) announces powerlog-service with only + * ep-has-desc-mgr and interface-id; match it by the + * announced name, as for props-less announces. + */ + epic_name = NULL; + epic_class = NULL; + } else if (ret) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: failed to extract init props: %d\n", ep->endpoint, ret); return; + } else { + service_name = epic_class; } - service_name = epic_class; } else { service_name = name; } diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index cc709dac66626..9001a1fe20c3c 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -170,9 +170,12 @@ static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, if (t8132_ext_crtc(crtc)) continue; dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); - if (dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - /* H17P applies the stored fourth byte as straight alpha. */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + /* + * H17P applies the stored fourth byte as straight alpha, + * the H17G (M5) profile too: an XRGB console with X = 0 + * stays black. ARGB makes fbcon write it opaque. + */ sizes->surface_bpp = 32; sizes->surface_depth = 32; break; @@ -915,6 +918,8 @@ static int apple_probe_per_dcp(struct device *dev, planes[zpos] = apple_plane_init(drm, 1U << num, surf, supports_l10r, supports_xrgb2101010, + dcp_data->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17G, plane_type, interchange, supports_premultiplied); } diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 6baf9290cf2e0..70e46bab4b653 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4122,6 +4122,18 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) #define DCP_BL_FALLBACK_DELAY msecs_to_jiffies(2000) +/* + * H17G (M5) internal panel: a pipe stop never comes back (see + * iomfb_blank_h17p()), so DPMS blanks the panel and keeps the pipe running. + */ +static bool dcp_h17g_panel(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && + dcp->connector_type == DRM_MODE_CONNECTOR_eDP && + dcp_has_panel(dcp); +} + static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) { /* The 14.7 panel has its own known-level blank and restore hooks. */ @@ -4212,6 +4224,12 @@ void dcp_poweron(struct platform_device *pdev) return; } + if (dcp_h17g_panel(dcp)) { + /* The pipe never stopped; the next swap restores the level. */ + dcp->brightness.update = true; + return; + } + if (dcp_is_typec_output(dcp)) { /* This CRTC now owns the current Type-C display session. */ scoped_guard(mutex, &dcp->hpd_mutex) @@ -4339,6 +4357,11 @@ void dcp_poweroff(struct platform_device *pdev) return; } + if (dcp_h17g_panel(dcp)) { + iomfb_blank_h17p(dcp); + return; + } + if (dcp->avep) av_service_disconnect(dcp); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index abca2a106223e..ad8307f602ea7 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -168,30 +168,37 @@ static u64 iomfb_nits_binary64_h17p(u32 nits) return value; } -void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, - u32 maximum, bool update) +/* @swap is the swap record of a present; H17G sends it unserialized. */ +void iomfb_encode_backlight_swap_h17p(u8 *swap, u32 nits, u32 maximum, + bool update) { u64 ceiling = iomfb_nits_binary64_h17p(maximum); u64 unity = iomfb_nits_binary64_h17p(1); /* This profile does not use the older DAC/power words. */ - memset(wire->swap + 0x32f, 0, 0xd); - memset(wire->swap + 0x354, 0, 0x9a); + memset(swap + 0x32f, 0, 0xd); + memset(swap + 0x354, 0, 0x9a); /* The ceiling pair must remain valid even on a brightness-idle frame. */ - put_unaligned_le64(ceiling, wire->swap + 0x36e); - put_unaligned_le64(ceiling, wire->swap + 0x376); + put_unaligned_le64(ceiling, swap + 0x36e); + put_unaligned_le64(ceiling, swap + 0x376); if (!update) return; /* Linux brightness changes and soft DPMS use the same activation bytes. */ - put_unaligned_le32(1, wire->swap + 0x354); - wire->swap[0x358] = 1; - wire->swap[0x359] = 1; - wire->swap[0x35a] = 1; - put_unaligned_le16(1, wire->swap + 0x35c); - put_unaligned_le64(iomfb_nits_binary64_h17p(nits), wire->swap + 0x35e); - put_unaligned_le64(unity, wire->swap + 0x37e); - put_unaligned_le64(unity, wire->swap + 0x3e6); + put_unaligned_le32(1, swap + 0x354); + swap[0x358] = 1; + swap[0x359] = 1; + swap[0x35a] = 1; + put_unaligned_le16(1, swap + 0x35c); + put_unaligned_le64(iomfb_nits_binary64_h17p(nits), swap + 0x35e); + put_unaligned_le64(unity, swap + 0x37e); + put_unaligned_le64(unity, swap + 0x3e6); +} + +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, + u32 maximum, bool update) +{ + iomfb_encode_backlight_swap_h17p(wire->swap, nits, maximum, update); } static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { @@ -308,9 +315,12 @@ static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return trampoline_zero(dcp, tag, out, in); - + /* + * H17G (M5) needs the same empty dictionary. A zeroed reply fails the + * firmware's OSDictionary deserialization ("OSDictionary + * deserialization failed" in the DCP crash log) and the DCP panics + * with "app NMI" one hour after boot. + */ trace_iomfb_callback(dcp, tag, __func__); /* The dispatcher has validated the D114 reply size. */ memset(out, 0, 0x1004); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 9880e82078178..112a4e002138d 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -158,6 +158,9 @@ static inline struct dcp_apply_property_h17p dcp_opaque_x_property_h17p(void) }; } +void iomfb_blank_h17p(struct apple_dcp *dcp); +void iomfb_encode_backlight_swap_h17p(u8 *swap, u32 nits, u32 maximum, + bool update); void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, u32 maximum, bool update); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 0fac3f679bcff..82e5310f3fd6e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2470,6 +2470,44 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) dev_info(dcp->dev, "dcp_poweroff() done\n"); } +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER && DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) +/* + * H17G (M5) internal panel DPMS off. A pipe stop does not come back: + * setPowerState(0) times out and the next setPowerState(1) is never + * answered. Blank instead and keep the pipe running: a black, + * surface-free swap that also takes the backlight to 0 nits. The next CRTC + * enable presents the scanout and the brightness again. + */ +void DCP_FW_NAME(iomfb_blank)(struct apple_dcp *dcp) +{ + struct DCP_FW_NAME(dcp_swap_start_req) swap_req = { 0 }; + typeof(DCP_FW_UNION(dcp->swap)) *swap = &DCP_FW_UNION(dcp->swap); + struct dcp_swap_cookie *cookie; + unsigned long ret; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp_prepare_clear_swap(dcp, swap); + if (dcp_has_panel(dcp)) + iomfb_encode_backlight_swap_h17p((u8 *)&swap->swap, 0, + dcp->brightness.maximum, true); + + dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); + if (!ret || READ_ONCE(cookie->status)) + dev_warn(dcp->dev, "%s: blank swap %s\n", __func__, + ret ? "failed" : "timed out"); + kref_put(&cookie->refcount, release_swap_cookie); +} +#endif + static void last_client_closed_sleep(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_DESKTOP_26(DCP_FW_VER) @@ -3847,6 +3885,21 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; #endif if (update_brightness) { +#if DCP_FW_H17(DCP_FW_VER) + /* + * H17G firmware ignores the DAC words and takes nits in the + * same swap bytes as H17P. Without a requested or reported + * level, send nothing rather than 0 nits. + */ + int nits = dcp->brightness.bl_dev ? + dcp->brightness.bl_dev->props.brightness : + dcp->brightness.nits; + + if (nits > 0) + iomfb_encode_backlight_swap_h17p((u8 *)&req->swap, nits, + dcp->brightness.maximum, + true); +#else #if DCP_FW_LAYOUT_14_7(DCP_FW_VER) req->swap.bl_update = 1; #else @@ -3854,6 +3907,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru #endif req->swap.bl_value = dcp->brightness.dac; req->swap.bl_power = 0x40; +#endif dcp->brightness.update = false; } #endif diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index f2b57a700ffb7..a8c37e29ed65a 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -743,6 +743,7 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, u32 iomfb_surf, bool supports_l10r, bool supports_xrgb2101010, + bool xrgb_is_alpha, enum drm_plane_type type, bool interchange, bool premultiplied) @@ -755,7 +756,16 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, switch (type) { case DRM_PLANE_TYPE_PRIMARY: - if (supports_l10r && supports_xrgb2101010) { + if (xrgb_is_alpha) { + /* + * The firmware blends the X byte as alpha and ignores the + * opaque-X property: offer only formats with alpha, so that + * compositors write it instead of leaving whatever the + * renderer stored there. + */ + fmts = dcp_overlay_formats; + num_fmts = ARRAY_SIZE(dcp_overlay_formats); + } else if (supports_l10r && supports_xrgb2101010) { fmts = dcp_primary_formats; num_fmts = ARRAY_SIZE(dcp_primary_formats); } else if (supports_l10r) { diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index 69229ad6a5775..32d7da10a6a4f 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -37,6 +37,7 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, u32 iomfb_surf, bool supports_l10r, bool supports_xrgb2101010, + bool xrgb_is_alpha, enum drm_plane_type type, bool interchange, bool premultiplied); diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index 5360aaf3e5804..38f667e114514 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -120,6 +120,10 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype } dcp->brightness.nits = mnits.millinits / 1000; + /* H17G publishes no NITS property; its first report registers the device. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && + dcp_has_panel(dcp) && !dcp->brightness.bl_dev && dcp->brightness.maximum > 0) + schedule_work(&dcp->bl_register_wq); return 0; } diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 6b28266b6b4d1..6c5c31de0528a 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -43,6 +43,26 @@ config DRM_ASAHI_G16 An aborted receipt disables that command class until reboot. Suspend, hibernation and kexec are blocked while native graphics owns DMA. +config DRM_ASAHI_G18 + bool "Apple M6 (G18G, T8152) support (experimental)" + depends on DRM_ASAHI && ARM64 + select WANT_DEV_COREDUMP + help + Backend for the Mac mini M6 (J873) GPU and its 26A428 firmware. + The GPU is started only when the boot loader reports the matching + firmware images and enables both GFX coprocessor mailboxes; the + firmware session and its memory are then retained until reboot. + Submission is synchronous: every command retires before the next + one starts. The module parameters prefixed g18_ select the bring-up + stage and the admitted command envelope. +config DRM_ASAHI_M5 + bool "Apple M5 (G17G, T8142) support (experimental)" + depends on DRM_ASAHI && ARM64 + help + Backend for the M5 GPU (J813 MacBook Air) running the 25G76 + firmware (RTKit-3255.160.4), as loaded and published by the aura + boot loader (/chosen/aurora,m5-gpu-firmware-profile). + config DRM_ASAHI_DEBUG_ALLOCATOR bool "Use debug allocator" depends on DRM_ASAHI diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index decf16ea09647..1a5cf1ba9a7c1 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -22,6 +22,10 @@ mod gem; #[cfg(CONFIG_DRM_ASAHI_G16)] mod g16; +#[cfg(CONFIG_DRM_ASAHI_G18)] +mod g18; +#[cfg(CONFIG_DRM_ASAHI_M5)] +mod m5; mod gpu; mod hw; mod initdata; @@ -78,6 +82,9 @@ struct DriverModule { _g15: G15Registration, // Field drop order unregisters/joins driver producers before queue teardown. _g16: Option>>, + #[cfg(CONFIG_DRM_ASAHI_G18)] + #[pin] + _g18: kernel::driver::Registration>, #[pin] _driver: kernel::driver::Registration>, _cleanup: cleanup::QueueOwner, @@ -99,6 +106,8 @@ impl kernel::InPlaceModule for DriverModule { module, ), _g16: register_g16(module)?, + #[cfg(CONFIG_DRM_ASAHI_G18)] + _g18 <- kernel::driver::Registration::new(g18::probe::DRIVER_NAME, module), // All fallible fields are initialized. This final unit initializer cannot fail. #[cfg(CONFIG_DRM_ASAHI_G15)] _g15_ready: g15::activate(), @@ -198,6 +207,10 @@ module! { default: 1, description: "Count held GPU memory in its creator's RSS for the OOM killer (1 = on)", }, + m5_fw_trace: u32 { + default: 0, + description: "M5: firmware trace classes (power bit 0, submission bit 1, recovery classifier bit 9)", + }, g16_deadline_ms: u32 { default: 5000, description: "G16: administrative completion-stamp deadline, ms (0 disables; long running work can exceed it)", diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index ff82d8c118e85..93c52f1de9968 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -142,6 +142,8 @@ pub(crate) enum ProbeConfig { Legacy(&'static hw::HwConfig), G17(&'static g17::Config), M3(crate::m3::Chip), + /// The M5 (T8142) backend, refused unless it is built. + M5, } // OF Device ID table. Linux 7.3's of_device_table!() takes no attributes on @@ -165,6 +167,10 @@ macro_rules! asahi_of_device_table { of::DeviceId::new(c"apple,agx-t6041"), ProbeConfig::G17(&hw::t6041::CONFIG) ), + ( + of::DeviceId::new(c"apple,agx-t8142"), + ProbeConfig::M5 + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) @@ -253,6 +259,10 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::Legacy(cfg) => *cfg, ProbeConfig::G17(cfg) => return AsahiPlatformDriver::probe_g17(pdev, cfg), ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), + #[cfg(CONFIG_DRM_ASAHI_M5)] + ProbeConfig::M5 => return AsahiPlatformDriver::probe_m5(pdev), + #[cfg(not(CONFIG_DRM_ASAHI_M5))] + ProbeConfig::M5 => return Err(ENODEV), }; // SAFETY: The selected SoC's UAT output width bounds all device DMA. @@ -444,6 +454,41 @@ impl AsahiPlatformDriver { } } +#[cfg(CONFIG_DRM_ASAHI_M5)] +impl AsahiPlatformDriver { + fn probe_m5<'bound>( + pdev: &'bound platform::Device>, + ) -> Result> { + // SAFETY: The M5 UAT has 42 output bits. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; + let resources = regs::Resources::new(pdev)?; + let res = regs::G17Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; + let gpu = crate::m5::probe(pdev, &drm, res)?; + let data = try_pin_init!(AsahiDrmRegistrationData { + gpu: gpu as Arc, + resources, + }); + // SAFETY: The registration is stored and unregistered by platform data destruction. + // The M5 firmware session retains its firmware-visible state until reboot. + let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + Ok(AsahiPlatformDriverData { + binding, + _device: reg.device().into(), + _reg: reg, + g17: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, + raw_dev: core::ptr::null_mut(), + raw_stats: core::ptr::null_mut(), + stats: None, + }) + } +} + #[cfg(CONFIG_DRM_ASAHI_G16)] impl AsahiPlatformDriver { pub(crate) fn probe_g16<'bound>( @@ -481,6 +526,63 @@ impl AsahiPlatformDriver { } } +#[cfg(CONFIG_DRM_ASAHI_G18)] +impl AsahiPlatformDriver { + /// Probe the M6 GPU through the G18 backend. + /// + /// The parameter gate refuses before any MMIO, DMA, mailbox or processor access. A + /// diagnostic bring-up stage retains its firmware owner without a DRM device; the + /// application stage registers the render node over the started session. + pub(crate) fn probe_g18<'bound>( + pdev: &'bound platform::Device>, + soc: &'static hw::t8152::SocConfig, + ) -> Result> { + crate::g18::probe::admit_t8152(pdev)?; + let resources = regs::Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: The G18 adapter closes the binding in unbind before devres; probe errors + // drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; + let started = crate::g18::probe::probe_t8152(pdev, &drm, soc)?; + let Some(gpu) = started.gpu else { + return Ok(crate::g18::probe::G18Bound { + runtime: started.runtime, + binding: Some(binding), + driver: None, + }); + }; + let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); + // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is + // unbound; it is never forgotten. + let reg = match unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0) } { + Ok(reg) => reg, + Err(error) => { + started.runtime.shutdown(); + return Err(error); + } + }; + dev_info!( + pdev.as_ref(), + "G18_DRM registered render node (aurora, apple,agx-t8152)\n" + ); + Ok(crate::g18::probe::G18Bound { + runtime: started.runtime, + binding: None, + driver: Some(AsahiPlatformDriverData { + binding, + _device: reg.device().into(), + _reg: reg, + g17: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, + raw_dev: core::ptr::null_mut(), + raw_stats: core::ptr::null_mut(), + stats: None, + }), + }) + } +} + impl AsahiPlatformDriverData<'_> { pub(crate) fn close_binding(&self) { self.binding.close(); diff --git a/drivers/gpu/drm/asahi/g18/address_map.rs b/drivers/gpu/drm/asahi/g18/address_map.rs new file mode 100644 index 0000000000000..1a73e894dd4a1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/address_map.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! J873 fixed VA reservations. Addresses are policy, never captured pointers. +//! The allocator owns suballocations inside the named legacy arenas. Compact +//! aliases intentionally share physical backing but have disjoint VA extents. +#![allow(dead_code)] + +pub(crate) const PAGE: u64 = 0x4000; +pub(crate) const KERNEL_BASE: u64 = 0xffff_fc20_0000_0000; +pub(crate) const COMPACT_BIAS: u64 = 0x10_0000_0000; +// Optional H1 control: the first Gallium VDM address from retained run123. +pub(crate) const GRAPHICS_HIGH_VDM: u64 = 0x5f_ffe2_8000; +// Native PPP page 0x58000 aliases the first Gallium PPP page. Preserve offsets +// between all four native PPP pages; these aliases share original backing. +pub(crate) const GRAPHICS_HIGH_PPP_BASE: u64 = 0x5f_ffea_c000; +pub(crate) const GRAPHICS_HIGH_PPP_PAGES: [u64; 4] = [0, 0x48000, 0x58000, 0x68000]; +pub(crate) const PB_TABLE: u64 = KERNEL_BASE + 0x015e_0000; +pub(crate) const UMA_TABLE: u64 = KERNEL_BASE + 0x015e_8000; +pub(crate) const COMPUTE_GRAPH: u64 = KERNEL_BASE + 0xc000_0000; +pub(crate) const COMPUTE_ENTRY_HIGH: u64 = KERNEL_BASE + 0x0039_0000; +pub(crate) const COMPUTE_ENTRY_LOW: u64 = 0x70_005f_0000; +pub(crate) const DYNAMIC_HIGH_START: u64 = KERNEL_BASE + 0x0168_0000; +pub(crate) const DYNAMIC_HIGH_END: u64 = KERNEL_BASE + 0x1000_0000; +pub(crate) const POOL_DIRECTORY: u64 = 0x70_0200_0000; +pub(crate) const POOL_BLOCK_LIST: u64 = 0x70_0400_0000; +pub(crate) const POOL_BLOCKS: u64 = 0x70_0500_0000; +pub(crate) const POOL_BLOCK_STRIDE: u64 = 0x20_0000; +// Job-VM-only 0x1a510 object for g18_app_preempt_mode=1: the first pages of +// the VM_CREATE auxiliary driver arena (m5_vm::auxiliary_range), which +// gem_bind refuses to userspace. Native uses a 48 KiB process object. +pub(crate) const APPLICATION_COMPUTE_PREEMPT_DRIVER: u64 = COMPACT_BIAS + (1 << 32); +pub(crate) const APPLICATION_COMPUTE_PREEMPT_DRIVER_BYTES: u64 = 0xc000; + +// Graphics reservations are beyond both the compute graph and its low arena. +// Queue/PB owner integration must consume these, or update this table as one +// unit; the legacy M5 graph/entries collide with COMPUTE_GRAPH/ENTRY_*. +pub(crate) const GRAPHICS_QUEUE_HIGH: u64 = KERNEL_BASE + 0xc200_0000; +pub(crate) const GRAPHICS_QUEUE_BYTES: usize = 0x40000; +pub(crate) const GRAPHICS_ENTRY_HIGH: u64 = KERNEL_BASE + 0x0050_0000; +pub(crate) const GRAPHICS_ENTRY_LOW: u64 = 0x70_1100_0000; +pub(crate) const GRAPHICS_ENTRY_BYTES: usize = 0x40000; +pub(crate) const GRAPHICS_JOB_LIST: u64 = GRAPHICS_QUEUE_HIGH + 0x400; +pub(crate) const GRAPHICS_QUEUE_CONTROL: u64 = GRAPHICS_QUEUE_HIGH + 0x500; +pub(crate) const GRAPHICS_JOB_HIGH: u64 = KERNEL_BASE + 0xc210_0000; +pub(crate) const GRAPHICS_JOB_LOW: u64 = 0x70_1200_0000; +pub(crate) const GRAPHICS_JOB_BYTES: usize = 0x40000; +pub(crate) const GRAPHICS_SIGNALS: u64 = KERNEL_BASE + 0xc218_0000; +pub(crate) const GRAPHICS_SIGNALS_BYTES: usize = 0x10000; +pub(crate) const GRAPHICS_SCHEDULER_START: u64 = KERNEL_BASE + 0xc220_0000; +pub(crate) const GRAPHICS_SCHEDULER_END: u64 = GRAPHICS_SCHEDULER_START + 0x8000; +pub(crate) const GRAPHICS_PB_HIGH: u64 = KERNEL_BASE + 0xc230_0000; +pub(crate) const GRAPHICS_PB_BYTES: usize = 0x100000; +pub(crate) const GRAPHICS_UMA_HIGH: u64 = KERNEL_BASE + 0xc300_0000; +pub(crate) const GRAPHICS_UMA_LIST: u64 = GRAPHICS_UMA_HIGH + 0x34000; +pub(crate) const GRAPHICS_UMA_STATE: u64 = GRAPHICS_UMA_HIGH + 0x38000; +pub(crate) const GRAPHICS_UMA_COUNTER: u64 = GRAPHICS_UMA_HIGH + 0x3c000; +pub(crate) const GRAPHICS_UMA_BACKING: u64 = GRAPHICS_UMA_HIGH + 0x100000; +pub(crate) const GRAPHICS_UMA_BYTES: u64 = 0x3000000; +pub(crate) const GRAPHICS_UMA_GROWTH: u64 = GRAPHICS_UMA_HIGH + 0x2a00000; +pub(crate) const GRAPHICS_IDEAL_BLOCKS: u64 = 0x70_2000_0000; +pub(crate) const GRAPHICS_TARGET_OFFSET: u64 = 0x0100_0000; +pub(crate) const GRAPHICS_TARGET_BYTES: usize = 0x0100_0000; +pub(crate) const GRAPHICS_PB_OFFSET: u64 = 0x0200_0000; +pub(crate) const GRAPHICS_PB_COMPACT_BYTES: usize = 0x0100_0000; +pub(crate) const PB_GRAPH: u64 = GRAPHICS_PB_HIGH + 0x4000; +pub(crate) const PB_PAGE_LIST_HIGH: u64 = GRAPHICS_PB_HIGH + 0x10000; +pub(crate) const PB_PAGE_LIST_LOW: u64 = GRAPHICS_PB_OFFSET; +pub(crate) const PB_PAGE_LIST_BYTES: usize = 0x1c000; +pub(crate) const PB_METRICS_HIGH: u64 = GRAPHICS_PB_HIGH + 0x30000; +pub(crate) const PB_METRICS_LOW: u64 = GRAPHICS_PB_OFFSET + 0x20000; +pub(crate) const PB_SCENE_SCRATCH: u64 = GRAPHICS_PB_OFFSET + 0x28000; +pub(crate) const PB_DISCARD: u64 = GRAPHICS_PB_OFFSET + 0x30000; +pub(crate) const PB_BLOCKS: u64 = GRAPHICS_PB_OFFSET + 0x100000; +pub(crate) const QUEUE_IDS: [u8; 2] = [9, 10]; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Region { pub name: &'static str, pub base: u64, pub bytes: u64 } +const fn r(name: &'static str, base: u64, bytes: u64) -> Region { Region { name, base, bytes } } + +pub(crate) const REGIONS: &[Region] = &[ + r("capture-compact", 0, 0x78000), + r("capture-compact-biased", COMPACT_BIAS, 0x78000), + r("capture-high-vdm-control", GRAPHICS_HIGH_VDM, PAGE), + r("capture-high-ppp-0", GRAPHICS_HIGH_PPP_BASE - 0x58000, PAGE), + r("capture-high-ppp-48000", GRAPHICS_HIGH_PPP_BASE - 0x10000, PAGE), + r("capture-high-ppp-58000", GRAPHICS_HIGH_PPP_BASE, PAGE), + r("capture-high-ppp-68000", GRAPHICS_HIGH_PPP_BASE + 0x10000, PAGE), + r("capture-data", 0x100_0000_0000, 0x3d8000), + r("init-private-roots-fwctl", KERNEL_BASE + 0x20000, 0x1a4000), + r("pb-table", PB_TABLE, 0x4000), r("uma-table", UMA_TABLE, 0x8000), + r("dynamic-high-pool-scheduler-compute", DYNAMIC_HIGH_START, + DYNAMIC_HIGH_END - DYNAMIC_HIGH_START), + r("compute-queue", COMPUTE_GRAPH, PAGE), + r("compute-entry-high", COMPUTE_ENTRY_HIGH, 0x20000), + r("compute-entry-low", COMPUTE_ENTRY_LOW, 0x20000), + r("init-shared", KERNEL_BASE + 0xc0788000, 0x28000), + r("init-sentinel", KERNEL_BASE + 0xc07e0000, PAGE), + r("completion-high", KERNEL_BASE + 0xc07e8000, 4 * 0x28000), + r("pb-table-low", 0x70_0183_8000, 0x4000), + r("uma-table-low", 0x70_0184_0000, 0x8000), + r("completion-low", 0x70_0184_8000, 4 * 0x28000), + r("pool-directory-reservation", POOL_DIRECTORY, 0x2000000), + r("pool-block-list", POOL_BLOCK_LIST, PAGE), + r("application-compute-directory", POOL_DIRECTORY + 0x40000000, 0x2000000), + r("application-compute-block-list", POOL_BLOCK_LIST + 0x40000000, PAGE), + r("application-compute-block0", POOL_BLOCKS + 0x40000000, 0x204000), + r("application-compute-block1", POOL_BLOCKS + 0x40300000, 0x204000), + r("application-compute-block2", POOL_BLOCKS + 0x40600000, 0x204000), + r("application-compute-block3", POOL_BLOCKS + 0x40900000, 0x204000), + r("application-compute-preempt", 0x70_5000_0000, PAGE), + r("application-compute-preempt-driver", APPLICATION_COMPUTE_PREEMPT_DRIVER, + APPLICATION_COMPUTE_PREEMPT_DRIVER_BYTES + PAGE), + // Replacement views have independent sparse directories and block lists. + // Each scratch aperture includes every supported 2-MiB block plus guards + // at a 3-MiB stride; four-GiB bank spacing also covers the full 2-GiB plan. + r("compute-replacement-a-directory", POOL_DIRECTORY + 0x2_4000_0000, 0x2000000), + r("compute-replacement-a-list", POOL_BLOCK_LIST + 0x2_4000_0000, PAGE), + r("compute-replacement-a-blocks", POOL_BLOCKS + 0x2_4000_0000, 0xc0000000), + r("compute-replacement-b-directory", POOL_DIRECTORY + 0x3_4000_0000, 0x2000000), + r("compute-replacement-b-list", POOL_BLOCK_LIST + 0x3_4000_0000, PAGE), + r("compute-replacement-b-blocks", POOL_BLOCKS + 0x3_4000_0000, 0xc0000000), + r("pool-block0-guard", POOL_BLOCKS, 0x104000), + r("pool-block1-guard", POOL_BLOCKS + POOL_BLOCK_STRIDE, 0x104000), + r("pool-block2-guard", POOL_BLOCKS + 2 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block3-guard", POOL_BLOCKS + 3 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block4-guard", POOL_BLOCKS + 4 * POOL_BLOCK_STRIDE, 0x104000), + // Compute currently allocates five blocks. Reserve twelve for the pending + // graphics pool so its additional backing cannot enter another arena. + r("pool-block5-guard", POOL_BLOCKS + 5 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block6-guard", POOL_BLOCKS + 6 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block7-guard", POOL_BLOCKS + 7 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block8-guard", POOL_BLOCKS + 8 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block9-guard", POOL_BLOCKS + 9 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block10-guard", POOL_BLOCKS + 10 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block11-guard", POOL_BLOCKS + 11 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block12-guard", POOL_BLOCKS + 12 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block13-guard", POOL_BLOCKS + 13 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block14-guard", POOL_BLOCKS + 14 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block15-guard", POOL_BLOCKS + 15 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block16-guard", POOL_BLOCKS + 16 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block17-guard", POOL_BLOCKS + 17 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block18-guard", POOL_BLOCKS + 18 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block19-guard", POOL_BLOCKS + 19 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block20-guard", POOL_BLOCKS + 20 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block21-guard", POOL_BLOCKS + 21 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block22-guard", POOL_BLOCKS + 22 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block23-guard", POOL_BLOCKS + 23 * POOL_BLOCK_STRIDE, 0x104000), + r("compute-low-arena", 0x70_0800_0000, 0x8000000), + r("graphics-queues", GRAPHICS_QUEUE_HIGH, GRAPHICS_QUEUE_BYTES as u64), + r("graphics-entries-high", GRAPHICS_ENTRY_HIGH, GRAPHICS_ENTRY_BYTES as u64), + r("graphics-entries-low", GRAPHICS_ENTRY_LOW, GRAPHICS_ENTRY_BYTES as u64), + r("graphics-job-high", GRAPHICS_JOB_HIGH, GRAPHICS_JOB_BYTES as u64), + r("graphics-job-low", GRAPHICS_JOB_LOW, GRAPHICS_JOB_BYTES as u64), + r("graphics-signals", GRAPHICS_SIGNALS, GRAPHICS_SIGNALS_BYTES as u64), + r("graphics-scheduler", GRAPHICS_SCHEDULER_START, + GRAPHICS_SCHEDULER_END - GRAPHICS_SCHEDULER_START), + r("graphics-pb-high", GRAPHICS_PB_HIGH, GRAPHICS_PB_BYTES as u64), + r("graphics-uma-high", GRAPHICS_UMA_HIGH, GRAPHICS_UMA_BYTES), + r("graphics-ideal-blocks", GRAPHICS_IDEAL_BLOCKS, 40 * POOL_BLOCK_STRIDE), + r("graphics-target", GRAPHICS_TARGET_OFFSET, GRAPHICS_TARGET_BYTES as u64), + r("graphics-target-biased", COMPACT_BIAS + GRAPHICS_TARGET_OFFSET, GRAPHICS_TARGET_BYTES as u64), + r("graphics-pb", GRAPHICS_PB_OFFSET, GRAPHICS_PB_COMPACT_BYTES as u64), + r("graphics-pb-biased", COMPACT_BIAS + GRAPHICS_PB_OFFSET, GRAPHICS_PB_COMPACT_BYTES as u64), +]; + +pub(crate) const fn nonoverlapping(regions: &[Region]) -> bool { + let mut i = 0; + while i < regions.len() { + let a = regions[i]; + if a.bytes == 0 || a.base & (PAGE - 1) != 0 || a.bytes & (PAGE - 1) != 0 + || a.base > u64::MAX - a.bytes { return false; } + let mut j = 0; + while j < i { + let b = regions[j]; + if a.base < b.base + b.bytes && b.base < a.base + a.bytes { return false; } + j += 1; + } + i += 1; + } + true +} +const _: () = assert!(nonoverlapping(REGIONS)); + +pub(crate) const GRAPHICS_UMA_REGIONS: &[Region] = &[ + r("uma-control", GRAPHICS_UMA_HIGH, 0x40000), + r("uma-ideal-backing", GRAPHICS_UMA_BACKING, 40 * 0x100000), + r("uma-ideal-growth", GRAPHICS_UMA_GROWTH, 12 * 21 * PAGE), +]; +const _: () = assert!(nonoverlapping(GRAPHICS_UMA_REGIONS)); +const _: () = assert!(GRAPHICS_UMA_GROWTH + 12 * 21 * PAGE <= GRAPHICS_UMA_HIGH + GRAPHICS_UMA_BYTES); + +pub(crate) const PB_REGIONS: &[Region] = &[ + r("pb-state", GRAPHICS_PB_HIGH, PAGE), r("pb-graph", PB_GRAPH, 0x8000), + r("pb-page-list-high", PB_PAGE_LIST_HIGH, PB_PAGE_LIST_BYTES as u64), + r("pb-metrics-high", PB_METRICS_HIGH, PAGE), + r("pb-page-list-low", PB_PAGE_LIST_LOW, PB_PAGE_LIST_BYTES as u64), + r("pb-metrics-low", PB_METRICS_LOW, PAGE), r("pb-scene-scratch", PB_SCENE_SCRATCH, PAGE), + r("pb-discard", PB_DISCARD, 0x8000), r("pb-blocks", PB_BLOCKS, 8 * 0x80000), +]; +const _: () = assert!(nonoverlapping(PB_REGIONS)); + +const fn pb_regions_contained() -> bool { + let mut i = 0; + while i < PB_REGIONS.len() { + let child = PB_REGIONS[i]; + let high = child.base >= GRAPHICS_PB_HIGH + && child.base + child.bytes <= GRAPHICS_PB_HIGH + GRAPHICS_PB_BYTES as u64; + let low = child.base >= GRAPHICS_PB_OFFSET + && child.base + child.bytes <= GRAPHICS_PB_OFFSET + GRAPHICS_PB_COMPACT_BYTES as u64; + if !high && !low { return false; } + i += 1; + } + true +} +const _: () = assert!(pb_regions_contained()); diff --git a/drivers/gpu/drm/asahi/g18/boot/compute_session.rs b/drivers/gpu/drm/asahi/g18/boot/compute_session.rs new file mode 100644 index 0000000000000..8af55d934298f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/boot/compute_session.rs @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Application compute submission on the retained firmware session. +use super::*; + +impl G18Session { + #[inline(never)] + pub(super) fn submit_application_compute(&mut self, vm: &mmu::Vm, + translated: &crate::g18::uapi::command::TranslatedComputeCommand) + -> core::result::Result<(), Failure> { + self.poll_idle_faults("application compute prepare")?; + let (command, pool, generation, knobs) = self.memory.as_mut().ok_or(EIO)? + .prepare_application_compute(&self.dev, vm, translated)?; + if generation == 1 { + self.queue_probe.as_mut().ok_or(EIO)?.prepare_initial_context(command.context_id)?; + for phase in 1..=3 { + self.check_faults("application compute binding")?; + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + match phase { + 1 => { + if probe.counters()? != [0, 0, 1] || probe.slot() != [0; 5] { + return Err(EBUSY.into()); + } + }, + 2 => probe.prepare_scheduler_bind()?, + _ => probe.prepare_pool_bind(pool)?, + } + let after = self.memory.as_mut().ok_or(EIO)? + .publish_application_compute_outer(phase)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + let mut passed = false; + for _ in 0..CONTROL_POLLS { + self.check_faults("application compute binding")?; + let outer = self.memory.as_mut().ok_or(EIO)?.compute_outer_counters()?; + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + if outer == [after; 3] && probe.counters()? == [phase; 3] + && G18QueueProbe::installed(probe.slot()) + && (phase < 2 || G18QueueProbe::scheduler_bound(probe.scheduler_status()?)) + && (phase < 3 || G18QueueProbe::pool_bound_for_context( + probe.pool_status(), pool, command.context_id)) { + passed = true; break; + } + fsleep(Delta::from_millis(1)); + } + if !passed { return Err(ETIMEDOUT.into()); } + dev_info!(self.device(), "G18_APP_COMPUTE_BIND_PASS phase={}\n", phase); + } + } + let (previous_pool, bound_context) = self.memory.as_ref().ok_or(EIO)? + .application_compute_previous_binding()?; + let previous_context = if generation == 1 { command.context_id } else { bound_context }; + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + if let Some(what) = probe.compute_unready(previous_pool, previous_context, generation)? { + let (counters, slot, scheduler, pool_words) = probe.compute_state()?; + dev_err!(self.device(), + "G18_APP_COMPUTE_UNREADY compute={} unready={} counters={:?} slot={:x?} scheduler={:x?} pool={:x?}\n", + generation, what, counters, slot, scheduler, pool_words); + return Err(EINVAL.into()); + } + self.memory.as_mut().ok_or(EIO)?.install_application_compute_pool(&self.dev)?; + self.queue_probe.as_mut().ok_or(EIO)? + .prepare_application_compute_rebind(command, pool, previous_pool, previous_context, generation)?; + self.poll_idle_faults("application compute pre-kick")?; + let memory = self.memory.as_mut().ok_or(EIO)?; + let completed = memory.compute().completion_snapshot()?; + if completed.stamps != [0; 2] || completed.timestamps != [0; 2] + || memory.completion_masks() != [0; 2] { return Err(EBUSY.into()); } + // The exact knob snapshot this command was built from (g18_app_*). + crate::g18_kick_info!(self.device(), "G18_APP_COMPUTE_KNOBS compute={} {}\n", generation, knobs); + let memory = self.memory.as_mut().ok_or(EIO)?; + let roots = memory.compute().publish_context()?; + if memory.scheduler().publish_kick(0)? != generation { return Err(EIO.into()); } + self.queue_probe.as_mut().ok_or(EIO)?.publish_compute_job()?; + let inner_after = (3 + generation * 3) % 256; + let outer_after = self.memory.as_mut().ok_or(EIO)? + .publish_application_compute_outer(inner_after)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + crate::g18_kick_info!(self.device(), + "G18_APP_KICK compute={} context={} roots={:x?} stamp={:#x}\n", + generation, command.context_id, roots, command.expected_stamp); + let expected_stamp = command.expected_stamp; + // Phases: outer, inner, scheduler submitted, completed, linked, stamp, + // completion mask, pool binding visibility. + let kicked = Instant::::now(); + let mut first = [0i64; 8]; + let mut gpu_ticks = 0u64; + let mut pool_pending_reported = false; + let end = self.wait_job("application compute completion", |session, ms, recovered, report| { + let inner = session.queue_probe.as_mut().ok_or(EIO)?.counters()?; + let pool_words = session.queue_probe.as_ref().ok_or(EIO)?.pool_status(); + let pool_bound = G18QueueProbe::pool_bound_for_context( + pool_words, pool, command.context_id); + let memory = session.memory.as_mut().ok_or(EIO)?; + let outer = memory.compute_outer_counters()?; + let completed = memory.compute().completion_snapshot()?; + let scheduler = memory.scheduler().snapshot(0)?; + let masks = memory.completion_masks(); + // What the firmware owns: the queue consumed and the scheduler + // retired. After a recovery the dispatch never completes, so its + // stamp, timestamps and completion event are not required. + let consumed = outer == [outer_after; 3] && inner == [inner_after; 3] + && scheduler.submitted == generation && scheduler.completed == generation + && scheduler.linked == 0; + let finished = completed.stamps[0] == expected_stamp + && completed.timestamps[0] != 0 + && completed.timestamps[1] >= completed.timestamps[0] + && masks[0] & (1 << QUEUE_ID) != 0; + recovery::note_phases(&mut first, &kicked, &[outer == [outer_after; 3], inner == [inner_after; 3], + scheduler.submitted == generation, scheduler.completed == generation, scheduler.linked == 0, + completed.stamps[0] == expected_stamp, masks[0] & (1 << QUEUE_ID) != 0, pool_bound]); + if finished { + gpu_ticks = completed.timestamps[1] - completed.timestamps[0]; + } + // Queue/stamp completion can precede an observable pool-table + // update. Keep both owners until the full binding is visible, + // within the same bounded wait as command retirement. + if consumed && finished && !pool_bound && !pool_pending_reported { + dev_info!(session.device(), + "G18_APP_COMPUTE_POOL_PENDING generation={} binding={:?} observed={:x?}; owners retained\n", + generation, pool, pool_words); + pool_pending_reported = true; + } + if report { + dev_info!(session.device(), + "G18_APP_COMPUTE_WAIT generation={} ms={} recovered={} outer={:?} inner={:?} stamps={:x?} timestamps={:x?} masks={:x?} scheduler={:?} expected_outer={} expected_inner={} expected_stamp={:#x} consumed={} finished={} pool_bound={} pool={:x?}\n", + generation, ms, recovered, outer, inner, completed.stamps, completed.timestamps, + masks, scheduler, outer_after, inner_after, expected_stamp, consumed, finished, pool_bound, pool_words); + } + Ok(consumed && ((finished && pool_bound) || recovered)) + })?; + let recovery = match end { + JobEnd::Retired => match self.poll_faults("application compute retired")? { + None => { + let observed = self.queue_probe.as_ref().ok_or(EIO)?.pool_status(); + if !G18QueueProbe::pool_bound_for_context(observed, pool, command.context_id) { + dev_err!(self.device(), + "G18_COMPUTE_POOL_VERIFY_FAILED compute={} binding={:?} observed={:x?}; both owners retained\n", + generation, pool, observed); + return Err(EIO.into()); + } + if pool.generation != previous_pool.generation { + crate::g18_kick_info!(self.device(), + "G18_COMPUTE_POOL_VERIFY compute={} generation={} state={:#x} context={} slot={} observed={:x?} verified=1\n", + generation, pool.generation, pool.state, command.context_id, pool.slot, observed); + } + if recovery::phases_due(generation) { + dev_info!(self.device(), + "G18_JOB_PHASES compute={} us={:?} gpu_ticks={} (outer inner submitted completed linked stamp mask pool)\n", + generation, first, gpu_ticks); + } + self.queue_probe.as_mut().ok_or(EIO)?.retire_application_compute(generation)?; + self.memory.as_mut().ok_or(EIO)? + .retire_application_compute(generation, command.context_id)?; + crate::g18_kick_info!(self.device(), + "G18_APP_FRAME_PASS compute={} stamp={:#x} scheduler={} outer={} inner={}\n", + generation, expected_stamp, generation, outer_after, inner_after); + return Ok(()); + } + // Retired, but a recovery followed at once: do not vouch for + // the dispatch. Its queue is retired either way. + Some(recovery) => recovery, + }, + JobEnd::Recovered(recovery) => recovery, + JobEnd::Stranded(recovery) => { + self.memory.as_mut().ok_or(EIO)?.quarantine_application_compute(); + dev_err!(self.device(), + "G18_APP_STRANDED compute={} epoch={} host={}: queue never retired; compute quarantined, owners retained\n", + generation, recovery.epoch, recovery.host); + return Err(Failure::Job(JobFailure::Stranded(recovery, JobPath::Compute))); + } + }; + self.queue_probe.as_mut().ok_or(EIO)?.retire_application_compute(generation)?; + self.memory.as_mut().ok_or(EIO)?.retire_recovered_application_compute(generation)?; + dev_err!(self.device(), "G18_APP_RECOVERED compute={} epoch={} host={}: queue retired\n", + generation, recovery.epoch, recovery.host); + // The next job needs the queue's firmware bindings as they were; a + // recovery may reset them. Refuse the path rather than kick into it. + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + let unready = probe.compute_unready(pool, command.context_id, generation + 1)?; + let (counters, slot, scheduler, pool_words) = probe.compute_state()?; + dev_err!(self.device(), + "G18_APP_COMPUTE_POSTRECOVERY compute={} unready={:?} counters={:?} slot={:x?} scheduler={:x?} pool={:x?}\n", + generation, unready, counters, slot, scheduler, pool_words); + match unready { + None => Err(Failure::Job(JobFailure::Recovered(recovery))), + Some(what) => Err(Failure::Job(JobFailure::Unbound(recovery, JobPath::Compute, what))), + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/boot/mod.rs b/drivers/gpu/drm/asahi/g18/boot/mod.rs new file mode 100644 index 0000000000000..7cb051ec680f6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/boot/mod.rs @@ -0,0 +1,2132 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! J873 / T8152 / G18G (26A428, RTKit 3514.0.15) staged live boot. +//! +//! Plan milestone M1 step 1.8 plus the M0 probes. Derived from +//! `m5_live_boot.rs`; the M5 (T8142) path is not touched by anything here. +//! +//! The stage comes from `aurora.g18_initdata_test`. The driver's T8152 probe +//! gate ([`crate::driver`]) has already sampled the three GPU PSD services, +//! mapped SGX and, only if the sgx PSD proved the block powered, read the ID +//! registers. It hands everything over in a [`T8152Probe`]. +//! +//! | stage | effect | approval | +//! |-------|--------|----------| +//! | 0 | (driver) refuse with ENODEV before any MMIO | none | +//! | 1 | read-only: PSD state, SGX ID, both wrappers' CPU control/status, GFX/GFX1 post-office status words. Sends nothing. ENODEV | none | +//! | 2 | GPU must ALREADY be powered (iBoot or stage 90). Start GFX1 then GFX ASC CPU, RTKit HELLO/EPMAP with syslog and crashlog attached, 2 s syslog window, stop. ENODEV | REQUIRES USER APPROVAL | +//! | 3 | stage 2 + init-data graph, publish GFX then GFX1 root on EP 0x20 (0x81 doorbell), wait for each 0x0009 ACK, stop. ENODEV | REQUIRES USER APPROVAL | +//! | 4 | stage 3 + 0x89 power-transition-end to both roles, EP 0x21 idle-policy wake, passive 60 s event/trace drain, stop. Returns the retained owner | REQUIRES USER APPROVAL | +//! | 5 | stage 4 plus one owned empty queue registration before the passive observation. No GPU job. Returns the retained owner | REQUIRES USER APPROVAL | +//! | 6 | stage 5 plus a second tag15 binding the owned scheduler/control object. No GPU job | REQUIRES USER APPROVAL | +//! | 7 | stage 6 plus the owned initial UMA pool binding. No GPU job or grow request | REQUIRES USER APPROVAL | +//! | 90 | PMGR2 PSD request 0 for sgx, service 3, ONLY. Poll ACK before reading STATE. No automatic restore. ENODEV | REQUIRES USER APPROVAL | +//! | 91 | request 2 for gfx-asc, service 81 (sgx must already read state 0) | REQUIRES USER APPROVAL | +//! | 92 | request 2 for gfx1-asc, service 82 (sgx must already read state 0) | REQUIRES USER APPROVAL | +//! | 93 | restore: idle request (6) on every GPU service not already idle, in reverse native order 81, 82, 3. ENODEV | REQUIRES USER APPROVAL | +//! +//! Stages 2..7 start the ASC CPUs, which the mailbox provider additionally +//! refuses unless `apple_mailbox.ascwrap_v8_cpu_control=1` (also REQUIRES +//! USER APPROVAL). Stages 90..93 are the only code in the driver that writes +//! a PSD request register; each writes one service (93: only the services +//! it restores), and none is ever implied by another stage. Native order is +//! 90, 92, 91. J873 hardware confirms STATE reads consume ACK: transition +//! waits poll ACK before reading STATE once. Later access gates require +//! each service's active state without requiring an already-consumed ACK. +//! +//! The GFX/GFX1 mailbox providers exist only when the loader enabled their +//! nodes, which it does together with the firmware receipt. A missing +//! provider is reported and turned into ENODEV, never EPROBE_DEFER, so the +//! driver core cannot re-run a stage in a deferral loop. The providers +//! receive by polling unless `apple_mailbox.ascwrap_v8_rx_irq=1` (REQUIRES +//! USER APPROVAL), so no GPU mailbox interrupt is enabled by default. +//! +//! Safety rules this module follows: +//! - Every SGX or ASC wrapper MMIO read follows a PSD read of that block that +//! proves it powered. Wrapper reads go through the provider's C helpers, +//! which read the role's PSD block first; SGX reads use +//! [`regs::t8152_prove_sgx_powered`]. The post-office status word is read +//! only after this module read the role's PSD block as powered. +//! - Only the provider's own 0x80-byte post-office window is touched (the +//! status word at +0x20); never the rest of the shared bank. +//! - Once an ASC CPU was asked to start, no firmware-visible owner (UAT, +//! init-data graph, RTKit transport) is ever freed: the session is stopped +//! and then leaked, and the module is pinned, until reboot. There is one +//! firmware init per boot; a second attempt is refused. +//! - On any firmware fault the crashlog is dumped: the RTKit crash callback +//! dumps a host buffer when there is one, and otherwise (and after any +//! failed stage) scans the role's `__DATA` reservation for the `CLHE` +//! crashlog header and dumps what it finds. + +use crate::{ + driver::AsahiDrmDevice, + g18::probe::{G18Runtime, T8152Probe}, + g18::initdata::G18FirmwareMemory, + g18::runtime::compute::G18PreparedComputeCommand, + g18::buffer::pool::G18UmaPoolBinding, + g18::queue::probe::{G18QueueProbe, QUEUE_ID}, + g18::fw::rtkit as g17_rtkit, mmu, g18::regs as regs, +}; +use core::{ + ffi::c_void, + mem::MaybeUninit, + ptr::NonNull, + sync::atomic::{AtomicBool, AtomicU32, Ordering}, +}; +use kernel::{ + bindings, c_str, + device::Core, + error::{from_err_ptr, to_result}, + io::{mem::DevresIoMem, Io, IoSysMap, Mmio, MmioRaw, Region}, + macros::vtable, + new_mutex, platform, + prelude::*, + soc::apple::rtkit, + sync::{aref::ARef, Arc, Mutex}, + time::{delay::fsleep, Delta, Instant, Monotonic}, +}; + +// --------------------------------------------------------------------------- +// Stages +// --------------------------------------------------------------------------- + +/// Read-only probe. +pub(crate) const STAGE_READ_ONLY: u32 = 1; +/// REQUIRES USER APPROVAL: ASC CPU start + RTKit HELLO/EPMAP. +pub(crate) const STAGE_RTKIT: u32 = 2; +/// REQUIRES USER APPROVAL: stage 2 + init-data roots and 0x0009 ACKs. +pub(crate) const STAGE_INITDATA: u32 = 3; +/// REQUIRES USER APPROVAL: stage 3 + openings, idle wake, 60 s heartbeat. +pub(crate) const STAGE_HEARTBEAT: u32 = 4; +/// Owned empty queue registration, then the same 60 s stability observation. +pub(crate) const STAGE_QUEUE_REGISTER: u32 = 5; +/// Register an owned queue and bind its scheduler/control, without a GPU job. +pub(crate) const STAGE_SCHEDULER_BIND: u32 = 6; +/// Bind the owned empty compute UMA pool, without submitting a GPU job. +pub(crate) const STAGE_POOL_BIND: u32 = 7; +/// One captured addition with retained resources and independent completion/output gates. +pub(crate) const STAGE_COMPUTE_ADD: u32 = 8; +pub(crate) const STAGE_RENDER: u32 = crate::g18::boot::render_stage::STAGE_RENDER; +pub(crate) const STAGE_DRM: u32 = 10; +pub(crate) mod render_stage; + +#[path = "session.rs"] +mod application; +pub(crate) use application::start_drm; +/// REQUIRES USER APPROVAL: PMGR2 PSD power-on request for sgx (service 3) +/// only. Standalone; no other stage requests PSD power. +pub(crate) const STAGE_PSD_POWER_ON: u32 = 90; +/// REQUIRES USER APPROVAL: as [`STAGE_PSD_POWER_ON`] for gfx-asc (81). +pub(crate) const STAGE_PSD_POWER_ON_GFX: u32 = 91; +/// REQUIRES USER APPROVAL: as [`STAGE_PSD_POWER_ON`] for gfx1-asc (82). +pub(crate) const STAGE_PSD_POWER_ON_GFX1: u32 = 92; +/// REQUIRES USER APPROVAL: idle request on every non-idle GPU service. +pub(crate) const STAGE_PSD_RESTORE_IDLE: u32 = 93; + +/// Stage values this module implements. The driver refuses anything else +/// with EINVAL before any MMIO. +pub(crate) const fn stage_is_known(stage: u32) -> bool { + matches!( + stage, + STAGE_READ_ONLY + | STAGE_RTKIT + | STAGE_INITDATA + | STAGE_HEARTBEAT + | STAGE_QUEUE_REGISTER + | STAGE_SCHEDULER_BIND + | STAGE_POOL_BIND + | STAGE_COMPUTE_ADD + | STAGE_RENDER + | STAGE_DRM + | STAGE_PSD_POWER_ON + | STAGE_PSD_POWER_ON_GFX + | STAGE_PSD_POWER_ON_GFX1 + | STAGE_PSD_RESTORE_IDLE + ) +} + +/// The single PSD service a power-on stage requests, if `stage` is one. +const fn psd_power_on_service(stage: u32) -> Option { + match stage { + STAGE_PSD_POWER_ON => Some(regs::T8152PsdService::Sgx), + STAGE_PSD_POWER_ON_GFX => Some(regs::T8152PsdService::GfxAsc), + STAGE_PSD_POWER_ON_GFX1 => Some(regs::T8152PsdService::Gfx1Asc), + _ => None, + } +} + +/// Stages that write a PSD request register. +const fn stage_writes_psd(stage: u32) -> bool { + psd_power_on_service(stage).is_some() || stage == STAGE_PSD_RESTORE_IDLE +} + +/// Stages that run the loaded firmware and therefore need the loader's +/// G18G firmware receipt and the 42-bit DMA mask. +pub(crate) const fn stage_runs_firmware(stage: u32) -> bool { + matches!(stage, STAGE_RTKIT | STAGE_INITDATA | STAGE_HEARTBEAT + | STAGE_QUEUE_REGISTER | STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD | STAGE_RENDER | STAGE_DRM) +} + +/// Set once any firmware stage reached its first ASC CPU start request. +/// The firmware initializes once per boot; the module is pinned from then on, +/// so this survives unbind/rebind and refuses a second attempt. +static G18_FIRMWARE_ATTEMPTED: AtomicBool = AtomicBool::new(false); +/// Bit per PSD request kind written by this module instance: bit 0 sgx, +/// bit 1 gfx-asc, bit 2 gfx1-asc (power-on), bit 3 restore. Each kind is +/// written at most once per module load; a re-bind does not repeat it. +static G18_PSD_REQUESTS: AtomicU32 = AtomicU32::new(0); +const PSD_REQUEST_RESTORE_BIT: u32 = 1 << 3; + +const fn psd_request_bit(service: regs::T8152PsdService) -> u32 { + match service { + regs::T8152PsdService::Sgx => 1 << 0, + regs::T8152PsdService::GfxAsc => 1 << 1, + regs::T8152PsdService::Gfx1Asc => 1 << 2, + } +} + +// --------------------------------------------------------------------------- +// Fixed 26A428 protocol facts (plan step 1.7, FW_PROVEN unless noted) +// --------------------------------------------------------------------------- + +const EP_GFX_MESSAGES: u8 = g17_rtkit::ENDPOINT_GFX_MESSAGES; // 0x20 +const EP_GFX_INTERRUPTS: u8 = g17_rtkit::ENDPOINT_GFX_INTERRUPTS; // 0x21 +/// Init-data ACK on EP 0x20 (b000 0x24c78), identical to M5. +const INITDATA_ACK: u64 = 0x0009_0000_0000_0000; +/// EP 0x21 device-control doorbell for the idle-policy wake (M5 value, +/// unchanged in the 26A428 EP 0x21 handler 0x254cc-0x25610). +const IDLE_POLICY_WAKE: u64 = 0x0084_0000_0000_0011; + +const ACK_POLLS: u32 = 200; // x 10 ms +const CONTROL_POLLS: u32 = 1000; // x 1 ms +const SYSLOG_WINDOW_MS: u32 = 2000; +const HEARTBEAT_MS: u32 = 60_000; +const HEARTBEAT_STEP_MS: u32 = 100; +const MESSAGE_LOG_BUDGET: u32 = 32; + +/// PSD service block layout (the GPU power tool's contract). +const PSD_STATE: usize = 0x80; +const PSD_REQUEST: usize = 0x90; +const PSD_ACK: usize = 0xbc; +const PSD_BLOCK_SIZE: u64 = 0x100; +/// Cold-boot idle state of all three GPU services on J873. +const PSD_STATE_IDLE: u32 = 6; +const PSD_POLLS: u32 = 1000; // x 1 ms, as gpu_power.py (1 s) + +/// ASCWrap v8 CPU control RUN bit (m1n1 asc.c; INFERRED for the GFX +/// wrappers, see the mailbox provider). +const ASCWRAP_V8_CPU_RUN: u32 = 1 << 4; +/// Counted-FIFO status word inside the role's 0x80-byte post-office window. +const POSTOFFICE_STATUS: usize = 0x20; +const POSTOFFICE_STATUS_WINDOW: usize = POSTOFFICE_STATUS + core::mem::size_of::(); + +/// RTKit crashlog container (`CLHE` header, sections, `CLHE` trailer). +const fn fourcc(a: u8, b: u8, c: u8, d: u8) -> u32 { + ((a as u32) << 24) | ((b as u32) << 16) | ((c as u32) << 8) | d as u32 +} +const CRASHLOG_HEADER: u32 = fourcc(b'C', b'L', b'H', b'E'); +const CRASHLOG_STRING: u32 = fourcc(b'C', b's', b't', b'r'); +const CRASHLOG_REGISTERS: u32 = fourcc(b'C', b'r', b'g', b'8'); +/// Largest crashlog this module copies and hex-dumps (the ADT +/// coredump-copy-buffer is 0x8000). +const CRASHLOG_MAX_BYTES: usize = 0x8000; +/// Granule of the `__DATA` crashlog scan. +const CRASHLOG_SCAN_STEP: usize = 0x1000; + +// --------------------------------------------------------------------------- +// Roles +// --------------------------------------------------------------------------- + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum G18Role { + /// Mailbox index 0, ASC `gfx-asc`, post-office mailbox 1. + Gfx, + /// Mailbox index 1, ASC `gfx1-asc`, post-office mailbox 2. + Gfx1, +} + +impl G18Role { + const ALL: [Self; 2] = [Self::Gfx, Self::Gfx1]; + /// M5 order, carried to G18G (INFERRED; the 26A428 bootFirmware publish + /// order is not re-derived): GFX1 starts and publishes first. + const START_ORDER: [Self; 2] = [Self::Gfx1, Self::Gfx]; + /// Reverse of the start order. + const STOP_ORDER: [Self; 2] = [Self::Gfx, Self::Gfx1]; + + const fn index(self) -> usize { + match self { + Self::Gfx => 0, + Self::Gfx1 => 1, + } + } + + const fn name(self) -> &'static str { + match self { + Self::Gfx => "GFX", + Self::Gfx1 => "GFX1", + } + } + + /// `enum apple_mbox_ascwrap_v6_role` value the provider reports. + const fn lifecycle_role(self) -> u32 { + match self { + Self::Gfx => 1, + Self::Gfx1 => 2, + } + } + + /// ADT gfx-asc/gfx1-asc reg[0] (iop,ascwrap-v8). + const fn wrapper(self) -> (u64, u64) { + match self { + Self::Gfx => (0x5_8260_0000, 0x7_4000), + Self::Gfx1 => (0x5_82e0_0000, 0x7_4000), + } + } + + /// ADT gfx-asc/gfx1-asc reg[1]. + const fn iop_vbar(self) -> (u64, u64) { + match self { + Self::Gfx => (0x5_8205_0000, 0x8), + Self::Gfx1 => (0x5_8285_0000, 0x8), + } + } + + /// The role's 0x80-byte post-office window (informational). + const fn postoffice_window(self) -> u64 { + match self { + Self::Gfx => 0x3_1602_0100, + Self::Gfx1 => 0x3_1602_0200, + } + } + + const fn psd(self) -> regs::T8152PsdService { + match self { + Self::Gfx => regs::T8152PsdService::GfxAsc, + Self::Gfx1 => regs::T8152PsdService::Gfx1Asc, + } + } + + const fn data_region_name(self) -> &'static CStr { + match self { + Self::Gfx => c_str!("gfx-data"), + Self::Gfx1 => c_str!("gfx1-data"), + } + } + + /// The `__DATA_SHARED_RO` reservation right after `__DATA`. On J873 the + /// RTKit crash callback reports the firmware-owned crash buffer here + /// (e.g. GFX1: 0x10003900000, 0x8000 bytes). + const fn shared_ro_region_name(self) -> &'static CStr { + match self { + Self::Gfx => c_str!("gfx-data-shared-ro"), + Self::Gfx1 => c_str!("gfx1-data-shared-ro"), + } + } + + /// 26A428 `__DATA` sizes (b000 / b010), as `Uat::new_t8152` checks. + const fn data_size(self) -> u64 { + match self { + Self::Gfx => 0xb_c000, + Self::Gfx1 => 0xb_4000, + } + } +} + +// Compile-time guards on the stage gates and the fixed J873 facts. +const _: () = { + assert!(!stage_is_known(0) && !stage_is_known(11) && !stage_is_known(89) && !stage_is_known(94)); + assert!(stage_is_known(STAGE_RENDER) && stage_runs_firmware(STAGE_RENDER)); + assert!(!stage_writes_psd(STAGE_RENDER)); + assert!(stage_is_known(STAGE_READ_ONLY) && stage_is_known(STAGE_PSD_POWER_ON)); + assert!(stage_is_known(STAGE_PSD_POWER_ON_GFX) && stage_is_known(STAGE_PSD_POWER_ON_GFX1)); + assert!(stage_is_known(STAGE_PSD_RESTORE_IDLE)); + // Neither the read-only probe nor a power stage runs firmware, and no + // firmware stage writes PSD. + assert!(!stage_runs_firmware(STAGE_READ_ONLY) && !stage_writes_psd(STAGE_READ_ONLY)); + assert!(!stage_runs_firmware(STAGE_PSD_POWER_ON) && !stage_runs_firmware(STAGE_PSD_POWER_ON_GFX)); + assert!(!stage_runs_firmware(STAGE_PSD_POWER_ON_GFX1) && !stage_runs_firmware(STAGE_PSD_RESTORE_IDLE)); + assert!(stage_runs_firmware(STAGE_RTKIT) && stage_runs_firmware(STAGE_INITDATA)); + assert!(stage_runs_firmware(STAGE_HEARTBEAT)); + assert!(stage_is_known(STAGE_POOL_BIND) && stage_runs_firmware(STAGE_POOL_BIND)); + assert!(stage_is_known(STAGE_COMPUTE_ADD) && stage_runs_firmware(STAGE_COMPUTE_ADD)); + assert!(!stage_writes_psd(STAGE_RTKIT) && !stage_writes_psd(STAGE_INITDATA)); + assert!(!stage_writes_psd(STAGE_HEARTBEAT)); + assert!(!stage_writes_psd(STAGE_POOL_BIND)); + assert!(!stage_writes_psd(STAGE_COMPUTE_ADD)); + // Exactly one service per power-on stage, sgx first. + assert!(matches!(psd_power_on_service(STAGE_PSD_POWER_ON), Some(regs::T8152PsdService::Sgx))); + assert!(matches!(psd_power_on_service(STAGE_PSD_POWER_ON_GFX), Some(regs::T8152PsdService::GfxAsc))); + assert!(matches!(psd_power_on_service(STAGE_PSD_POWER_ON_GFX1), Some(regs::T8152PsdService::Gfx1Asc))); + assert!(psd_power_on_service(STAGE_PSD_RESTORE_IDLE).is_none() && stage_writes_psd(STAGE_PSD_RESTORE_IDLE)); + assert!(psd_request_bit(regs::T8152PsdService::Sgx) | psd_request_bit(regs::T8152PsdService::GfxAsc) + | psd_request_bit(regs::T8152PsdService::Gfx1Asc) | PSD_REQUEST_RESTORE_BIT == 0xf); + // GFX1 starts and publishes first; stop is the reverse. + assert!(G18Role::START_ORDER[0].index() == 1 && G18Role::START_ORDER[1].index() == 0); + assert!(G18Role::STOP_ORDER[0].index() == 0 && G18Role::STOP_ORDER[1].index() == 1); + // PSD services 81/82 and the post-office windows of mailboxes 1/2 + // ((2 * mb + !po) * 0x80 into 0x316020000). + assert!(G18Role::Gfx.psd().physical_base() == 0x3_03a1_5100); + assert!(G18Role::Gfx1.psd().physical_base() == 0x3_03a1_5200); + assert!(G18Role::Gfx.postoffice_window() == 0x3_1602_0000 + 2 * 0x80); + assert!(G18Role::Gfx1.postoffice_window() == 0x3_1602_0000 + 4 * 0x80); + assert!(POSTOFFICE_STATUS_WINDOW <= 0x80); + // __DATA sizes are 16 KiB multiples and hold at least one crash view. + assert!(G18Role::Gfx.data_size() % 0x4000 == 0 && G18Role::Gfx1.data_size() % 0x4000 == 0); + assert!(CRASHLOG_MAX_BYTES as u64 <= G18Role::Gfx1.data_size()); + assert!(CRASHLOG_HEADER == 0x434c_4845); +}; + +// --------------------------------------------------------------------------- +// PSD service blocks +// --------------------------------------------------------------------------- + +const fn psd_resource_name(service: regs::T8152PsdService) -> &'static CStr { + match service { + regs::T8152PsdService::Sgx => c_str!("psd-sgx"), + regs::T8152PsdService::GfxAsc => c_str!("psd-gfx-asc"), + regs::T8152PsdService::Gfx1Asc => c_str!("psd-gfx1-asc"), + } +} + +/// One GPU PSD service block, mapped (not claimed) from the GPU node for the +/// duration of a stage. Only [`PsdBlock::request`] writes, and only stages +/// 90..93 call it. +struct PsdBlock { + service: regs::T8152PsdService, + io: DevresIoMem, +} + +impl PsdBlock { + fn map(pdev: &platform::Device>, service: regs::T8152PsdService) -> Result { + let name = psd_resource_name(service); + let resource = pdev.resource_by_name(name).ok_or(ENODEV)?; + if resource.start() != service.physical_base() || resource.size() != PSD_BLOCK_SIZE { + dev_err!( + pdev.as_ref(), + "G18_PSD service {} resource {:#x}+{:#x} is not {:#x}+{:#x}\n", + service.service_id(), + resource.start(), + resource.size(), + service.physical_base(), + PSD_BLOCK_SIZE + ); + return Err(EINVAL); + } + let request = pdev.io_request_by_name(name).ok_or(ENODEV)?; + let io = request.iomap()?.into_devres()?; + Ok(Self { service, io }) + } + + fn sample(&self) -> Result { + let io = self.io.try_access().ok_or(ENODEV)?; + // Reading STATE consumes ACK; capture ACK first for telemetry. + let ack = io.try_read32(PSD_ACK)?; + let state = io.try_read32(PSD_STATE)?; + Ok(regs::T8152PsdSample { + service: self.service, + state, + ack, + }) + } + + /// Poll ACK without touching STATE until the transition completes. + fn acknowledged_sample(&self) -> Result> { + let io = self.io.try_access().ok_or(ENODEV)?; + let ack = io.try_read32(PSD_ACK)?; + if ack & 1 == 0 { + return Ok(None); + } + let state = io.try_read32(PSD_STATE)?; + Ok(Some(regs::T8152PsdSample { service: self.service, state, ack })) + } + + /// REQUIRES USER APPROVAL. Accept only this service's native active + /// state (SGX 0, ASCs 2) or idle (6). Called by stages 90..93 only. + fn request(&self, value: u32) -> Result { + if value != self.service.active_state() && value != PSD_STATE_IDLE { + return Err(EINVAL); + } + let io = self.io.try_access().ok_or(ENODEV)?; + io.try_write32(value, PSD_REQUEST) + } +} + +fn psd_line(sample: ®s::T8152PsdSample) -> (u32, u32, u32, u32, &'static str) { + ( + sample.service.service_id(), + sample.state, + sample.power_state(), + sample.ack, + sample.verdict(), + ) +} + +fn log_psd(pdev: &platform::Device>, label: &str, sample: ®s::T8152PsdSample) { + let (id, state, power, ack, verdict) = psd_line(sample); + dev_info!( + pdev.as_ref(), + "G18_PSD {} service={} state={:#x} power-state={} ack={:#x} -> {}\n", + label, + id, + state, + power, + ack, + verdict + ); +} + +/// Fresh read-only samples of all three GPU services. +fn read_all_psd(pdev: &platform::Device>) -> Result<[regs::T8152PsdSample; 3]> { + let mut samples = [regs::T8152PsdSample { + service: regs::T8152PsdService::Sgx, + state: 0, + ack: 0, + }; 3]; + for (slot, service) in samples.iter_mut().zip(regs::T8152PsdService::ALL) { + *slot = regs::t8152_read_psd(pdev, service)?; + } + Ok(samples) +} + +// --------------------------------------------------------------------------- +// Mailbox providers (apple,t8152-gfx-ascwrap-v8) +// --------------------------------------------------------------------------- + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +struct G18Lifecycle { + role: u32, + wrapper: (u64, u64), + iop_vbar: (u64, u64), + missing: u32, +} + +/// Borrowed view of one ASCWrap v8 provider. The C provider owns every +/// mapping and every lifecycle transition, and reads the role's PSD block +/// before each wrapper access. This view never touches the wrapper itself; +/// the one direct read it makes is the post-office status word, gated here +/// on the role's PSD block. +struct G18Provider { + mbox: *mut bindings::apple_mbox, + role: G18Role, +} + +// SAFETY: the provider is kept alive by the device link apple_mbox_get() +// establishes; every mutation is serialized inside the C provider. +unsafe impl Send for G18Provider {} +// SAFETY: as above; the only Rust-side MMIO is one relaxed status read. +unsafe impl Sync for G18Provider {} + +impl G18Provider { + /// Look up the role's provider. A provider that is not bound makes + /// apple_mbox_get() return EPROBE_DEFER; that is reported and turned into + /// ENODEV here so that no stage can put the GPU device into a deferral + /// loop (every retry would repeat the PSD and SGX reads). The loader + /// enables both mailbox nodes only together with the firmware receipt. + fn get(dev: &kernel::device::Device, role: G18Role) -> Result { + // SAFETY: `dev` is live; apple_mbox_get() links the provider to it. + let mbox = unsafe { + from_err_ptr(bindings::apple_mbox_get(dev.as_raw(), role.index().try_into()?)) + }; + match mbox { + Ok(mbox) => Ok(Self { mbox, role }), + Err(error) => Err(no_probe_defer(dev, role.name(), "mailbox provider", error)), + } + } + + fn lifecycle(&self) -> Result { + let mut raw = MaybeUninit::::uninit(); + // SAFETY: `self.mbox` is a live provider and `raw` is writable. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_lifecycle(self.mbox, raw.as_mut_ptr()) + })?; + // SAFETY: the successful call initialized every field. + let raw = unsafe { raw.assume_init() }; + Ok(G18Lifecycle { + role: raw.role as u32, + wrapper: (raw.wrapper_start, raw.wrapper_size), + iop_vbar: (raw.iop_vbar_start, raw.iop_vbar_size), + missing: raw.missing, + }) + } + + /// Exact role and resources of the J873 v8 provider. The 0x74000 + /// wrapper size identifies the v8 provider, whose `regs` mapping is the + /// role's 0x80-byte post-office window. + fn validate(&self, dev: &kernel::device::Device) -> Result { + let lifecycle = self.lifecycle()?; + if lifecycle.role != self.role.lifecycle_role() + || lifecycle.wrapper != self.role.wrapper() + || lifecycle.iop_vbar != self.role.iop_vbar() + { + dev_err!( + dev, + "G18 {} mailbox provider mismatch: role={} wrapper={:x?} iop-vbar={:x?}; expected role={} wrapper={:x?} iop-vbar={:x?}\n", + self.role.name(), + lifecycle.role, + lifecycle.wrapper, + lifecycle.iop_vbar, + self.role.lifecycle_role(), + self.role.wrapper(), + self.role.iop_vbar() + ); + return Err(EINVAL); + } + Ok(lifecycle) + } + + fn require_safe_cpu_lifecycle(&self) -> Result { + // SAFETY: capability check on the live provider. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(self.mbox) }) + } + + /// REQUIRES USER APPROVAL (the provider refuses with EPERM unless + /// `apple_mailbox.ascwrap_v8_cpu_control=1`; it also refuses an unpowered + /// block). + fn start_cpu(&self) -> Result { + // SAFETY: the provider serializes the lifecycle and owns the MMIO. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_start_cpu(self.mbox) }) + } + + /// As [`Self::start_cpu`]; a no-op unless this provider started the CPU. + fn stop_cpu(&self) -> Result { + // SAFETY: as above. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }) + } + + /// CPU_CONTROL (+0x44) and CPU_STATUS (+0x48). The provider reads the + /// role's PSD first and returns EHOSTDOWN, without touching the wrapper, + /// when the block is not powered. + fn cpu_state(&self) -> Result<(u32, u32)> { + let mut control = 0; + let mut status = 0; + // SAFETY: live provider; both outputs are writable stack storage. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_cpu_state(self.mbox, &mut control, &mut status) + })?; + Ok((control, status)) + } + + /// Read the counted-FIFO status word (+0x20) of this role's post-office + /// window, after reading the role's PSD block. Returns `None` for the + /// word when the block is not powered. Call only after [`Self::validate`]. + fn postoffice_status( + &self, + pdev: &platform::Device>, + ) -> Result<(regs::T8152PsdSample, Option)> { + let psd = regs::t8152_read_psd(pdev, self.role.psd())?; + if !psd.powered() { + return Ok((psd, None)); + } + // SAFETY: the provider (device-link protected) owns this mapping. + let (window, removing) = unsafe { + ( + (*self.mbox).regs as usize, + core::ptr::read_volatile(core::ptr::addr_of!((*self.mbox).removing)), + ) + }; + if window == 0 || removing { + return Err(ENODEV); + } + let raw = MmioRaw::>::new_region(window, POSTOFFICE_STATUS_WINDOW)?; + // SAFETY: `raw` lies inside the provider's 0x80-byte window and + // covers exactly the single status read below. + let status = unsafe { Mmio::from_raw(raw) }.relaxed().read32(POSTOFFICE_STATUS); + Ok((psd, Some(status))) + } +} + +/// Map EPROBE_DEFER to ENODEV with an explanation; pass anything else on. +fn no_probe_defer(dev: &kernel::device::Device, role: &str, what: &str, error: Error) -> Error { + if error != EPROBE_DEFER { + return error; + } + dev_err!( + dev, + "G18 {} {} not bound: the GFX/GFX1 mailbox nodes are disabled (the loader enables them only together with the firmware receipt) or the provider failed to probe; ENODEV, not deferring\n", + role, + what + ); + ENODEV +} + +fn log_cpu_state(dev: &kernel::device::Device, role: G18Role, label: &str, provider: &G18Provider) { + match provider.cpu_state() { + Ok((control, status)) => dev_info!( + dev, + "G18_ASC {} {} CPU_CONTROL={:#x} (RUN={}) CPU_STATUS={:#x}\n", + role.name(), + label, + control, + (control & ASCWRAP_V8_CPU_RUN != 0) as u32, + status + ), + Err(error) if error == regs::t8152_not_powered() => dev_info!( + dev, + "G18_ASC {} {} wrapper not read: role PSD not powered\n", + role.name(), + label + ), + Err(error) => dev_err!( + dev, + "G18_ASC {} {} CPU state read failed ({:?})\n", + role.name(), + label, + error + ), + } +} + +// --------------------------------------------------------------------------- +// Crashlog dump +// --------------------------------------------------------------------------- + +fn le_u32(raw: &[u8], offset: usize) -> Option { + Some(u32::from_le_bytes(raw.get(offset..offset.checked_add(4)?)?.try_into().ok()?)) +} + +fn le_u64(raw: &[u8], offset: usize) -> Option { + Some(u64::from_le_bytes(raw.get(offset..offset.checked_add(8)?)?.try_into().ok()?)) +} + +fn fourcc_text(kind: u32) -> [u8; 4] { + let mut text = kind.to_be_bytes(); + for byte in text.iter_mut() { + if !byte.is_ascii_graphic() { + *byte = b'.'; + } + } + text +} + +/// Log a crashlog: parsed sections (strings, the exception register frame +/// with the G17P `Crg8` layout, INFERRED for G18G) and a hex dump of every +/// non-zero 32-byte line, bounded to [`CRASHLOG_MAX_BYTES`]. +fn dump_crashlog(dev: &kernel::device::Device, role: &str, source: &str, raw: &[u8]) { + let header = le_u32(raw, 0).unwrap_or(0); + let declared = le_u32(raw, 8).unwrap_or(0) as usize; + let valid = header == CRASHLOG_HEADER && declared >= 0x20 && declared <= raw.len(); + let total = if valid { declared } else { raw.len() }.min(CRASHLOG_MAX_BYTES); + dev_err!( + dev, + "G18_CRASHLOG {} source={} bytes={} header={} declared={:#x} dumped={:#x}\n", + role, + source, + raw.len(), + core::str::from_utf8(&fourcc_text(header)).unwrap_or("????"), + declared, + total + ); + + if valid { + let mut offset = 0x20usize; + for _ in 0..64 { + let (Some(kind), Some(size)) = (le_u32(raw, offset), le_u32(raw, offset + 12)) else { + break; + }; + let size = size as usize; + if kind == CRASHLOG_HEADER { + break; + } + let Some(end) = offset.checked_add(size) else { + break; + }; + if size < 16 || end > declared { + dev_err!(dev, "G18_CRASHLOG {} section at {:#x} has bad size {:#x}\n", role, offset, size); + break; + } + dev_err!( + dev, + "G18_CRASHLOG {} section {} at {:#x} size {:#x}\n", + role, + core::str::from_utf8(&fourcc_text(kind)).unwrap_or("????"), + offset, + size + ); + let body = offset + 16; + if kind == CRASHLOG_STRING && size >= 20 { + let id = le_u32(raw, body).unwrap_or(0); + let bytes = &raw[body + 4..end]; + let length = bytes.iter().position(|b| *b == 0).unwrap_or(bytes.len()).min(256); + match core::str::from_utf8(&bytes[..length]) { + Ok(message) => dev_err!(dev, "G18_CRASHLOG {} message {}: {}\n", role, id, message), + Err(_) => dev_err!(dev, "G18_CRASHLOG {} message {}: (not UTF-8)\n", role, id), + } + } else if kind == CRASHLOG_REGISTERS && size >= 16 + 0x350 { + // Crg8 begins with an eight-byte frame descriptor, then X0. + // Verified against the 0x401 frame in the stage6 CLHE dump. + let reg = |index: usize| le_u64(raw, body + 8 + index * 8).unwrap_or(0); + for first in (0..31).step_by(4) { + let last = (first + 4).min(31); + let mut words = [0u64; 4]; + for (slot, index) in words.iter_mut().zip(first..last) { + *slot = reg(index); + } + dev_err!(dev, "G18_CRASHLOG {} x{}..x{}: {:x?}\n", role, first, last - 1, + &words[..last - first]); + } + dev_err!( + dev, + "G18_CRASHLOG {} exception pc={:#x} far={:#x} esr={:#x} sp={:#x} psr={:#x}\n", + role, + le_u64(raw, body + 0x108).unwrap_or(0), + le_u64(raw, body + 0x330).unwrap_or(0), + le_u32(raw, body + 0x340).unwrap_or(0), + le_u64(raw, body + 0x100).unwrap_or(0), + le_u64(raw, body + 0x110).unwrap_or(0) + ); + } + offset = end; + } + } + + let mut zero_lines = 0u32; + let mut offset = 0usize; + while offset + 32 <= total { + let mut words = [0u32; 8]; + for (index, word) in words.iter_mut().enumerate() { + *word = le_u32(raw, offset + index * 4).unwrap_or(0); + } + if words.iter().all(|word| *word == 0) { + zero_lines += 1; + } else { + dev_err!( + dev, + "G18_CRASHLOG {} +{:05x}: {:08x} {:08x} {:08x} {:08x} {:08x} {:08x} {:08x} {:08x}\n", + role, offset, words[0], words[1], words[2], words[3], words[4], words[5], + words[6], words[7] + ); + } + offset += 32; + } + dev_err!(dev, "G18_CRASHLOG {} end; {} all-zero lines omitted\n", role, zero_lines); +} + +/// Invalidate the AP's view of a range before reading memory the GPU +/// firmware wrote (clean+invalidate: nothing here is ever written by the AP). +fn invalidate_dcache_range(pointer: *const u8, length: usize) { + let ctr: u64; + // SAFETY: CTR_EL0 is readable at EL1. + unsafe { + core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)); + } + let line = 4usize << ((ctr >> 16) & 0xf); + let mut address = pointer as usize & !(line - 1); + let end = pointer as usize + length; + while address < end { + // SAFETY: `address` lies in a live cacheable mapping owned by the + // caller; clean+invalidate by VA has no architectural side effect on + // the data. + unsafe { + core::arch::asm!("dc civac, {address}", address = in(reg) address, + options(nostack, preserves_flags)); + } + address += line; + } + // SAFETY: barrier only. + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; +} + +/// Read-only write-back mapping of a firmware carveout range (not Linux +/// RAM), used only to find and copy a crashlog. +struct PhysMap { + pointer: NonNull, + size: usize, +} + +impl PhysMap { + fn new(physical: u64, size: usize) -> Result { + if size == 0 || physical & (crate::pgtable::UAT_PGMSK as u64) != 0 { + return Err(EINVAL); + } + // SAFETY: page-aligned firmware reservation validated by the caller + // against the GPU node; released by Drop and only read. + let pointer = unsafe { bindings::memremap(physical, size, bindings::MEMREMAP_WB as _) }; + Ok(Self { pointer: NonNull::new(pointer).ok_or(ENOMEM)?, size }) + } + + fn copy(&self, offset: usize, out: &mut [u8]) -> Result { + let end = offset.checked_add(out.len()).ok_or(EOVERFLOW)?; + if (offset | out.len()) & 3 != 0 || end > self.size { + return Err(EINVAL); + } + // SAFETY: bounds checked above against the mapping size. + let base = unsafe { (self.pointer.as_ptr() as *const u8).add(offset) }; + invalidate_dcache_range(base, out.len()); + for (index, chunk) in out.chunks_exact_mut(4).enumerate() { + // SAFETY: 4-byte aligned, inside the mapping. + let word = unsafe { core::ptr::read_volatile(base.add(index * 4) as *const u32) }; + chunk.copy_from_slice(&word.to_le_bytes()); + } + Ok(()) + } +} + +impl Drop for PhysMap { + fn drop(&mut self) { + // SAFETY: returned by memremap and not yet unmapped. + unsafe { bindings::memunmap(self.pointer.as_ptr()) }; + } +} + +// --------------------------------------------------------------------------- +// RTKit +// --------------------------------------------------------------------------- + +/// Per-role callback state. Callbacks only publish atomics and log. +struct G18RoleState { + dev: ARef, + role: G18Role, + /// The role's `__DATA` reservation (physical base, size). + data: (u64, u64), + /// The role's `__DATA_SHARED_RO` reservation, (0, 0) when absent. + shared_ro: (u64, u64), + acked: AtomicBool, + crashed: AtomicBool, + crash_dumped: AtomicBool, + messages: AtomicU32, +} + +impl G18RoleState { + fn crashed(&self) -> bool { + self.crashed.load(Ordering::Acquire) + } + + /// Scan the role's `__DATA` for a `CLHE` crashlog and dump it. Firmware + /// with a firmware-owned crash buffer (the G17P RTKit mode) hands the + /// host no bytes; on M5 that buffer is `__DATA + 0xa0000`. The G18G + /// offset is unknown, so scan every 4 KiB. Runs at most once per role. + fn dump_data_crashlog(&self, reason: &str) { + if self.crash_dumped.swap(true, Ordering::AcqRel) { + return; + } + let name = self.role.name(); + let mut found = self.scan_crashlog("__DATA", self.data, reason); + if self.shared_ro.1 != 0 { + found += self.scan_crashlog("__DATA_SHARED_RO", self.shared_ro, reason); + } + if found == 0 { + dev_err!(self.dev.as_ref(), "G18_CRASHLOG {} no CLHE header in __DATA or __DATA_SHARED_RO\n", name); + } + } + + /// Scan one reservation every CRASHLOG_SCAN_STEP for a `CLHE` crashlog + /// and dump what it finds. Read-only mapping. Returns the count found. + fn scan_crashlog(&self, label: &str, region: (u64, u64), reason: &str) -> u32 { + let dev: &kernel::device::Device = self.dev.as_ref().as_ref(); + let name = self.role.name(); + let (base, size) = region; + let Ok(size) = usize::try_from(size) else { + return 0; + }; + dev_err!(dev, "G18_CRASHLOG {} scanning {} {:#x}+{:#x} ({})\n", name, label, base, size, reason); + let map = match PhysMap::new(base, size) { + Ok(map) => map, + Err(error) => { + dev_err!(dev, "G18_CRASHLOG {} {} map failed ({:?})\n", name, label, error); + return 0; + } + }; + let mut found = 0u32; + let mut offset = 0usize; + while offset + 16 <= size { + let mut head = [0u8; 16]; + if map.copy(offset, &mut head).is_ok() && le_u32(&head, 0) == Some(CRASHLOG_HEADER) { + let declared = le_u32(&head, 8).unwrap_or(0) as usize; + if declared < 0x20 || declared > size - offset { + dev_err!(dev, "G18_CRASHLOG {} CLHE at {}+{:#x} declares {:#x} bytes; skipped\n", + name, label, offset, declared); + } else { + let length = (declared.min(CRASHLOG_MAX_BYTES) + 3) & !3; + let length = length.min(size - offset); + match KVVec::from_elem(0u8, length, GFP_KERNEL) { + Ok(mut bytes) => { + if map.copy(offset, &mut bytes).is_ok() { + dev_err!(dev, "G18_CRASHLOG {} found at {}+{:#x} (PA {:#x})\n", + name, label, offset, base + offset as u64); + dump_crashlog(dev, name, label, &bytes); + found += 1; + } + } + Err(_) => dev_err!(dev, "G18_CRASHLOG {} no memory for {:#x} bytes\n", name, length), + } + } + } + offset += CRASHLOG_SCAN_STEP; + } + found + } +} + +/// No host-backed RTKit buffers: exactly the M5 transport configuration +/// (`G17PNoRtkitBuffer`). Host-allocated requests use the RTKit core's +/// coherent-DMA fallback; the firmware-owned crash buffer is echoed. +struct G18NoRtkitBuffer; + +impl rtkit::Buffer for G18NoRtkitBuffer { + fn iova(&self) -> Result { + Err(ENOTSUPP) + } + + fn buf(&mut self) -> Result> { + Err(ENOTSUPP) + } +} + +struct G18RtkitOps; + +#[vtable] +impl rtkit::Operations for G18RtkitOps { + type Data = Arc; + type Buffer = G18NoRtkitBuffer; + + fn recv_message_early( + data: ::Borrowed<'_>, + endpoint: u8, + message: u64, + ) -> bool { + if endpoint == EP_GFX_MESSAGES && message == INITDATA_ACK { + data.acked.store(true, Ordering::Release); + true + } else { + false + } + } + + fn recv_message( + data: ::Borrowed<'_>, + endpoint: u8, + message: u64, + ) { + let ordinal = data.messages.fetch_add(1, Ordering::Relaxed); + if ordinal < MESSAGE_LOG_BUDGET || ordinal.is_power_of_two() { + dev_info!( + data.dev.as_ref(), + "G18_MSG role={} ordinal={} ep={:#x} type={:#06x} payload={:#018x}\n", + data.role.name(), + ordinal, + endpoint, + message >> 48, + message + ); + } + } + + fn crashed( + data: ::Borrowed<'_>, + crashlog: Option<&[u8]>, + ) { + data.crashed.store(true, Ordering::Release); + dev_err!( + data.dev.as_ref(), + "G18 firmware crashed role={} host-crashlog={}\n", + data.role.name(), + crashlog.map_or(0, |bytes| bytes.len()) + ); + match crashlog { + Some(bytes) if !bytes.is_empty() => { + data.crash_dumped.store(true, Ordering::Release); + dump_crashlog(data.dev.as_ref().as_ref(), data.role.name(), "rtkit", bytes); + } + // Firmware-owned buffer: the RTKit core logged its DVA. + _ => data.dump_data_crashlog("rtkit crash callback, firmware-owned buffer"), + } + } +} + +// --------------------------------------------------------------------------- +// Session +// --------------------------------------------------------------------------- + +struct G18RoleSlot { + role: G18Role, + provider: G18Provider, + rtkit: rtkit::RtKit, + started: bool, +} + +struct G18Session { + roles: [G18RoleSlot; 2], + states: [Arc; 2], + memory: Option>, + queue_probe: Option, + render_job: Option>, + application_job: Option>, + uat: Arc, + _registers: regs::Resources, + dev: ARef, + stage: u32, +} + +fn cntvct() -> u64 { + let value: u64; + // SAFETY: read-only architectural counter. + unsafe { + core::arch::asm!("mrs {value}, cntvct_el0", value = out(reg) value, + options(nomem, nostack, preserves_flags)); + } + value +} + +impl G18Session { + fn device(&self) -> &kernel::device::Device { + self.dev.as_ref().as_ref() + } + + fn any_crashed(&self) -> bool { + self.states.iter().any(|state| state.crashed()) + } + + /// Drain the report rings once the graph exists. Returns whether a + /// recovery (type 4) event was seen. + fn drain(&mut self) -> Result { + match self.memory.as_mut() { + Some(memory) => memory.drain_reports(&self.dev), + None => Ok(false), + } + } + + /// Fault check used by every wait loop. + fn check_faults(&mut self, label: &str) -> Result { + // No host SGX MMIO here: firmware power-gates SGX at will, and a + // gated read is a synchronous external abort (stage34 oops at + // SGX+0xd8c0 during root publication). A GMMU fault reaches us as a + // firmware recovery event. Read retained DRAM for diagnosis; a + // userspace read of power-gated SGX registers is unsafe as well. + if self.drain()? { + dev_err!(self.device(), "G18 {}: firmware posted a recovery event\n", label); + return Err(EIO); + } + if self.any_crashed() { + dev_err!(self.device(), "G18 {}: firmware crashed\n", label); + return Err(EIO); + } + Ok(()) + } + + fn stop_cpus(&mut self) -> Result { + let mut failure = None; + for role in G18Role::STOP_ORDER { + let slot = &mut self.roles[role.index()]; + if slot.started { + match slot.provider.stop_cpu() { + Ok(()) => slot.started = false, + Err(error) => failure = Some(error), + } + } + log_cpu_state(self.dev.as_ref().as_ref(), role, "after-stop", &self.roles[role.index()].provider); + } + failure.map_or(Ok(()), Err) + } + + /// Dump whatever crash evidence exists after a failed stage. + fn dump_fault_evidence(&mut self, reason: &str) { + for _ in 0..4 { + if self.drain().is_err() { + break; + } + } + if let Some(memory) = self.memory.as_mut() { + match memory.control_counters() { + Ok(counters) => dev_err!(self.dev.as_ref(), "G18 fault: control counters {:?}\n", counters), + Err(error) => dev_err!(self.dev.as_ref(), "G18 fault: counters unreadable ({:?})\n", error), + } + memory.log_summary(&self.dev); + dev_err!(self.dev.as_ref(), "G18 fault: report counts {:?}\n", memory.report_counts()); + } + for state in self.states.iter() { + state.dump_data_crashlog(reason); + } + } + + fn run(&mut self) -> Result { + // Defense in depth for any future caller that bypasses driver probe. + if self.stage == STAGE_RENDER { + crate::g18::boot::render_stage::admit(self.device(), + crate::g18::params::g18_render_variant.value(), + crate::g18::params::g18_render_profile.value())?; + } + self.boot_rtkit()?; + if self.stage == STAGE_RTKIT { + self.syslog_window()?; + return Ok(()); + } + self.build_initdata()?; + self.publish_roots()?; + if self.stage == STAGE_INITDATA { + return Ok(()); + } + self.retire_openings()?; + self.wake_idle_policy()?; + if matches!(self.stage, STAGE_QUEUE_REGISTER | STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD) { + self.register_queue()?; + } + if matches!(self.stage, STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD) { + self.bind_scheduler()?; + } + if self.stage == STAGE_POOL_BIND { + let binding = self.memory.as_mut().ok_or(EIO)?.prepare_uma_pool_binding()?; + self.bind_pool(binding)?; + } + if self.stage == STAGE_COMPUTE_ADD { + let (command, pool) = self.memory.as_mut().ok_or(EIO)?.prepare_addition(&self.dev)?; + self.bind_pool(pool)?; + self.submit_addition(command, pool)?; + } + if self.stage == STAGE_RENDER { self.submit_render()?; } + if self.stage == STAGE_DRM {return Ok(());} + self.heartbeat() + } + + /// Stage 2: start each ASC CPU (GFX1, then GFX) and run RTKit + /// HELLO/EPMAP. The RTKit core starts the crashlog endpoint before + /// acknowledging the EPMAP that advertises it (firmware-owned crash + /// buffer, as G17P/M5), then syslog and the other system endpoints. + fn boot_rtkit(&mut self) -> Result { + for role in G18Role::START_ORDER { + let dev = self.dev.clone(); + let slot = &mut self.roles[role.index()]; + log_cpu_state(dev.as_ref().as_ref(), role, "before-start", &slot.provider); + // The provider re-reads the role's PSD block and refuses an + // unpowered block; it refuses without the CPU-control approval. + let (control, _) = slot.provider.cpu_state()?; + if control & ASCWRAP_V8_CPU_RUN == 0 { + slot.provider.start_cpu()?; + } else { + dev_info!(dev.as_ref(), "G18_ASC {} adopted: RUN already set, start skipped\n", role.name()); + } + slot.started = true; + log_cpu_state(dev.as_ref().as_ref(), role, "after-start", &slot.provider); + let started = Instant::::now(); + let booted = Pin::new(&mut slot.rtkit).boot(); + let elapsed = started.elapsed().as_micros_ceil(); + let mut endpoints = 0u64; + for endpoint in 0..64u8 { + if Pin::new(&mut slot.rtkit).has_endpoint(endpoint) { + endpoints |= 1 << endpoint; + } + } + dev_info!( + dev.as_ref(), + "G18_RTKIT role={} boot={:?} after {} us; EPMAP[0..64]={:#018x} (crashlog={} syslog={} ep20={} ep21={} ep23={})\n", + role.name(), + booted, + elapsed, + endpoints, + (endpoints >> 1) & 1, + (endpoints >> 2) & 1, + (endpoints >> EP_GFX_MESSAGES) & 1, + (endpoints >> EP_GFX_INTERRUPTS) & 1, + (endpoints >> 0x23) & 1 + ); + booted?; + if self.states[role.index()].crashed() { + return Err(EIO); + } + } + Ok(()) + } + + /// Stage 2 only: give the syslog endpoint time to print its banner + /// (expected 'G18 Firmware initializing', 'FW Build version: b0'; the + /// RTKit core logs each entry as "RTKit: syslog message"). + fn syslog_window(&mut self) -> Result { + dev_info!(self.device(), "G18_STAGE2 syslog window open for {} ms\n", SYSLOG_WINDOW_MS); + for _ in 0..SYSLOG_WINDOW_MS / 10 { + if self.any_crashed() { + return Err(EIO); + } + fsleep(Delta::from_millis(10)); + } + for role in G18Role::ALL { + log_cpu_state(self.dev.as_ref().as_ref(), role, "after-syslog-window", &self.roles[role.index()].provider); + dev_info!( + self.device(), + "G18_STAGE2 role={} non-system messages={}\n", + role.name(), + self.states[role.index()].messages.load(Ordering::Relaxed) + ); + } + dev_info!(self.device(), "G18_STAGE2 syslog window closed\n"); + Ok(()) + } + + /// Stage 3: map the init-data graph under the firmware-published roots + /// (same order as M5; no SGX pre-init pokes are issued, none are derived + /// for G18G yet). + fn build_initdata(&mut self) -> Result { + let roots = self.uat.prepare_t8140_firmware_high_roots()?; + let memory = G18FirmwareMemory::new_with_uat(&self.dev, self.uat.clone(), cntvct(), matches!(self.stage, STAGE_RENDER | STAGE_DRM))?; + self.memory = Some(memory); + if matches!(self.stage, STAGE_RENDER | STAGE_DRM) { + self.render_job = Some(crate::g18::boot::render_stage::prepare(&self.dev, &self.uat)?); + self.memory.as_mut().ok_or(EIO)?.prepare_render(&self.dev, + self.render_job.as_mut().ok_or(EIO)?)?; + } + if matches!(self.stage, STAGE_QUEUE_REGISTER | STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD) + || (self.stage == STAGE_DRM && crate::g18::params::g18_admit_compute.value() == 1) { + self.queue_probe = Some(G18QueueProbe::new(&self.dev, &self.uat, + matches!(self.stage, STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD | STAGE_DRM))?); + } + self.uat.confirm_t8140_primary_high_root(roots)?; + self.uat.mirror_t8140_secondary_high_root()?; + let memory = self.memory.as_mut().ok_or(EIO)?; + if matches!(self.stage, STAGE_COMPUTE_ADD | STAGE_RENDER | STAGE_DRM) { + memory.use_firmware_queue_programming(&self.dev)?; + } + memory.log_summary(&self.dev); + dev_info!(self.dev.as_ref(), "G18_INIT status links before publish {:x?}; counters {:?}\n", + memory.status_links()?, memory.control_counters()?); + Ok(()) + } + + /// Stage 3: publish GFX's root, then GFX1's, each on EP 0x20 with the + /// 0x81 doorbell, and wait for that role's 0x0009 ACK. Native 26A428 + /// captures 051/053/073 all publish in this order. The ASC CPU/RTKit + /// startup order is separate and remains GFX1 first. + fn publish_roots(&mut self) -> Result { + for role in [G18Role::Gfx, G18Role::Gfx1] { + let index = role.index(); + let address = self.memory.as_ref().ok_or(EIO)?.root(role == G18Role::Gfx1); + let message = g17_rtkit::encode_initdata_doorbell(address).map_err(|_| EINVAL)?; + { + let slot = &mut self.roles[index]; + for endpoint in [EP_GFX_MESSAGES, EP_GFX_INTERRUPTS] { + if !Pin::new(&mut slot.rtkit).has_endpoint(endpoint) { + dev_err!(self.dev.as_ref(), "G18_INIT role={} did not advertise EP {:#x}\n", + role.name(), endpoint); + return Err(ENODEV); + } + } + Pin::new(&mut slot.rtkit).start_endpoint(EP_GFX_MESSAGES)?; + Pin::new(&mut slot.rtkit).start_endpoint(EP_GFX_INTERRUPTS)?; + Pin::new(&mut slot.rtkit).send_message(EP_GFX_MESSAGES, message)?; + } + dev_info!(self.device(), "G18_INIT role={} published root={:#x} doorbell={:#018x}\n", + role.name(), address, message); + let started = Instant::::now(); + let mut acked = false; + for _ in 0..ACK_POLLS { + self.check_faults("initdata ACK wait")?; + if self.states[index].acked.load(Ordering::Acquire) { + acked = true; + break; + } + fsleep(Delta::from_millis(10)); + } + if !acked { + dev_err!(self.device(), "G18_INIT role={} no 0x0009 ACK within {} ms\n", + role.name(), ACK_POLLS * 10); + return Err(ETIMEDOUT); + } + dev_info!(self.device(), "G18_INIT role={} initdata-ack after {} us\n", + role.name(), started.elapsed().as_micros_ceil()); + } + let memory = self.memory.as_mut().ok_or(EIO)?; + dev_info!(self.dev.as_ref(), "G18_INIT both roles ACKed; status links {:x?}; counters {:?}; completion cursors {:?}\n", + memory.status_links()?, memory.control_counters()?, memory.completion_cursors()?); + self.check_faults("after both ACKs") + } + + /// Stage 4: 0x89 power-transition-end to GFX, then GFX1, and wait for + /// both opening records (0x15 / 0x2a) to retire. + fn retire_openings(&mut self) -> Result { + let before = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + dev_info!(self.device(), "G18_OPEN before power-transition-end {:?}\n", before); + for role in G18Role::STOP_ORDER { + Pin::new(&mut self.roles[role.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, g17_rtkit::encode_power_transition_end())?; + } + let mut counters = before; + for _ in 0..CONTROL_POLLS { + self.check_faults("opening retire")?; + counters = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + if counters == [[1; 3]; 2] { + dev_info!(self.device(), "G18_OPEN both control openings retired {:?}\n", counters); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.device(), "G18_OPEN openings did not retire {:?}\n", counters); + Err(ETIMEDOUT) + } + + /// Stage 4: publish IdlePowerOff(0) (opcode 0x0a) and ring EP 0x21 with + /// the idle-policy wake, then wait for GFX's control ring to retire it. + fn wake_idle_policy(&mut self) -> Result { + self.memory.as_mut().ok_or(EIO)?.publish_idle_wake()?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, IDLE_POLICY_WAKE)?; + let mut counters = [[0; 3]; 2]; + for _ in 0..CONTROL_POLLS { + self.check_faults("idle-policy wake")?; + counters = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + if counters == [[2; 3], [1; 3]] { + dev_info!(self.device(), "G18_POWER idle policy retired {:?}\n", counters); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.device(), "G18_POWER wake did not retire {:?}\n", counters); + Err(ETIMEDOUT) + } + + fn queue_probe_status(&mut self) -> Result<([u32; 3], [u32; 3], [u64; 5])> { + let outer = self.memory.as_mut().ok_or(EIO)?.compute_outer_counters()?; + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + Ok((outer, probe.counters()?, probe.slot())) + } + + /// Register one empty queue. No scheduler binding, event or tag14 kick. + fn register_queue(&mut self) -> Result { + let before = self.queue_probe_status()?; + dev_info!(self.device(), "G18_QUEUE before tag15-only {:?}\n", before); + if before != ([0; 3], [0, 0, 1], [0; 5]) { return Err(EBUSY); } + let address = self.queue_probe.as_ref().ok_or(EIO)?.queue_address(); + self.memory.as_mut().ok_or(EIO)?.publish_queue_probe(address, QUEUE_ID, 1)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + for _ in 0..CONTROL_POLLS { + self.check_faults("queue registration")?; + let (outer, inner, slot) = self.queue_probe_status()?; + if outer == [1; 3] && inner == [1; 3] && G18QueueProbe::installed(slot) { + dev_info!(self.device(), + "G18_QUEUE installed qid11 outer={:?} inner={:?} slot={:x?}; no kicks\n", + outer, inner, slot); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + let status = self.queue_probe_status()?; + dev_err!(self.device(), "G18_QUEUE registration timeout {:?}\n", status); + Err(ETIMEDOUT) + } + + fn bind_scheduler(&mut self) -> Result { + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + let before = probe.scheduler_status()?; + dev_info!(self.dev.as_ref(), "G18_SCHED before bind {:x?}\n", before); + probe.prepare_scheduler_bind()?; + let address = probe.queue_address(); + self.memory.as_mut().ok_or(EIO)?.publish_queue_probe(address, QUEUE_ID, 2)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + for _ in 0..CONTROL_POLLS { + self.check_faults("scheduler bind")?; + let (outer, inner, slot) = self.queue_probe_status()?; + let status = self.queue_probe.as_mut().ok_or(EIO)?.scheduler_status()?; + if outer == [2; 3] && inner == [2; 3] && G18QueueProbe::installed(slot) + && G18QueueProbe::scheduler_bound(status) { + dev_info!(self.device(), + "G18_SCHED bound qid11 outer={:?} inner={:?} status={:x?}; no kicks\n", + outer, inner, status); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + let queues = self.queue_probe_status()?; + let status = self.queue_probe.as_mut().ok_or(EIO)?.scheduler_status()?; + dev_err!(self.device(), "G18_SCHED bind timeout queues={:?} status={:x?}\n", queues, status); + Err(ETIMEDOUT) + } + + fn bind_pool(&mut self, binding: G18UmaPoolBinding) -> Result { + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + let before = probe.pool_status(); + dev_info!(self.dev.as_ref(), "G18_POOL before bind {:x?}; state={:#x} generation={} slot={}\n", + before, binding.state, binding.generation, binding.slot); + probe.prepare_pool_bind(binding)?; + let address = probe.queue_address(); + self.memory.as_mut().ok_or(EIO)?.publish_queue_probe(address, QUEUE_ID, 3)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + for _ in 0..CONTROL_POLLS { + self.check_faults("UMA pool bind")?; + let (outer, inner, slot) = self.queue_probe_status()?; + let status = self.queue_probe.as_ref().ok_or(EIO)?.pool_status(); + if outer == [3; 3] && inner == [3; 3] && G18QueueProbe::installed(slot) + && G18QueueProbe::pool_bound(status, binding) { + dev_info!(self.device(), + "G18_POOL bound qid11 outer={:?} inner={:?} status={:x?}; no kicks or growth\n", + outer, inner, status); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + let queues = self.queue_probe_status()?; + let status = self.queue_probe.as_ref().ok_or(EIO)?.pool_status(); + dev_err!(self.device(), "G18_POOL bind timeout queues={:?} status={:x?}\n", queues, status); + Err(ETIMEDOUT) + } + + /// Submit exactly one addition; a retired setup or changed stamp alone is + /// insufficient. Preserve the whole firmware owner after every outcome. + fn submit_addition(&mut self, command: G18PreparedComputeCommand, + pool: G18UmaPoolBinding) -> Result { + let memory = self.memory.as_mut().ok_or(EIO)?; + let output = memory.addition_output()?; + let completion = memory.compute().completion_snapshot()?; + let scheduler = memory.scheduler().snapshot(0)?; + if output.checked != 4096 || output.untouched != 4096 || output.correct != 0 + || completion.stamps != [0; 2] || memory.completion_masks() != [0; 2] + || completion.timestamps != [0; 2] || scheduler.submitted != 0 + || scheduler.completed != 0 || scheduler.linked != 0 { + return Err(EBUSY); + } + let roots = memory.compute().publish_context()?; + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + probe.prepare_compute_job(command, pool)?; + let address = probe.queue_address(); + // From this counter onward, even a failed publication retains all + // memory. Firmware may already observe the host-owned counter. + let submitted = self.memory.as_mut().ok_or(EIO)?.scheduler().publish_kick(0)?; + self.queue_probe.as_mut().ok_or(EIO)?.publish_compute_job()?; + self.memory.as_mut().ok_or(EIO)?.publish_compute_queue(address, QUEUE_ID)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + dev_info!(self.device(), + "G18_COMPUTE submitted qid11 context={} roots={:x?} count={} expected_stamp={:#x}\n", + command.context_id, roots, submitted, command.expected_stamp); + // Bounded first-use run. Polling observes completion; it sends no + // additional commands and never retries a GPU job in this boot. + for step in 0..10_000 { + self.check_faults("compute addition")?; + let (outer, inner, _) = self.queue_probe_status()?; + let memory = self.memory.as_mut().ok_or(EIO)?; + let completion = memory.compute().completion_snapshot()?; + let scheduler = memory.scheduler().snapshot(0)?; + let masks = memory.completion_masks(); + if outer == [4; 3] && inner == [6; 3] + // Original b000 19234/19268 writes through the first qword. + // The second qword is carried into trace 0xe as an identity, + // not written by this completion path (stage29 hardware). + && completion.stamps[0] == command.expected_stamp + && masks[0] & (1u64 << QUEUE_ID) != 0 + && completion.timestamps[0] != 0 + && completion.timestamps[1] >= completion.timestamps[0] + && scheduler.submitted == 1 && scheduler.completed == 1 && scheduler.linked == 0 { + let output = memory.addition_output()?; + dev_info!(self.dev.as_ref(), + "G18_COMPUTE completion outer={:?} inner={:?} stamps={:x?} times={:x?} event_masks={:x?} scheduler={:?} output={:?}\n", + outer, inner, completion.stamps, completion.timestamps, masks, scheduler, output); + if crate::g18::params::g18_compute_variant.value() == 2 { + let pixels = memory.triangle_output()?; + dev_info!(self.dev.as_ref(), "G18_COMPUTE_TRIANGLE output={:?}\n", pixels); + if !pixels.passed() { return Err(EIO); } + self.check_faults("compute triangle output")?; + dev_info!(self.device(), "G18_COMPUTE_TRIANGLE_PASS: 4096 GPU pixels match after independent completion; compute rasterization, no TA/3D or Mesa\n"); + return Ok(()); + } + // g18_compute_variant=1 dispatches nothing: every output must stay poisoned. + let empty = crate::g18::params::g18_compute_variant.value() == 1; + let expected = if empty { (0, 4096) } else { (4096, 0) }; + if output.checked != 4096 || (output.correct, output.untouched) != expected { + if empty { + dev_err!(self.device(), "G18_COMPUTE_EMPTY_STREAM_FAIL: completed without a dispatch, but output changed {:?}\n", output); + } + return Err(EIO); + } + self.check_faults("compute output")?; + if empty { + dev_info!(self.device(), "G18_COMPUTE_EMPTY_STREAM_PASS: completion observed with no dispatch; all 4096 outputs still poisoned\n"); + } else { + dev_info!(self.device(), "G18_COMPUTE_ADD_PASS: all 4096 outputs match after independent completion\n"); + } + return Ok(()); + } + if step % 1000 == 999 { + dev_info!(self.dev.as_ref(), + "G18_COMPUTE wait={}ms outer={:?} inner={:?} stamps={:x?} scheduler={:?}\n", + step + 1, outer, inner, completion.stamps, scheduler); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.device(), "G18_COMPUTE timeout; owners retained, no resubmission\n"); + Err(ETIMEDOUT) + } + + /// A single bounded H1 pass; all owners already live in the retained session. + fn submit_render(&mut self) -> Result { + let pass = self.render_job.as_ref().ok_or(EIO)?.pass()?; + if self.memory.as_ref().ok_or(EIO)?.completion_masks()!=[0;2] {return Err(EBUSY);} + for phase in 1..=3 { + self.check_faults("render binding")?; + let queues = self.render_job.as_mut().ok_or(EIO)?.queues(); + let records = match phase { + 1 => queues.install_records(&pass)?, + 2 => queues.scheduler_records(&pass)?, + _ => queues.pool_records(&pass)?, + }; + self.memory.as_mut().ok_or(EIO)?.publish_outer_records(&records)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + let mut passed = false; + for _ in 0..CONTROL_POLLS { + self.check_faults("render binding")?; + let outer=self.memory.as_mut().ok_or(EIO)?.compute_outer_counters()?; + if outer==[phase*2;3] && self.render_job.as_mut().ok_or(EIO)?.queues().stage_gate(phase,&pass)? { + dev_info!(self.device(), "G18_RENDER_BIND_PASS phase={} outer={:?} stages=0,1\n",phase,outer); + passed=true;break; + } + fsleep(Delta::from_millis(1)); + } + if !passed { + let outer=self.memory.as_mut().ok_or(EIO)?.compute_outer_counters()?; + let job=self.render_job.as_mut().ok_or(EIO)?; + dev_err!(self.dev.as_ref(), "G18_RENDER_BIND_FAIL phase={} outer={:?} inner={:?} firmware={:x?} pool_pb={:x?}\n", + phase,outer,job.queues().counters()?,job.queues().firmware_status()?,job.queues().pool_and_pb_status(0,0)?); + return Err(ETIMEDOUT); + } + } + let job=self.render_job.as_mut().ok_or(EIO)?; + if job.parameter_buffer().count_submitted_render()?!=1 {return Err(EIO);} + self.memory.as_mut().ok_or(EIO)?.audit_render(&self.dev,job)?; + job.queues().prepare_pass(&pass)?; + job.queues().audit_prepared_pass(&self.dev,&pass)?; + self.check_faults("render pre-kick")?; + let job=self.render_job.as_mut().ok_or(EIO)?; + let roots=job.publish_h1()?; + let records=job.queues().publish_pass()?; + self.memory.as_mut().ok_or(EIO)?.publish_outer_records(&records)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + dev_info!(self.device(), "G18_RENDER_KICK context={} roots={:x?} order=3D,TA submitted=2 barrier=1 PB-generation=1\n",pass.context_id,roots); + for step in 0..10_000 { + self.check_faults("render completion")?; + let memory=self.memory.as_mut().ok_or(EIO)?; + let outer=memory.compute_outer_counters()?;let masks=memory.completion_masks(); + let job=self.render_job.as_mut().ok_or(EIO)?; + let inner=job.queues().counters()?;let stamps=job.stamps()?; + let scheduler=job.scheduler_snapshot()?;let list=job.queues().completion_list()?; + if outer==[8;3] && inner==[[6;3];2] && stamps==[0x100;2] + && scheduler.submitted==2 && scheduler.completed==2 && scheduler.linked==0 + && list==[0,job.queues().job_list()] && masks[0]&0x600==0x600 { + let pixels=job.pixels()?; + dev_info!(self.dev.as_ref(), "G18_RENDER_COMPLETION outer={:?} inner={:?} stamps={:x?} masks={:x?} scheduler={:?} list={:x?}\n",outer,inner,stamps,masks,scheduler,list); + dev_info!(self.dev.as_ref(), "G18_RENDER_PIXELS sha256={:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x} background={} poison={}\n", + pixels.0[0],pixels.0[1],pixels.0[2],pixels.0[3],pixels.0[4],pixels.0[5],pixels.0[6],pixels.0[7],pixels.1,pixels.2); + self.check_faults("render pixels")?; + dev_info!(self.device(), "G18_RENDER_H1_PASS variant={} completion=independent pixels=exact owners=retained\n",crate::g18::params::g18_render_variant.value()); + return Ok(()); + } + if step%1000==999 { + dev_info!(self.dev.as_ref(), "G18_RENDER_WAIT ms={} outer={:?} inner={:?} stamps={:x?} masks={:x?} scheduler={:?} list={:x?}\n",step+1,outer,inner,stamps,masks,scheduler,list); + } + fsleep(Delta::from_millis(1)); + } + Err(ETIMEDOUT) + } + + /// Stage 4..9: passive drain of both roles' event and trace rings. Sends + /// nothing. This is the milestone-1 heartbeat. + fn heartbeat(&mut self) -> Result { + dev_info!(self.device(), "G18_HEARTBEAT passive drain for {} ms, no host messages\n", HEARTBEAT_MS); + let steps = HEARTBEAT_MS / HEARTBEAT_STEP_MS; + for step in 0..steps { + self.check_faults("heartbeat")?; + if step % 100 == 99 { + let memory = self.memory.as_mut().ok_or(EIO)?; + dev_info!(self.dev.as_ref(), + "G18_HEARTBEAT t={} ms reports [role][event,trace]={:?} counters={:?} cursors={:?}\n", + (step + 1) * HEARTBEAT_STEP_MS, memory.report_counts(), + memory.control_counters()?, memory.completion_cursors()?); + } + fsleep(Delta::from_millis(HEARTBEAT_STEP_MS as i64)); + } + let counts = self.memory.as_ref().ok_or(EIO)?.report_counts(); + let alive = counts.iter().all(|role| role[1] != 0); + dev_info!(self.device(), "G18_HEARTBEAT done: reports={:?}; trace advanced on both roles={}; no recovery record\n", + counts, alive); + Ok(()) + } +} + +/// Holds a session that may have firmware running. Dropping it stops both +/// CPUs and then LEAKS every owner: the halt is not qualified, and the +/// firmware may still reference the UAT, the graph and the transports. +struct Retained(Option>); + +impl Retained { + fn stop(&mut self, label: &str) -> Result { + let Some(session) = self.0.as_mut() else { + return Ok(()); + }; + let stopped = session.stop_cpus(); + dev_info!(session.device(), "G18 {}: stop requests={:?}; owners retained until reboot\n", + label, stopped); + stopped + } +} + +impl Drop for Retained { + fn drop(&mut self) { + let _ = self.stop("session release"); + if let Some(session) = self.0.take() { + core::mem::forget(session); + } + } +} + +/// Stage-4 owner kept by the driver (`AsahiRuntime::G18`). +#[pin_data] +pub(crate) struct G18Owner { + #[pin] + session: Mutex, +} + +impl G18Runtime for G18Owner { + fn shutdown(&self) { + let _ = self.session.lock().stop("unbind"); + } +} + +// --------------------------------------------------------------------------- +// Stage 1: read-only probe +// --------------------------------------------------------------------------- + +fn stage_read_only(pdev: &platform::Device>, probe: &T8152Probe) { + let dev: &kernel::device::Device> = pdev.as_ref(); + for sample in probe.psd.iter() { + log_psd(pdev, "stage1 (probe sample)", sample); + } + match &probe.observed { + Some(id) => dev_info!(dev, "G18_STAGE1 SGX identity (sgx PSD proven powered): {:?}\n", id), + None => dev_info!(dev, "G18_STAGE1 SGX identity not read: sgx PSD not powered\n"), + } + dev_info!(dev, "G18_STAGE1 firmware receipt {}\n", + if probe.firmware_profile_admitted { "admitted" } else { "absent" }); + + for role in G18Role::ALL { + let provider = match G18Provider::get(dev, role) { + Ok(provider) => provider, + Err(error) => { + // G18Provider::get already explained an unbound provider. + dev_info!(dev, "G18_STAGE1 {} mailbox provider unavailable ({:?}); wrapper and post-office reads skipped\n", + role.name(), error); + continue; + } + }; + let lifecycle = match provider.validate(dev) { + Ok(lifecycle) => lifecycle, + Err(error) => { + dev_err!(dev, "G18_STAGE1 {} provider rejected ({:?}); no reads\n", role.name(), error); + continue; + } + }; + dev_info!(dev, "G18_STAGE1 {} provider wrapper={:x?} iop-vbar={:x?} post-office={:#x} missing={:#x} (bit0: CPU control not approved)\n", + role.name(), lifecycle.wrapper, lifecycle.iop_vbar, role.postoffice_window(), lifecycle.missing); + log_cpu_state(dev, role, "stage1", &provider); + match provider.postoffice_status(pdev) { + Ok((psd, Some(status))) => { + log_psd(pdev, "stage1 post-office gate", &psd); + dev_info!(dev, "G18_STAGE1 {} post-office status={:#x} (rx-occupancy={} tx-free={})\n", + role.name(), status, status & 0xf, (status >> 8) & 0xf); + } + Ok((psd, None)) => { + log_psd(pdev, "stage1 post-office gate", &psd); + dev_info!(dev, "G18_STAGE1 {} post-office status not read: role PSD not powered\n", role.name()); + } + Err(error) => dev_err!(dev, "G18_STAGE1 {} post-office read failed ({:?})\n", role.name(), error), + } + } + dev_info!(dev, "G18_STAGE1 done; nothing sent, no write issued; ENODEV\n"); +} + +// --------------------------------------------------------------------------- +// Stages 90..93: PSD requests (each REQUIRES USER APPROVAL) +// --------------------------------------------------------------------------- + +/// Native PSD::getPsdAck order: poll ACK, then read STATE once. STATE +/// clears ACK on J873; reading it while waiting loses the acknowledgement. +fn wait_psd_state(pdev: &platform::Device>, block: &PsdBlock, target: u32) -> Result { + let started = Instant::::now(); + for poll in 0..PSD_POLLS { + if let Some(sample) = block.acknowledged_sample()? { + dev_info!(pdev.as_ref(), "G18_PSD service={} poll={} t={} us state={:#x} ack={:#x} (ACK read before STATE)\n", + sample.service.service_id(), poll, started.elapsed().as_micros_ceil(), + sample.state, sample.ack); + if sample.power_state() != target { + dev_err!(pdev.as_ref(), "G18_PSD service={} acknowledged unexpected state {}, wanted {}\n", + sample.service.service_id(), sample.power_state(), target); + return Err(EIO); + } + dev_info!(pdev.as_ref(), "G18_PSD service={} reached state {} after {} us; ack={:#x}\n", + sample.service.service_id(), target, started.elapsed().as_micros_ceil(), sample.ack); + return Ok(sample); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(pdev.as_ref(), "G18_PSD service={} did not acknowledge state {} within {} ms\n", + block.service.service_id(), target, PSD_POLLS); + Err(ETIMEDOUT) +} + +/// Claim one PSD request kind for this module instance. +fn claim_psd_request(dev: &kernel::device::Device, bit: u32) -> Result { + if G18_FIRMWARE_ATTEMPTED.load(Ordering::Acquire) { + dev_err!(dev, "G18_PSD refused: a firmware stage already ran this boot\n"); + return Err(EBUSY); + } + if G18_PSD_REQUESTS.fetch_or(bit, Ordering::AcqRel) & bit != 0 { + dev_err!(dev, "G18_PSD refused: this request was already written by this module instance\n"); + return Err(EBUSY); + } + Ok(()) +} + +fn map_psd_blocks(pdev: &platform::Device>) -> Result> { + let mut blocks = KVec::new(); + for service in regs::T8152PsdService::ALL { + blocks.push(PsdBlock::map(pdev, service)?, GFP_KERNEL)?; + } + Ok(blocks) +} + +/// REQUIRES USER APPROVAL. Stages 90 (sgx), 91 (gfx-asc), 92 (gfx1-asc): +/// request the native active state (SGX 0, ASCs 2) on exactly one service. +/// +/// Refuses unless that service reads cold idle (state 6); gfx-asc and +/// gfx1-asc additionally need sgx to read state 0 first. Native order is +/// 3, 82, 81. Polls ACK before STATE. On a timeout nothing is restored: the service is +/// left as requested and the power state is reported UNKNOWN, to be +/// restored by stage 93 under its own approval. The SGX ID is read only if +/// sgx then proves powered. +fn stage_psd_power_on(pdev: &platform::Device>, probe: &T8152Probe, service: regs::T8152PsdService) -> Result { + let dev: &kernel::device::Device> = pdev.as_ref(); + dev_warn!(dev, "G18_PSD stage {}: PMGR2 PSD power-on of service {} only (REQUIRES USER APPROVAL)\n", + probe.stage, service.service_id()); + let blocks = map_psd_blocks(pdev)?; + let mut before = [regs::T8152PsdSample { service: regs::T8152PsdService::Sgx, state: 0, ack: 0 }; 3]; + for (slot, block) in before.iter_mut().zip(blocks.iter()) { + *slot = block.sample()?; + log_psd(pdev, "power-on before", slot); + } + let index = regs::T8152PsdService::ALL.iter().position(|s| *s == service).ok_or(EINVAL)?; + let target = &before[index]; + let active_state = service.active_state(); + if target.powered() { + dev_info!(dev, "G18_PSD service {} already reads state {} ({}); no request written\n", + service.service_id(), active_state, target.verdict()); + return Ok(()); + } + if target.power_state() != PSD_STATE_IDLE { + dev_err!(dev, "G18_PSD refused: service {} must read cold idle state {} first\n", + service.service_id(), PSD_STATE_IDLE); + return Err(EBUSY); + } + if service != regs::T8152PsdService::Sgx && !before[0].powered() { + dev_err!(dev, "G18_PSD refused: sgx (service 3) must read state 0 before service {} (run stage 90 first)\n", + service.service_id()); + return Err(EBUSY); + } + claim_psd_request(dev, psd_request_bit(service))?; + let block = &blocks[index]; + dev_warn!(dev, "G18_PSD write service={} request={} (REQUIRES USER APPROVAL)\n", + service.service_id(), active_state); + let result = block.request(active_state).and_then(|()| wait_psd_state(pdev, block, active_state)); + for block in blocks.iter() { + if let Ok(sample) = block.sample() { + log_psd(pdev, "power-on after", &sample); + } + } + if let Err(error) = result { + dev_err!(dev, "G18_PSD service {} power-on failed ({:?}); left as requested, GPU power state UNKNOWN; no automatic restore (stage 93 restores, REQUIRES USER APPROVAL)\n", + service.service_id(), error); + return Err(error); + } + if service == regs::T8152PsdService::Sgx { + match regs::t8152_prove_sgx_powered(pdev) { + Ok(powered) => { + let id = probe.registers.get_gpu_id_t8152(&powered, probe.soc)?; + dev_info!(dev, "G18_PSD sgx powered after acknowledged transition; SGX identity {:?}\n", id); + } + Err(error) => dev_err!(dev, "G18_PSD sgx reached state 0 but is not provably powered ({:?}); SGX not read\n", error), + } + } + dev_info!(dev, "G18_PSD stage {} done; service {} left in state {}, no firmware touched; ENODEV\n", + probe.stage, service.service_id(), active_state); + Ok(()) +} + +/// REQUIRES USER APPROVAL. Stage 93: request idle (6) on every GPU service +/// that reads its active state, in reverse native order (81, 82, 3), +/// polling ACK before STATE. Refuses if a service is neither active nor +/// idle, or if a firmware stage ran this boot. Stops at the first +/// failure and reports the power state UNKNOWN. +fn stage_psd_restore_idle(pdev: &platform::Device>) -> Result { + let dev: &kernel::device::Device> = pdev.as_ref(); + dev_warn!(dev, "G18_PSD stage 93: PMGR2 PSD idle restore (REQUIRES USER APPROVAL)\n"); + let blocks = map_psd_blocks(pdev)?; + let mut samples = [regs::T8152PsdSample { service: regs::T8152PsdService::Sgx, state: 0, ack: 0 }; 3]; + for (slot, block) in samples.iter_mut().zip(blocks.iter()) { + *slot = block.sample()?; + log_psd(pdev, "restore before", slot); + } + if samples.iter().any(|sample| !sample.powered() && sample.power_state() != PSD_STATE_IDLE) { + dev_err!(dev, "G18_PSD restore refused: a service is neither active nor idle\n"); + return Err(EBUSY); + } + if samples.iter().all(|sample| sample.power_state() == PSD_STATE_IDLE) { + dev_info!(dev, "G18_PSD every GPU service already reads idle; no request written\n"); + return Ok(()); + } + claim_psd_request(dev, PSD_REQUEST_RESTORE_BIT)?; + let mut result = Ok(()); + for index in [1, 2, 0] { + let block = &blocks[index]; + let sample = &samples[index]; + if sample.power_state() == PSD_STATE_IDLE { + continue; + } + dev_warn!(dev, "G18_PSD write service={} request={} (REQUIRES USER APPROVAL)\n", + block.service.service_id(), PSD_STATE_IDLE); + if let Err(error) = block.request(PSD_STATE_IDLE).and_then(|()| wait_psd_state(pdev, block, PSD_STATE_IDLE)) { + dev_err!(dev, "G18_PSD restore of service {} failed ({:?}); GPU power state is UNKNOWN; stopping\n", + block.service.service_id(), error); + result = Err(error); + break; + } + } + for block in blocks.iter() { + if let Ok(sample) = block.sample() { + log_psd(pdev, "restore after", &sample); + } + } + result +} + +// --------------------------------------------------------------------------- +// Stages 2..6 +// --------------------------------------------------------------------------- + +fn prepare_firmware_session(pdev: &platform::Device>, probe: T8152Probe) -> Result { + let dev: &kernel::device::Device> = pdev.as_ref(); + let stage = probe.stage; + dev_warn!(dev, "G18 stage {}: starts the GPU ASC CPUs (REQUIRES USER APPROVAL)\n", stage); + if !probe.firmware_profile_admitted { + return Err(ENODEV); + } + if G18_FIRMWARE_ATTEMPTED.load(Ordering::Acquire) { + dev_err!(dev, "G18 refused: the firmware was already started this boot; reboot first\n"); + return Err(EBUSY); + } + + // The GPU must already be powered (by iBoot, or by approved stages + // 90..92 on this boot). This path never requests PSD power. STATE reads + // consume transition ACK, so this steady-state gate checks the native + // active states (SGX 0, ASCs 2), not an already-consumed acknowledgement. + let psd = read_all_psd(pdev)?; + for sample in psd.iter() { + log_psd(pdev, "firmware-stage gate", sample); + } + if !psd.iter().all(|sample| sample.powered()) { + dev_err!(dev, "G18 stage {} refused: the GPU PSD services are not all powered; nothing started\n", stage); + return Err(ENODEV); + } + // Fresh SGX proof plus identity read. + let powered = regs::t8152_prove_sgx_powered(pdev)?; + let id = probe.registers.get_gpu_id_t8152(&powered, probe.soc)?; + dev_info!(dev, "G18 SGX identity {:?}\n", id); + + // J873 hardware, 2026-09-25: with SGX+0xe90000 left at 0x30000, the GFX1 + // firmware accepts the init data but takes an async bus error on + // SGX+0xbc08 right after the 0x89 power-transition-end. The register + // reads 0x30000 here too, the state M5 native sets to 0x30001 before its + // firmware runs; apply the same checked bit-0 step before any init data. + if stage >= STAGE_INITDATA { + probe.registers.m5_match_pmgr_boot_control()?; + } + + // Providers: exact v8 resources, CPU control approved, CPUs stopped. + let mut providers = KVec::new(); + for role in G18Role::ALL { + let provider = G18Provider::get(dev, role)?; + let lifecycle = provider.validate(dev)?; + if lifecycle.missing != 0 || provider.require_safe_cpu_lifecycle().is_err() { + dev_err!(dev, "G18 {} provider lifecycle missing={:#x}: boot with apple_mailbox.ascwrap_v8_cpu_control=1 (REQUIRES USER APPROVAL)\n", + role.name(), lifecycle.missing); + return Err(EPERM); + } + let (control, status) = provider.cpu_state()?; + dev_info!(dev, "G18_ASC {} before start CPU_CONTROL={:#x} CPU_STATUS={:#x}\n", + role.name(), control, status); + // J873 hardware, 2026-09-25: after the PSD power-on both ASCs read + // RUN=1 with one message waiting in the post-office RX FIFO (the + // firmware booted on power-up and posted its HELLO). Adopt them: + // boot_rtkit skips the CPU start and the RTKit core reads that HELLO. + if control & ASCWRAP_V8_CPU_RUN != 0 { + dev_info!(dev, "G18_ASC {} already running (RUN set); adopting the live CPU\n", + role.name()); + } + providers.push(provider, GFP_KERNEL)?; + } + + // Firmware __DATA reservations (crashlog scan source), as new_t8152 + // checks them. + let node = dev.of_node().ok_or(EINVAL)?; + let mut data = [(0u64, 0u64); 2]; + for role in G18Role::ALL { + let region = node.reserved_mem_region_to_resource_byname(role.data_region_name())?; + if region.size() != role.data_size() || region.start() & (crate::pgtable::UAT_PGMSK as u64) != 0 { + return Err(EINVAL); + } + data[role.index()] = (region.start(), region.size()); + } + let mut shared_ro = [(0u64, 0u64); 2]; + for role in G18Role::ALL { + if let Ok(region) = node.reserved_mem_region_to_resource_byname(role.shared_ro_region_name()) { + shared_ro[role.index()] = (region.start(), region.size()); + } + } + + let drm = probe.drm.clone(); + let uat = Arc::new(mmu::Uat::new_t8152(&drm, None)?, GFP_KERNEL)?; + let mut states = KVec::new(); + let mut slots = KVec::new(); + let mut providers = providers.into_iter(); + for role in G18Role::ALL { + let state = Arc::new(G18RoleState { + dev: drm.clone(), + role, + data: data[role.index()], + shared_ro: shared_ro[role.index()], + acked: AtomicBool::new(false), + crashed: AtomicBool::new(false), + crash_dumped: AtomicBool::new(false), + messages: AtomicU32::new(0), + }, GFP_KERNEL)?; + let rtkit = rtkit::RtKit::::new_g17p(drm.as_ref().as_ref(), None, role.index(), state.clone()) + .map_err(|error| no_probe_defer(dev, role.name(), "RTKit mailbox", error))?; + let provider = providers.next().ok_or(EINVAL)?; + slots.push(G18RoleSlot { role, provider, rtkit, started: false }, GFP_KERNEL)?; + states.push(state, GFP_KERNEL)?; + } + let mut slots = slots.into_iter(); + let mut state_iter = states.into_iter(); + let roles = [slots.next().ok_or(EINVAL)?, slots.next().ok_or(EINVAL)?]; + let states = [state_iter.next().ok_or(EINVAL)?, state_iter.next().ok_or(EINVAL)?]; + if roles[0].role != G18Role::Gfx || roles[1].role != G18Role::Gfx1 { + return Err(EINVAL); + } + let session = KBox::new(G18Session { + roles, + states, + memory: None, + queue_probe: None, + render_job: None, + application_job: None, + uat, + _registers: probe.registers, + dev: drm, + stage, + }, GFP_KERNEL)?; + + // Point of no return: from here every path retains the session, and the + // module stays loaded so RTKit callbacks keep valid code until reboot. + G18_FIRMWARE_ATTEMPTED.store(true, Ordering::Release); + // SAFETY: THIS_MODULE is this module; the reference is never dropped. + unsafe { + bindings::__module_get( + ::THIS_MODULE.as_ptr(), + ) + }; + Ok(Retained(Some(session))) +} + +fn stage_firmware(pdev: &platform::Device>, probe: T8152Probe) -> Result> { + let dev: &kernel::device::Device> = pdev.as_ref(); + let stage = probe.stage; + let mut retained = prepare_firmware_session(pdev, probe)?; + if stage == STAGE_RENDER { + // A shallow workqueue stack avoids nesting render builders beneath + // platform probe/module loading. The closure owns the retained session + // before any ASC start, including on spawn failure and every exit. + kernel::workqueue::system_long().try_spawn(GFP_KERNEL, move || { + let result = match retained.0.as_mut() { + Some(session) => session.run(), None => Err(EIO), + }; + if let Some(session) = retained.0.as_mut() { + dev_info!(session.device(), "G18_RENDER_STAGE_DONE result={:?} owners=retained\n", result); + if result.is_err() { + dev_err!(session.device(), "G18_RENDER_H1_FAIL result={:?} owners=retained no-retry\n", result); + session.dump_fault_evidence("render worker"); + } + } + let _ = retained.stop("render worker end"); + if let Some(session) = retained.0.as_mut() { + for _ in 0..4 { if session.drain().is_err() { break; } } + if let Some(memory) = session.memory.as_ref() { memory.log_trace_journal(&session.dev); } + } + drop(retained); + })?; + dev_info!(dev, "G18_RENDER_WORKER queued; ENODEV is diagnostic, watch G18_RENDER_H1_PASS/FAIL and G18_RENDER_STAGE_DONE\n"); + return Err(ENODEV); + } + let session = retained.0.as_mut().ok_or(EIO)?; + let result = session.run(); + dev_info!(dev, "G18 stage {} transaction result={:?}\n", stage, result); + if result.is_err() { + if stage == STAGE_RENDER { dev_err!(dev, "G18_RENDER_H1_FAIL result={:?} owners=retained no-retry\n", result); } + session.dump_fault_evidence("failed stage"); + } + let _ = retained.stop("stage end"); + + // Bulk logging must not delay active job polling. Stop is not qualified, + // so drain only a bounded tail and retain all owners as before. The export + // accounts for consumed records; it does not assert the producer is halted. + if let Some(session) = retained.0.as_mut() { + for _ in 0..4 { + if session.drain().is_err() { + break; + } + } + if let Some(memory) = session.memory.as_ref() { + memory.log_trace_journal(&session.dev); + } + } + + match result { + Ok(()) if matches!(stage, STAGE_HEARTBEAT | STAGE_QUEUE_REGISTER | STAGE_SCHEDULER_BIND | STAGE_POOL_BIND) => { + let owner: Arc = Arc::pin_init( + try_pin_init!(G18Owner { session <- new_mutex!(retained, "G18Owner::session") }), + GFP_KERNEL, + )?; + dev_info!(dev, "G18 stage {} complete; owner retained by the driver\n", stage); + let owner: Arc = owner; + Ok(owner) + } + Ok(()) => { + dev_info!(dev, "G18 stage {} complete; exiting ENODEV with every owner retained\n", stage); + drop(retained); + Err(ENODEV) + } + Err(error) => { + drop(retained); + Err(error) + } + } +} + +/// Entry point called by the driver's T8152 probe gate for +/// `g18_initdata_test >= 1`. +pub(crate) fn start(pdev: &platform::Device>, probe: T8152Probe) -> Result> { + let result = match probe.stage { + STAGE_READ_ONLY => { + stage_read_only(pdev, &probe); + Err(ENODEV) + } + stage if stage_writes_psd(stage) => { + let result = match psd_power_on_service(stage) { + Some(service) => stage_psd_power_on(pdev, &probe, service), + None => stage_psd_restore_idle(pdev), + }; + dev_info!(pdev.as_ref(), "G18 stage {} result={:?}\n", stage, result); + result.and(Err(ENODEV)) + } + stage if stage_runs_firmware(stage) => stage_firmware(pdev, probe), + _ => Err(EINVAL), + }; + // Belt and braces: no stage may ever defer the GPU probe. + let dev: &kernel::device::Device> = pdev.as_ref(); + result.map_err(|error| no_probe_defer(dev, "GPU", "stage", error)) +} diff --git a/drivers/gpu/drm/asahi/g18/boot/recovery.rs b/drivers/gpu/drm/asahi/g18/boot/recovery.rs new file mode 100644 index 0000000000000..d60912f869a6f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/boot/recovery.rs @@ -0,0 +1,432 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! GPU fault and hang recovery on the retained firmware session. +//! +//! The 26A428 GFX firmware resets the GPU itself (recovery routine at b000 +//! 0x6ef0) and hands the host two edges of a status-B handshake: it writes +//! state 1 and spins until the host writes 2, resets, then writes 3 and spins +//! until the host writes 0. It begins one on its own after a fault or its +//! watchdog, posting a type-4 event first, or when the host asks with EP 0x21 +//! type 6 (`AGXArmFirmware::stopFirmwareForRecoveryInspection`), posting no +//! event. This is the halted/resume protocol upstream Asahi drives on +//! G13/G14, in its newer shape. Servicing it makes a fault or hang end one job +//! instead of the boot. +use super::*; +use crate::g18::initdata::{G18RecoveryCause, G18RecoveryWord}; + +/// Recoveries serviced this boot. There is one firmware session per boot. +static RECOVERIES: AtomicU32 = AtomicU32::new(0); + +/// EP 0x21 type 6: b000 0x255b0 requests a host recovery when the low 44 +/// bits equal the status-B recovery epoch. +const HOST_RECOVERY_TAG: u64 = 0x0086_0000_0000_0000; +const HOST_RECOVERY_EPOCH_MASK: u64 = (1 << 44) - 1; +/// The firmware's per-QID queue records, which its restart walk (b000 +/// 0x10760-0x10800) reads: +0x18 an activity gate it refreshes from the +/// scheduler, +0x40 its stamp position, +0x48 the count of stamps to flush. +/// They live in the GFX role's __DATA reservation, whose first byte is VA +/// 0xfffffc0000050000; the host reads them only while the firmware is halted. +const FW_DATA_VA: u64 = 0xffff_fc00_0005_0000; +const FW_QUEUE_RECORDS_VA: u64 = 0xffff_fc00_000b_b530; +const FW_QUEUE_RECORD_STRIDE: usize = 0x58; +/// Queues the application paths submit to: both graphics stages, compute. +const APPLICATION_QIDS: [usize; 3] = [9, 10, 11]; + +/// The firmware halts right after it posts its event or takes the request. +const RECOVERY_HALT_POLLS: u32 = 2000; // x 1 ms +/// Between the host's 2 and the firmware's 3 the GPU is reset. +const RECOVERY_RESET_POLLS: u32 = 5000; // x 1 ms +/// A job that has neither retired nor been recovered by the firmware after +/// this long is hung; the host then requests the recovery itself. +const JOB_WATCHDOG_MS: u32 = 10_000; +/// After a recovery, how long the ended job's queues may take to show that +/// the firmware consumed it. +const RECOVERED_RETIRE_MS: u32 = 2000; + +/// Poll interval for a wait that has lasted `us`: fine while a job is +/// likely to be about to retire, coarse once it has clearly not. +pub(super) const fn poll_interval_us(us: i64) -> i64 { + if us < 2_000 { 20 } else if us < 50_000 { 200 } else { 1_000 } +} + +/// Upper bounds, in microseconds, of the latency buckets; the last bucket is +/// open-ended. +const LATENCY_BOUNDS_US: [i64; 6] = [100, 300, 1_000, 3_000, 10_000, 100_000]; +const LATENCY_BUCKETS: usize = LATENCY_BOUNDS_US.len() + 1; + +/// A cumulative latency histogram for this boot. +pub(super) struct Latency { + buckets: [AtomicU32; LATENCY_BUCKETS], + max_us: AtomicU32, +} + +impl Latency { + const fn new() -> Self { + #[allow(clippy::declare_interior_mutable_const)] + const EMPTY: AtomicU32 = AtomicU32::new(0); + Self { buckets: [EMPTY; LATENCY_BUCKETS], max_us: EMPTY } + } + + /// Count one sample; returns the samples counted so far. + pub(super) fn record(&self, us: i64) -> u32 { + let index = LATENCY_BOUNDS_US.iter().position(|&bound| us <= bound).unwrap_or(LATENCY_BOUNDS_US.len()); + self.buckets[index].fetch_add(1, Ordering::Relaxed); + self.max_us.fetch_max(us.clamp(0, u32::MAX as i64) as u32, Ordering::Relaxed); + self.buckets.iter().map(|bucket| bucket.load(Ordering::Relaxed)).sum() + } + + fn snapshot(&self) -> ([u32; LATENCY_BUCKETS], u32) { + let mut buckets = [0; LATENCY_BUCKETS]; + for (count, bucket) in buckets.iter_mut().zip(&self.buckets) { + *count = bucket.load(Ordering::Relaxed); + } + (buckets, self.max_us.load(Ordering::Relaxed)) + } +} + +/// Kick to retirement of jobs that retired normally, as the session worker +/// sees it. +static JOB_LATENCY: Latency = Latency::new(); +/// A submitting thread's whole wait on the session worker: queueing, +/// preparation, kick, completion and retirement. +pub(super) static SUBMIT_LATENCY: Latency = Latency::new(); + +/// Microseconds after `since` at which each condition was first seen true +/// (0: not yet). Diagnostic: shows which firmware-owned condition a job's +/// retirement waits on. +pub(super) fn note_phases(first: &mut [i64], since: &Instant, conditions: &[bool]) { + let us = since.elapsed().as_micros_ceil().max(1); + for (seen, &holds) in first.iter_mut().zip(conditions) { + if holds && *seen == 0 { + *seen = us; + } + } +} + +/// Whether the `n`th job of a path is sampled for phase timing. +pub(super) fn phases_due(n: u32) -> bool { + n <= 3 || n % 1024 == 0 +} + +/// Whether the `jobs`th retired job is one whose latency summary is logged. +fn latency_report_due(jobs: u32) -> bool { + (jobs >= 64 && jobs.is_power_of_two()) || jobs % 4096 == 0 +} + +/// One serviced firmware recovery. +#[derive(Debug, Copy, Clone)] +pub(super) struct G18Recovery { + pub(super) epoch: u64, + pub(super) host: bool, + pub(super) cause: G18RecoveryCause, + pub(super) us: u64, +} + +/// How one job's bounded wait ended. +pub(super) enum JobEnd { + /// The job retired normally. + Retired, + /// A recovery ended the job; its queues then showed it consumed. + Recovered(G18Recovery), + /// A recovery ended the job; its queues never showed it consumed, so the + /// firmware may still reference its memory. + Stranded(G18Recovery), +} + +/// The two application job paths; each can be quarantined alone. +#[derive(Debug, Copy, Clone)] +pub(super) enum JobPath { + Render, + Compute, +} + +impl JobPath { + pub(super) const fn bit(self) -> u32 { + match self { + JobPath::Render => 1, + JobPath::Compute => 2, + } + } +} + +/// A job failure that leaves the session running. +pub(super) enum JobFailure { + /// Ended by a recovery; the queues were retired and stay usable. + Recovered(G18Recovery), + /// Ended by a recovery; its owners stay retained and its path refuses + /// further jobs. + Stranded(G18Recovery, JobPath), + /// Ended by a recovery; its queues retired and its memory was released, + /// but the path's firmware bindings did not survive the recovery, so the + /// path refuses further jobs. The client is not poisoned. + Unbound(G18Recovery, JobPath, &'static str), + /// [`Self::Recovered`] as reported back to the submitting thread, which + /// needs only whether the job's memory was released. + Released, + /// [`Self::Stranded`] as reported back to the submitting thread. + Retained, +} + +/// How far a failed task reaches. +pub(super) enum Failure { + /// The session can no longer be trusted and is stopped. + Fatal(Error), + /// Only this job failed; the session keeps serving. + Job(JobFailure), +} + +impl From for Failure { + fn from(error: Error) -> Self { + Failure::Fatal(error) + } +} + +impl From for Failure { + fn from(error: kernel::alloc::AllocError) -> Self { + Failure::Fatal(error.into()) + } +} + +pub(super) fn recoveries() -> u32 { + RECOVERIES.load(Ordering::Relaxed) +} + +impl G18Session { + /// Drain the report rings and service any recovery the firmware has + /// begun. The state word, not the event, is authoritative: a host + /// recovery posts no event. + pub(super) fn poll_faults(&mut self, label: &str) -> Result> { + let event = self.drain()?; + if self.any_crashed() { + dev_err!(self.device(), "G18 {}: firmware crashed\n", label); + return Err(EIO); + } + let halted = match self.memory.as_mut() { + Some(memory) => memory.recovery_word()?.state != 0, + None => false, + }; + if !event && !halted { + return Ok(None); + } + if event { + dev_err!(self.device(), "G18 {}: firmware posted a recovery event\n", label); + } + self.recover(label, false).map(Some) + } + + /// Poll before a kick. A recovery with no job in flight blames nothing + /// this job owns, so the job proceeds. + pub(super) fn poll_idle_faults(&mut self, label: &str) -> Result { + if let Some(recovery) = self.poll_faults(label)? { + dev_warn!(self.device(), "G18_RECOVERY_IDLE {} epoch={} host={}: no job was in flight\n", + label, recovery.epoch, recovery.host); + } + Ok(()) + } + + /// Complete one recovery handshake. With `request`, first ask the idle + /// firmware for one. Resumes from whichever edge the firmware is at, so + /// an interrupted handshake can be finished. + pub(super) fn recover(&mut self, label: &str, request: bool) -> Result { + let started = Instant::::now(); + let word = self.recovery_word()?; + if request && word.state == 0 { + let message = HOST_RECOVERY_TAG | (word.epoch & HOST_RECOVERY_EPOCH_MASK); + dev_err!(self.device(), "G18_RECOVERY_REQUEST {} epoch={} message={:#018x}\n", + label, word.epoch, message); + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, message)?; + } + let mut word = word; + if word.state == 0 { + word = self.await_recovery_state(label, "halt", 0, 1, RECOVERY_HALT_POLLS)?; + } + let mut cause = G18RecoveryCause::default(); + if word.state == 1 { + cause = self.memory.as_mut().ok_or(EIO)?.recovery_cause()?; + dev_err!(self.device(), + "G18_RECOVERY_HALT {} epoch={} host={} source-count={} command={:#x} reason={:#x} detail={:#x} slot-mask={:#x} request={:x?}\n", + label, word.epoch, word.host, cause.source_count, cause.command, cause.reason, + cause.detail, cause.slot_mask, cause.request); + let memory = self.memory.as_mut().ok_or(EIO)?; + let progress = [memory.queue_progress(9)?, memory.queue_progress(10)?, memory.queue_progress(11)?]; + dev_err!(self.device(), "G18_RECOVERY_PROGRESS {} qid9={:x?} qid10={:x?} qid11={:x?}\n", + label, progress[0], progress[1], progress[2]); + self.log_firmware_queue_records(label, "halted"); + self.log_parameter_buffer(label, "halted"); + self.publish_queue_progress(label)?; + self.memory.as_mut().ok_or(EIO)?.acknowledge_recovery(1, 2)?; + word.state = 2; + } + if word.state == 2 { + word = self.await_recovery_state(label, "reset", 2, 3, RECOVERY_RESET_POLLS)?; + } + if word.state != 3 { + dev_err!(self.device(), "G18_RECOVERY_FAIL {} phase=clear state={} epoch={}\n", + label, word.state, word.epoch); + return Err(EIO); + } + let info = self.memory.as_mut().ok_or(EIO)?.recovery_cause()?; + self.log_firmware_queue_records(label, "restarted"); + self.log_parameter_buffer(label, "restarted"); + self.memory.as_mut().ok_or(EIO)?.acknowledge_recovery(3, 0)?; + self.drain()?; + // The restart walk retired every stamp up to the last-submitted table + // published before 1->2, so each queue's next kick names its last + // kicked stamp as its parent, as native encodeDependencyList does + // (queue +0xa0 survives restartWorkQueue unchanged). + let serviced = RECOVERIES.fetch_add(1, Ordering::Relaxed) + 1; + let recovery = G18Recovery { + epoch: word.epoch, + host: word.host != 0, + cause: G18RecoveryCause { valid_entries: info.valid_entries, first_entry: info.first_entry, + pending: info.pending, ..cause }, + us: started.elapsed().as_micros_ceil() as u64, + }; + dev_err!(self.device(), + "G18_RECOVERY_DONE {} epoch={} host={} us={} recoveries={} info-valid={} info-first={:x?}\n", + label, recovery.epoch, recovery.host, recovery.us, serviced, + info.valid_entries, info.first_entry); + let logged = (info.valid_entries as usize).min(info.pending.len()); + for &(index, raw) in &info.pending[..logged] { + let entry = raw as u32; + dev_err!(self.device(), "G18_RECOVERY_PENDING {} index={} slot={} flags={:#x} value={:#x} raw={:#018x}\n", + label, index, (entry >> 4) & 0x7f, entry & 0xf, (raw >> 32) as u32, raw); + } + Ok(recovery) + } + + /// Diagnostic: the render PB's firmware-visible words and scene record. + fn log_parameter_buffer(&mut self, label: &str, phase: &str) { + let Some(job) = self.application_job.as_mut() else { return }; + match job.job.parameter_buffer().recovery_words() { + Ok((state, scene_words, counter, scene)) => dev_err!(self.device(), + "G18_RECOVERY_PB {} {} counter={} scene={} state={:x?} scene_record={:x?}\n", + label, phase, counter, scene, state, scene_words), + Err(error) => dev_err!(self.device(), "G18_RECOVERY_PB {} {} unreadable={:?}\n", label, phase, error), + } + } + + /// What native `restartWorkQueue` does before resuming a halted + /// firmware: publish each queue's last submitted kick stamp. The restart + /// walk flushes every stamp from the queue's own position up to this one + /// (count = stamp - position + 1), which is what retires a stamp the + /// recovery killed; without it the queue's next kick waits on that stamp + /// forever. Queues that never kicked keep their zero entry. + fn publish_queue_progress(&mut self, label: &str) -> Result { + let render = self.application_job.as_ref().map_or(0, |job| job.submitted_generation()); + let memory = self.memory.as_mut().ok_or(EIO)?; + let compute = memory.application_compute_generation(); + for (qid, stamp) in [(9, render), (10, render), (11, compute)] { + if stamp != 0 { + memory.publish_queue_progress(qid, 1, u64::from(stamp))?; + } + } + let progress = [memory.queue_progress(9)?, memory.queue_progress(10)?, memory.queue_progress(11)?]; + dev_err!(self.device(), "G18_RECOVERY_PUBLISH {} qid9={:x?} qid10={:x?} qid11={:x?}\n", + label, progress[0], progress[1], progress[2]); + Ok(()) + } + + /// Log the firmware's own records for the application queues. Read-only + /// and diagnostic: a failure to map is logged, never fatal. + fn log_firmware_queue_records(&self, label: &str, phase: &str) { + let (base, size) = self.states[G18Role::Gfx.index()].data; + let map = match usize::try_from(size).map_err(|_| EINVAL).and_then(|size| PhysMap::new(base, size)) { + Ok(map) => map, + Err(error) => { + dev_warn!(self.device(), "G18_RECOVERY_FWQ {} {}: __DATA map failed ({:?})\n", label, phase, error); + return; + } + }; + for qid in APPLICATION_QIDS { + let offset = (FW_QUEUE_RECORDS_VA - FW_DATA_VA) as usize + qid * FW_QUEUE_RECORD_STRIDE; + let mut raw = [0u8; FW_QUEUE_RECORD_STRIDE]; + if let Err(error) = map.copy(offset, &mut raw) { + dev_warn!(self.device(), "G18_RECOVERY_FWQ {} {} qid={}: read failed ({:?})\n", label, phase, qid, error); + continue; + } + let qword = |at: usize| u64::from_le_bytes(raw[at..at + 8].try_into().unwrap_or([0; 8])); + let words: [u64; FW_QUEUE_RECORD_STRIDE / 8] = core::array::from_fn(|index| qword(index * 8)); + dev_err!(self.device(), + "G18_RECOVERY_FWQ {} {} qid={} gate={:#x} position={:#x} count={:#x} raw={:x?}\n", + label, phase, qid, words[3] as u32, words[8], words[9], words); + } + } + + fn recovery_word(&mut self) -> Result { + self.memory.as_mut().ok_or(EIO)?.recovery_word() + } + + fn await_recovery_state(&mut self, label: &str, phase: &str, from: u32, to: u32, polls: u32) + -> Result { + let mut word = self.recovery_word()?; + for _ in 0..polls { + if word.state == to || word.state != from || self.any_crashed() { + break; + } + // Keep the report rings moving while the firmware works. + self.drain()?; + fsleep(Delta::from_millis(1)); + word = self.recovery_word()?; + } + if word.state == to { + return Ok(word); + } + dev_err!(self.device(), "G18_RECOVERY_FAIL {} phase={} state={} epoch={} host={} crashed={}\n", + label, phase, word.state, word.epoch, word.host, self.any_crashed()); + Err(ETIMEDOUT) + } + + /// Wait for one kicked job. `retired(session, ms, recovered, report)` + /// reports whether the job's queues have reached their retired state; + /// once a recovery has ended the job it checks only what the firmware + /// owns, since no stage will complete. `report` is set once per second + /// of waiting (every 250 ms after a recovery) for progress logging. The + /// host watchdog requests a recovery for a job the firmware never flags. + /// Timing is wall clock; most jobs retire within a few hundred + /// microseconds, so polling starts fine and backs off. + pub(super) fn wait_job(&mut self, label: &str, mut retired: F) -> Result + where + F: FnMut(&mut Self, u32, bool, bool) -> Result, + { + let mut recovery: Option = None; + let mut since = Instant::::now(); + let mut reported = 0u32; + loop { + if let Some(serviced) = self.poll_faults(label)? { + if recovery.is_none() { + since = Instant::::now(); + reported = 0; + recovery = Some(serviced); + } + } + let us = since.elapsed().as_micros_ceil(); + let ms = (us / 1000) as u32; + let period = if recovery.is_some() { 250 } else { 1000 }; + let report = ms / period > reported; + if report { + reported = ms / period; + } + if retired(self, ms, recovery.is_some(), report)? { + if recovery.is_none() && latency_report_due(JOB_LATENCY.record(us)) { + let (job, job_max) = JOB_LATENCY.snapshot(); + let (submit, submit_max) = SUBMIT_LATENCY.snapshot(); + dev_info!(self.device(), "G18_JOB_LATENCY bounds_us={:?} job={:?} job_max_us={} submit={:?} submit_max_us={}\n", + LATENCY_BOUNDS_US, job, job_max, submit, submit_max); + } + return Ok(recovery.map_or(JobEnd::Retired, JobEnd::Recovered)); + } + if let Some(ended) = recovery { + if ms >= RECOVERED_RETIRE_MS { + return Ok(JobEnd::Stranded(ended)); + } + } else if ms >= JOB_WATCHDOG_MS { + dev_err!(self.device(), "G18_JOB_WATCHDOG {} ms={}: requesting recovery\n", label, ms); + recovery = Some(self.recover(label, true)?); + since = Instant::::now(); + reported = 0; + } + fsleep(Delta::from_micros(poll_interval_us(us))); + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/boot/render_stage.rs b/drivers/gpu/drm/asahi/g18/boot/render_stage.rs new file mode 100644 index 0000000000000..163111f5ee368 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/boot/render_stage.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! H1 admission and retained-owner construction. H2 is deliberately excluded. +//! The caller must execute every runtime readback gate before publication. +use crate::{driver::AsahiDrmDevice, g18::runtime::graphics_job::G18GraphicsJob, + g18::fw::render_assembly::RenderVariant, mmu}; +use kernel::prelude::*; + +pub(crate) const STAGE_RENDER: u32 = 9; +pub(crate) const H1_ADMITTED: bool = true; + +pub(crate) fn options(variant: u32, profile: u32) -> Result { + let variant = RenderVariant::parse(variant).ok_or(EINVAL)?; + let admitted = ((1..=2).contains(&profile) + && matches!(variant, RenderVariant::Color | RenderVariant::Clear)) + || (profile == 4 && variant == RenderVariant::Color); + if !admitted { return Err(EINVAL); } + Ok(variant) +} + +/// Called before prepare_t8152_device: rejection precedes MMIO, allocation, +/// firmware startup and the init-data publication boundary. +pub(crate) fn admit(dev: &kernel::device::Device, variant: u32, profile: u32) -> Result { + let variant = options(variant, profile)?; + if !H1_ADMITTED { + dev_err!(dev, "G18_RENDER_BLOCKED variant={:?} profile={} frames={} stages={}: native command policy, graphics UMA backing, PB/queue binding and complete pointer/difference ledger are not admitted; refused before MMIO\n", + variant, profile, variant.frames(), variant.stage_count()); + return Err(ENOTSUPP); + } + dev_info!(dev, "G18_RENDER_ADMIT variant={:?} profile={} policy=captured-M5-H1 single-pass\n", variant, profile); + dev_info!(dev, "G18_RENDER_RESOURCE_PAIRS {:?}\n", crate::g18::uapi::render_policy::resource_pairs()); + dev_info!(dev, "G18_RENDER_TA_BARRIER_FLAG {}\n", crate::g18::uapi::render_policy::ta_barrier_flag()); + Ok(()) +} + +#[inline(never)] +pub(crate) fn prepare(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result> { + let variant = crate::g18::params::g18_render_variant.value(); + let profile = crate::g18::params::g18_render_profile.value(); + let options = options(variant, profile)?; + G18GraphicsJob::from_payload(dev, uat, profile, options) +} diff --git a/drivers/gpu/drm/asahi/g18/boot/session.rs b/drivers/gpu/drm/asahi/g18/boot/session.rs new file mode 100644 index 0000000000000..a44fe7f872f22 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/boot/session.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Stage10 owner: synchronous DRM boundary, shallow worker stack. A job that a +//! firmware recovery ends fails alone; any other fault is sticky. +use super::*; +#[path = "compute_session.rs"] +mod compute_session; +#[path = "recovery.rs"] +mod recovery; +use recovery::{Failure, JobEnd, JobFailure, JobPath}; +use core::{ops::Range,sync::atomic::AtomicI32}; +use crate::{g18::uapi::application::{G18ApplicationRender,G18RenderJobSink,G18RenderRetirement}, + g18::drm::G18DrmRuntime, + g18::runtime::graphics_job::application::ApplicationJob}; + +pub(crate) struct ApplicationOwner { + session: Arc>, + failed: Arc, + /// [`JobPath::bit`]s of paths a stranded job quarantined. + quarantined: Arc, +} + +impl ApplicationOwner { + fn admit(&self,path:JobPath)->Result { + if self.quarantined.load(Ordering::Acquire)&path.bit()==0 {return Ok(());} + if let Some(session)=self.session.lock().0.as_ref() { + dev_err!(session.device(),"G18_APP_JOB_FAILED error={:?} refused path={:?} quarantined session=live\n",EIO,path); + } + Err(EIO) + } + + /// Run one task on the session worker. The outcome crosses back as one + /// word: [`TICKET_RUNNING`], 0, a negative errno for a fatal failure, or + /// a positive [`JobOutcome`] code for a job that failed alone. + fn work(&self, task: F) -> core::result::Result<(),Failure> + where F: FnOnce(&mut G18Session)->core::result::Result<(),Failure> + Send + 'static { + if self.failed.load(Ordering::Acquire) {return Err(Failure::Fatal(EIO));} + let session=self.session.clone();let failed=self.failed.clone(); + let quarantined=self.quarantined.clone(); + let ticket=Arc::new(AtomicI32::new(TICKET_RUNNING),GFP_KERNEL)?;let done=ticket.clone(); + kernel::workqueue::system_long().try_spawn(GFP_KERNEL,move || { + let mut owner=session.lock(); + let result=if failed.load(Ordering::Acquire) {Err(Failure::Fatal(EIO))} else { + owner.0.as_mut().ok_or(Failure::Fatal(EIO)).and_then(|session|task(session)) + }; + // A job that a recovery ended fails alone; the firmware session + // keeps serving other jobs. Anything else ends the session. + let result=match result { + Ok(()) => Ok(()), + Err(Failure::Job(failure)) => match owner.0.as_ref() { + Some(session) if !session.any_crashed() => { + if let JobFailure::Stranded(_,path) | JobFailure::Unbound(_,path,_)=&failure { + quarantined.fetch_or(path.bit(),Ordering::AcqRel); + } + log_job_failure(session,&failure); + Err(Failure::Job(failure)) + } + _ => Err(Failure::Fatal(EIO)), + }, + Err(fatal) => Err(fatal), + }; + if let Err(Failure::Fatal(error))=result { + failed.store(true,Ordering::Release); + if let Some(session)=owner.0.as_mut() { + if let Some(job)=session.application_job.as_mut() {job.quarantine();} + if let Some(memory)=session.memory.as_mut() {memory.quarantine_application_compute();} + dev_err!(session.device(),"G18_APP_FAIL error={:?} owners=retained no-retry\n",error); + session.dump_fault_evidence("application worker"); + if let Some(memory)=session.memory.as_ref() {memory.log_trace_journal(&session.dev);} + } + let _=owner.stop("application fault"); + } + done.store(match result { + Ok(())=>0, + Err(Failure::Fatal(error))=>error.to_errno(), + Err(Failure::Job(JobFailure::Recovered(_) | JobFailure::Unbound(..) | JobFailure::Released))=>JobOutcome::Recovered as i32, + Err(Failure::Job(JobFailure::Stranded(..) | JobFailure::Retained))=>JobOutcome::Stranded as i32, + },Ordering::Release); + })?; + let waited=Instant::::now(); + loop { + match ticket.load(Ordering::Acquire) { + TICKET_RUNNING => fsleep(Delta::from_micros(recovery::poll_interval_us(waited.elapsed().as_micros_ceil()))), + 0 => {recovery::SUBMIT_LATENCY.record(waited.elapsed().as_micros_ceil());return Ok(())} + code if code==JobOutcome::Recovered as i32 => return Err(Failure::Job(JobFailure::Released)), + code if code==JobOutcome::Stranded as i32 => return Err(Failure::Job(JobFailure::Retained)), + errno => return Err(Failure::Fatal(Error::from_errno(errno))), + } + } + } + + /// Fail one client's job. Its VM is poisoned only while the firmware may + /// still reference the job's memory; a job whose queues retired after a + /// recovery fails alone, as a faulting job does upstream. + fn finish(vm:&mmu::Vm,result:core::result::Result<(),Failure>)->Result { + match result { + Ok(()) => Ok(()), + Err(Failure::Job(JobFailure::Released)) | Err(Failure::Job(JobFailure::Recovered(_))) => Err(EIO), + Err(Failure::Job(_)) => {if let Some(status)=vm.status(){status.record(EIO);}Err(EIO)} + Err(Failure::Fatal(error)) => {if let Some(status)=vm.status(){status.record(error);}Err(error)} + } + } +} + +/// Worker ticket values besides 0 and negative errnos. +const TICKET_RUNNING: i32 = 1; +#[repr(i32)] +enum JobOutcome { + Recovered = 2, + Stranded = 3, +} + +impl G18Runtime for ApplicationOwner { + fn shutdown(&self) { + self.failed.store(true,Ordering::Release); + let _=self.session.lock().stop("DRM unbind"); + } +} +impl G18DrmRuntime for ApplicationOwner { + fn submit_compute(&self,vm:&mmu::Vm,command:&crate::g18::uapi::command::TranslatedComputeCommand) + ->Result { + let command=KBox::new(*command,GFP_KERNEL)?;let job_vm=vm.clone(); + self.admit(JobPath::Compute)?; + Self::finish(vm,self.work(move |session|session.submit_application_compute(&job_vm,&command)))?; + Ok(G18RenderRetirement {stage_ticks:None}) + } + fn is_crashed(&self)->bool { + self.failed.load(Ordering::Acquire) + || self.session.lock().0.as_ref().map_or(true,|s|s.any_crashed()) + } + fn new_user_vm(&self,id:u64,kernel_range:Range)->Result { + if self.failed.load(Ordering::Acquire) {return Err(EIO);} + self.session.lock().0.as_ref().ok_or(EIO)?.uat.new_vm(id,kernel_range)?.with_t8152_lifetime() + } +} +impl G18RenderJobSink for ApplicationOwner { + fn submit_render(&self,vm:&mmu::Vm,render:&G18ApplicationRender)->Result { + let render=KBox::new(*render,GFP_KERNEL)?;let job_vm=vm.clone(); + self.admit(JobPath::Render)?; + Self::finish(vm,self.work(move |session|session.submit_application(&job_vm,&render)))?; + // K8 uses the adapter's architectural-counter bracket until firmware + // timestamp units and stage boundaries have hardware qualification. + Ok(G18RenderRetirement {stage_ticks:None}) + } +} + +pub(crate) fn start_drm(pdev:&platform::Device>,probe:T8152Probe) + ->Result<(Arc,Arc)> { + if probe.stage!=STAGE_DRM {return Err(EINVAL);} + let retained=prepare_firmware_session(pdev,probe)?; + let session=Arc::pin_init(kernel::new_mutex!(retained,"G18 application session"),GFP_KERNEL)?; + let owner=Arc::new(ApplicationOwner {session,failed:Arc::new(AtomicBool::new(false),GFP_KERNEL)?, + quarantined:Arc::new(AtomicU32::new(0),GFP_KERNEL)?},GFP_KERNEL)?; + owner.work(|session|Ok(session.run()?)).map_err(|failure| match failure { + Failure::Fatal(error) => error, + Failure::Job(_) => EIO, + })?; + dev_info!(pdev.as_ref(),"G18_DRM_READY firmware=live graphics=on-first-submit timestamps=K8\n"); + crate::g18::debug::g18_mark_drm_live(); + Ok((owner.clone(),owner)) +} + +fn log_job_failure(session:&G18Session,failure:&JobFailure) { + match failure { + JobFailure::Recovered(r) => dev_err!(session.device(), + "G18_APP_JOB_FAILED error={:?} recovery epoch={} host={} us={} reason={:#x} slot-mask={:#x} queues=retired session=live recoveries={}\n", + EIO,r.epoch,r.host,r.us,r.cause.reason,r.cause.slot_mask,recovery::recoveries()), + JobFailure::Stranded(r,path) => dev_err!(session.device(), + "G18_APP_JOB_FAILED error={:?} recovery epoch={} host={} us={} reason={:#x} slot-mask={:#x} queues=stranded path={:?} quarantined session=live recoveries={}\n", + EIO,r.epoch,r.host,r.us,r.cause.reason,r.cause.slot_mask,path,recovery::recoveries()), + JobFailure::Unbound(r,path,unready) => dev_err!(session.device(), + "G18_APP_JOB_FAILED error={:?} recovery epoch={} host={} us={} reason={:#x} slot-mask={:#x} queues=retired unbound={} path={:?} quarantined session=live recoveries={}\n", + EIO,r.epoch,r.host,r.us,r.cause.reason,r.cause.slot_mask,unready,path,recovery::recoveries()), + // Only the submitting thread's summaries; the worker logs the rest. + JobFailure::Released | JobFailure::Retained => {} + } +} + +impl G18Session { + #[inline(never)] + fn submit_application(&mut self,vm:&mmu::Vm,render:&G18ApplicationRender)->core::result::Result<(),Failure> { + self.poll_idle_faults("application prepare")?; + if self.application_job.is_none() { + self.application_job=Some(ApplicationJob::new(self.render_job.take().ok_or(EIO)?)?); + } + let job=self.application_job.as_mut().ok_or(EIO)?; + job.prepare(&self.dev,&self.uat,vm,render)?; + if job.pool_aliases.is_empty() { + job.pool_aliases=self.memory.as_mut().ok_or(EIO)?.application_aliases(vm)?; + } + let frame=job.frame();let pass=job.pass()?; + if frame.generation==1 { + for phase in 1..=3 { + self.check_faults("application binding")?; + let queues=self.application_job.as_mut().ok_or(EIO)?.job.queues(); + let records=match phase {1=>queues.install_records(&pass)?,2=>queues.scheduler_records(&pass)?,_=>queues.pool_records(&pass)?}; + let outer_after=self.memory.as_mut().ok_or(EIO)?.publish_application_records(&records)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit).send_message(EP_GFX_INTERRUPTS,0x0083_0000_0000_000a)?; + let mut passed=false; + for _ in 0..CONTROL_POLLS { + self.check_faults("application binding")?; + if self.memory.as_mut().ok_or(EIO)?.compute_outer_counters()?==[outer_after;3] + && self.application_job.as_mut().ok_or(EIO)?.job.queues().stage_gate(phase,&pass)? {passed=true;break;} + fsleep(Delta::from_millis(1)); + } + if !passed {return Err(ETIMEDOUT.into());} + dev_info!(self.device(),"G18_APP_BIND_PASS phase={}\n",phase); + } + } + let job=self.application_job.as_mut().ok_or(EIO)?; + if job.job.parameter_buffer().count_submitted_render()?!=frame.generation {return Err(EIO.into());} + self.memory.as_mut().ok_or(EIO)?.audit_application(&self.dev,job)?; + job.job.queues().prepare_application_pass(&self.dev,&pass,frame)?; + self.poll_idle_faults("application pre-kick")?; + let job=self.application_job.as_mut().ok_or(EIO)?; + let traced=job.traced() || crate::g18::params::g18_graphics_stall_trace.value()!=0; + let roots=job.publish()?; + let records=job.job.queues().publish_application_pass(frame)?; + let outer_after=self.memory.as_mut().ok_or(EIO)?.publish_application_outer(&records,frame)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit).send_message(EP_GFX_INTERRUPTS,0x0083_0000_0000_000a)?; + // The gallery needs a kick/retirement pair for every isolated client: + // each client's first frames are logged. Other frames stay sparse; + // explicit stall tracing records each job. + if traced { + crate::g18_kick_info!(self.device(),"G18_APP_KICK frame={} context={} roots={:x?} order=3D,TA stamp={:#x}\n",frame.generation,pass.context_id,roots,frame.stamp()); + } + // Phases: outer, inner, scheduler submitted, completed, linked, list, + // stamps, completion masks. + let kicked=Instant::::now();let mut first=[0i64;8]; + let end=self.wait_job("application completion",|session,ms,recovered,report| { + let memory=session.memory.as_mut().ok_or(EIO)?; + let outer=memory.compute_outer_counters()?;let masks=memory.completion_masks(); + let job=session.application_job.as_mut().ok_or(EIO)?; + let inner=job.job.queues().counters()?;let stamps=job.job.stamps()?; + let scheduler=job.job.scheduler_snapshot()?;let list=job.job.queues().completion_list()?; + // What the firmware owns: both queues consumed and the scheduler + // retired. After a recovery no stage completes, so the stamps and + // completion event are not required. + let consumed=outer==[outer_after;3] && inner==[[frame.inner_after();3];2] + && scheduler.submitted==frame.submitted() && scheduler.completed==frame.submitted() + && scheduler.linked==0 && list==[0,job.job.queues().job_list()]; + let completed=stamps==[frame.stamp();2] && masks[0]&0x600==0x600; + recovery::note_phases(&mut first,&kicked,&[outer==[outer_after;3],inner==[[frame.inner_after();3];2], + scheduler.submitted==frame.submitted(),scheduler.completed==frame.submitted(),scheduler.linked==0, + list==[0,job.job.queues().job_list()],stamps==[frame.stamp();2],masks[0]&0x600==0x600]); + if report { + dev_info!(session.device(),"G18_APP_WAIT frame={} ms={} recovered={} outer={:?} inner={:?} stamps={:x?} masks={:x?} scheduler={:?} list={:x?}\n", + frame.generation,ms,recovered,outer,inner,stamps,masks,scheduler,list); + } + if report && !recovered && (5000..6000).contains(&ms) { + let (state,scene_words,counter,scene)=session.application_job.as_mut().ok_or(EIO)?.job.parameter_buffer().recovery_words()?; + dev_info!(session.device(),"G18_APP_PB_STALL frame={} counter={} scene={} state={:x?} scene_record={:x?}\n", + frame.generation,counter,scene,state,scene_words); + } + Ok(consumed && (completed || recovered)) + })?; + let recovery=match end { + JobEnd::Retired => match self.poll_faults("application retired")? { + None => { + if recovery::phases_due(frame.generation) { + dev_info!(self.device(),"G18_JOB_PHASES frame={} us={:?} (outer inner submitted completed linked list stamps masks)\n", + frame.generation,first); + } + self.application_job.as_mut().ok_or(EIO)?.retired(); + if traced || frame.generation%256==0 { + crate::g18_kick_info!(self.device(),"G18_APP_FRAME_PASS frame={} stamp={:#x} scheduler={} outer={} inner={}\n", + frame.generation,frame.stamp(),frame.submitted(),outer_after,frame.inner_after()); + } + return Ok(()); + } + // Retired, but a recovery followed at once: do not vouch for + // the frame. Its queues are retired either way. + Some(recovery) => recovery, + }, + JobEnd::Recovered(recovery) => recovery, + JobEnd::Stranded(recovery) => { + self.application_job.as_mut().ok_or(EIO)?.quarantine(); + dev_err!(self.device(),"G18_APP_STRANDED frame={} epoch={} host={}: queues never retired; render quarantined, owners retained\n", + frame.generation,recovery.epoch,recovery.host); + return Err(Failure::Job(JobFailure::Stranded(recovery,JobPath::Render))); + } + }; + // No stage will write this frame's completion stamps; the next + // frame's audit requires them as its previous stamp. + let job=self.application_job.as_mut().ok_or(EIO)?; + job.job.publish_recovered_stamps(frame.previous_stamp(),frame.stamp())?; + job.retired(); + dev_err!(self.device(),"G18_APP_RECOVERED frame={} epoch={} host={}: queues retired\n", + frame.generation,recovery.epoch,recovery.host); + let pb=self.application_job.as_mut().ok_or(EIO)?.job.parameter_buffer(); + let (state,scene_words,counter,scene)=pb.recovery_words()?; + dev_err!(self.device(),"G18_APP_PB_POSTRECOVERY frame={} counter={} scene={} state={:x?} scene_record={:x?}\n", + frame.generation,counter,scene,state,scene_words); + // The next frame needs the queues' firmware bindings as they were; a + // recovery may reset them. Refuse the path rather than kick into it. + let queues=self.application_job.as_mut().ok_or(EIO)?.job.queues(); + let unready=queues.render_unready(&pass,frame)?; + let (status,pool,slot)=queues.render_state(&pass)?; + dev_err!(self.device(),"G18_APP_RENDER_POSTRECOVERY frame={} unready={:?} status={:x?} pool={:x?} pb={:x?}\n", + frame.generation,unready,status,pool,slot); + match unready { + None => Err(Failure::Job(JobFailure::Recovered(recovery))), + Some(what) => Err(Failure::Job(JobFailure::Unbound(recovery,JobPath::Render,what))), + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/live.rs b/drivers/gpu/drm/asahi/g18/buffer/live.rs new file mode 100644 index 0000000000000..a9f7ed88b8e54 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/live.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded volatile stores to host-owned shared firmware words. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. +use kernel::prelude::*; + +// Live shared control words are read asynchronously by firmware, including while +// older kicks retire. Byte copies are not a single-copy publication on strict- +// alignment builds. The Runtime lock serializes host RMWs; volatile accesses +// preserve native aligned LDR/STR width, not an atomic RMW across processors. +#[inline] +fn g17p_live_word_bounds(bytes: &[u8], offset: usize, width: usize) -> Result { + if !matches!(width, 4 | 8) + || offset.checked_add(width).is_none_or(|end| end > bytes.len()) + || (bytes.as_ptr() as usize).checked_add(offset) + .is_none_or(|address| address & (width - 1) != 0) + { + return Err(EINVAL); + } + Ok(()) +} + +#[inline] +pub(crate) fn g17p_live_write_u32(bytes: &mut [u8], offset: usize, value: u32) -> Result { + g17p_live_word_bounds(bytes, offset, 4)?; + // SAFETY: checked bounds and natural alignment; host owns this shared word. + unsafe { core::ptr::write_volatile(bytes.as_mut_ptr().add(offset).cast::(), value.to_le()) }; + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/mod.rs b/drivers/gpu/drm/asahi/g18/buffer/mod.rs new file mode 100644 index 0000000000000..5dd07cc47e1f5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 memory owners: kernel-mapped objects and the UMA/compute pools. + +#[allow(dead_code)] +pub(crate) mod pool; +#[allow(dead_code)] +pub(crate) mod object; +#[allow(dead_code)] +pub(crate) mod live; diff --git a/drivers/gpu/drm/asahi/g18/buffer/object.rs b/drivers/gpu/drm/asahi/g18/buffer/object.rs new file mode 100644 index 0000000000000..6697a5b3fa105 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/object.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Kernel-owned GEM objects with fixed firmware-VA mappings (`MappedObject`). +//! +//! Items copied from the J873 module-1134 sources (provenance in +//! the port's extraction ledger), then adapted: +//! the type and the methods G18 calls are `pub(crate)` (in module 1134 they +//! were private to g17_resources, the parent of every G18 resource module); +//! the G17P firmware-cached object pool (fields `fw_pool`/`fw_node`, its +//! Drop path and `new_fw_pooled`) and the constructors and capture helpers +//! only G17P uses are not carried. G18 never creates pooled objects, so its +//! objects always took the non-pooled path, which is the one kept. + +use core::ops::Range; + +use kernel::prelude::*; + +use super::live::g17p_live_write_u32; +use crate::{driver::AsahiDrmDevice, gem, mmu}; + +/// Zero a new kernel object through its cached CPU mapping. +fn zero_object(object: &mut gem::ObjectRef) -> Result { + let size = object.size(); + let vmap = object.vmap()?; + // SAFETY: the VMap covers the object's complete allocation of `size` bytes, + // and the new object is not yet visible to the GPU or another CPU user. + unsafe { core::ptr::write_bytes(vmap.addr.cast::(), 0, size) }; + Ok(()) +} + +pub(crate) struct MappedObject { + /// Present for an owning GPU mapping and absent for a subview retained by + /// another `MappedObject` in the same resource bundle. + pub(crate) mapping: Option, + pub(crate) object: gem::ObjectRef, + pub(crate) iova: u64, + pub(crate) object_offset: usize, + pub(crate) logical_size: usize, + /// Records the existing owning CPU mapping; capture never creates a + /// physical alias or upgrades a WB observation to GPU-coherent evidence. + pub(crate) cpu_wc: bool, +} + +impl MappedObject { + pub(crate) fn new_wc_in_range_with_guards( + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: Range, size: usize, + alignment: u64, prot: mmu::Prot, guard_pages: usize, + ) -> Result { + let mut object = gem::new_kernel_object_wc(dev, size)?; + zero_object(&mut object)?; + let mapping = vm.map_in_range_with_guards(&object.gem, 0..object.size(), + alignment, range, prot, guard_pages)?; + let iova = mapping.iova(); + Ok(Self { mapping: Some(mapping), object, iova, object_offset: 0, + logical_size: size, cpu_wc: true }) + } + + pub(crate) fn map_alias_at_with_guards(&mut self, vm: &mmu::Vm, address: u64, + prot: mmu::Prot, guard_pages: usize) -> Result { + if self.object_offset != 0 { return Err(EINVAL); } + self.object.map_at_with_guards(vm, address, prot, guard_pages) + } + + + pub(crate) fn new_wc_in_range( + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + range: Range, + size: usize, + alignment: u64, + prot: mmu::Prot, + ) -> Result { + let mut object = gem::new_kernel_object_wc(dev, size)?; + zero_object(&mut object)?; + let mapping = object.map_into_range(vm, range, alignment, prot, true)?; + let iova = mapping.iova(); + Ok(Self { + mapping: Some(mapping), + object, + iova, + object_offset: 0, + logical_size: size, + cpu_wc: true, + }) + } + + pub(crate) fn new_wc_at( + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + address: u64, + size: usize, + prot: mmu::Prot, + ) -> Result { + let mut object = gem::new_kernel_object_wc(dev, size)?; + zero_object(&mut object)?; + let mapping = object.map_at(vm, address, prot, false)?; + let iova = mapping.iova(); + Ok(Self { + mapping: Some(mapping), + object, + iova, + object_offset: 0, + logical_size: size, + cpu_wc: true, + }) + } + + /// Replace this object's backing with a larger one at the SAME address. + /// + /// The replacement is allocated and zeroed before the old aperture is + /// released, so an allocation failure leaves the caller's object intact + /// and mapped. Only the caller can know that nothing is reading the old + /// backing; this helper does not synchronize. + pub(crate) fn regrow_wc_at( + &mut self, + dev: &AsahiDrmDevice, + vm: &mmu::Vm, + address: u64, + size: usize, + ) -> Result { + if size <= self.logical_size { + return Ok(()); + } + let mut object = gem::new_kernel_object_wc(dev, size)?; + zero_object(&mut object)?; + self.mapping = None; + let mapping = object.map_at(vm, address, mmu::PROT_GPU_FW_SHARED_RW, false)?; + self.iova = mapping.iova(); + self.mapping = Some(mapping); + self.object = object; + self.object_offset = 0; + self.logical_size = size; + self.cpu_wc = true; + Ok(()) + } + + pub(crate) fn iova(&self) -> u64 { + self.iova + } + + pub(crate) fn with_bytes_mut(&mut self, f: impl FnOnce(&mut [u8]) -> Result) -> Result { + let vmap = self.object.vmap()?; + let bytes = unsafe { + // SAFETY: the VMap covers this object's complete, page-rounded + // allocation; construction bounds-checks the view offset and size. + core::slice::from_raw_parts_mut( + vmap.addr.cast::().add(self.object_offset), + self.logical_size, + ) + }; + f(bytes) + } + + pub(crate) fn read_u32(&mut self, address: u64) -> Result { + let offset = address.checked_sub(self.iova()).ok_or(EINVAL)? as usize; + let end = offset.checked_add(4).ok_or(EINVAL)?; + if end > self.logical_size { + return Err(EINVAL); + } + self.with_bytes_mut(|bytes| { + Ok(u32::from_le_bytes( + bytes[offset..end].try_into().map_err(|_| EINVAL)?, + )) + }) + } + + pub(crate) fn read_u64(&mut self, address: u64) -> Result { + let offset = address.checked_sub(self.iova()).ok_or(EINVAL)? as usize; + let end = offset.checked_add(8).ok_or(EINVAL)?; + if end > self.logical_size { + return Err(EINVAL); + } + self.with_bytes_mut(|bytes| { + Ok(u64::from_le_bytes( + bytes[offset..end].try_into().map_err(|_| EINVAL)?, + )) + }) + } + + pub(crate) fn write_u32(&mut self, address: u64, value: u32) -> Result { + let offset: usize = address.checked_sub(self.iova()).ok_or(EINVAL)?.try_into()?; + self.with_bytes_mut(|bytes| g17p_live_write_u32(bytes, offset, value)) + } + + pub(crate) fn map_alias_at( + &mut self, + vm: &mmu::Vm, + address: u64, + prot: mmu::Prot, + ) -> Result { + if self.object_offset != 0 { + return Err(EINVAL); + } + self.object.map_at(vm, address, prot, false) + } + + pub(crate) fn map_alias_in_range( + &mut self, + vm: &mmu::Vm, + range: Range, + alignment: u64, + prot: mmu::Prot, + ) -> Result { + if self.object_offset != 0 { + return Err(EINVAL); + } + self.object + .map_into_range(vm, range, alignment, prot, false) + } + +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/pool.rs b/drivers/gpu/drm/asahi/g18/buffer/pool.rs new file mode 100644 index 0000000000000..a05985085b8bb --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/pool.rs @@ -0,0 +1,708 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned initial G18 UMA pool resources. No firmware binding is published here. +//! +//! Flags 3 use a sparse directory: a page at each USC stride plus the final +//! page, with hardware addressing starting one KiB before the first page end. +//! Initial mappings reserve their trailing holes using the existing fixed +//! mapping guards. The fixed ideal-witness probe precommits native tail extents +//! before publication; runtime growth is not implemented. +//! The state must be fully encoded before a caller may bind or submit work. +//! A job's pool also needs its hardware UMA descriptor loaded into the table +//! slot before the tag15 that names the pool is published. + +#![allow(dead_code)] + +#[path = "pool_application.rs"] +mod application; +#[path = "pool_shared_compute.rs"] +mod shared_compute; +#[path = "pool_owner_probe.rs"] +mod owner_probe; + +use crate::g18::buffer::object::MappedObject; +use crate::{driver::AsahiDrmDevice, g18::fw::uma as g18_uma, mmu}; +use core::ops::Range; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +// Host-selected addresses in the shared GPU view, disjoint from the descriptor +// and completion aliases. Actual extents are checked before allocation. +const DIRECTORY_BASE: u64 = 0x70_0200_0000; +const BLOCK_LIST_BASE: u64 = 0x70_0400_0000; +const BLOCK_DATA_BASE: u64 = 0x70_0500_0000; +const BLOCK_DATA_STRIDE: u64 = 0x20_0000; +const STATE_BYTES: usize = g18_uma::G18_POOL_STATE_BYTES; +/// Hardware UMA slot of this boot's single pool. Native +/// `AGXUSCPrivMemPool::prepareLocked` takes it from the accelerator's ID +/// manager (accelerator+0x13b98, 0x100 IDs, init 0xfffffe0008c458f4); the +/// first grant on a fresh accelerator is 0. The same ID is the tag15 +0x46 +/// firmware pool slot (descriptor+0x7a8) and the low half of register 0x10241. +pub(crate) const G18_UMA_HARDWARE_SLOT: u32 = 0; +/// prepareLocked rejects IDs >= 0x100 (0xfffffe0008c8f334): 0x8000 bytes. +const UMA_TABLE_SLOTS: usize = 0x100; + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18UmaPoolBinding { + pub(crate) state: u64, + pub(crate) generation: u64, + pub(crate) slot: u32, +} + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18UmaPoolView { + pub(crate) directory_base: u64, + pub(crate) reservation_bytes: u64, + pub(crate) stride: u64, + pub(crate) page_count: usize, + pub(crate) gpu_base_offset: u64, + pub(crate) state_high: u64, + pub(crate) block_list_high: u64, + pub(crate) block_list_low: u64, +} + +pub(crate) struct G18UmaPoolMemory { + // Drop the GPU aliases and their reservations before the CPU/FW mappings. + // Every mapping independently retains its GEM backing and owning VM. + // The job-context aliases go first: that VM's roots may still be live. + process_aliases: KVec, + aliases: KVec, + scratch_aliases: KVec, + scratch: KVec, + scratch_plan: Option, + directory: KVec, + directory_growth: KVec, + directory_growth_offsets: KVec, + shared_compute_plan: Option, + block_list: MappedObject, + state: MappedObject, + // Firmware dereferences state+0x88 during pool retirement. Keep the + // independently allocated counter alive for the entire pool lifetime. + preempt_counter: MappedObject, + geometry: g18_uma::G18UmaGeometry, + layout: g18_uma::G18UmaSparseDirectory, + encoded: bool, + descriptor_loaded: bool, + render: bool, + low_bias: u64, + hardware_slot: u32, + block_stride: u64, +} + +impl G18UmaPoolMemory { + /// Separate application compute pool. The four native gallery compute + /// captures use four two-MiB blocks; reserve their maximum observed result + /// pair/counter for the initial bounded, non-spilling dispatch path. + pub(crate) fn new_application_compute(dev: &AsahiDrmDevice, uat: &mmu::Uat, + high_range: Range) -> Result { + let mut pool = Self::new_layout(dev, uat, high_range.clone(), 12, + true, false, 0x40000000, 1, 0x200000)?; + pool.prepare_compute_backing(dev, uat, high_range, g18_uma::G18ComputeScratchPlan { + block_bytes: 0x200000, block_count: 4, + register_pair: [30, 12], chunk_counters: [0, 260], + })?; + Ok(pool) + } + + pub(crate) fn compute_application_aliases(&mut self, dev: &AsahiDrmDevice, + uat: &mmu::Uat, vm: &mmu::Vm) -> Result> { + if self.render || ![0x40000000, 0x2_4000_0000, 0x3_4000_0000].contains(&self.low_bias) + || !self.process_aliases.is_empty() { + return Err(EINVAL); + } + self.alias_into_process(dev, uat, vm)?; + Ok(core::mem::replace(&mut self.process_aliases, KVec::new())) + } + + /// The retained backing plan; application compute bounds its SMA pair by it. + pub(crate) fn scratch_plan(&self) -> Option { + self.scratch_plan + } + + pub(crate) fn shared_compute_plan(&self) -> Option { + self.shared_compute_plan + } + + pub(crate) fn new( + dev: &AsahiDrmDevice, + uat: &mmu::Uat, + high_range: Range, + uscs: u32, + ) -> Result { + Self::new_with_guard(dev, uat, high_range, uscs, true, false) + } + + /// `guard` reserves one unmapped page after each firmware-view object. + /// The render pool packs its 16 objects into the fixed 0x40000 window + /// of g18_address_map (list/state/counter at +0x34000/+0x38000/+0x3c000), + /// which has no room for guards: with them the directory pages alone + /// exhaust the window (stage33 H1, ENOSPC). Compute keeps its guards. + fn new_with_guard( + dev: &AsahiDrmDevice, + uat: &mmu::Uat, + high_range: Range, + uscs: u32, + guard: bool, + ideal: bool, + ) -> Result { + Self::new_layout(dev, uat, high_range, uscs, guard, ideal, 0, 0, 0x100000) + } + + fn new_layout(dev: &AsahiDrmDevice, uat: &mmu::Uat, high_range: Range, + uscs: u32, guard: bool, ideal: bool, low_bias: u64, hardware_slot: u32, + block_bytes: u64) -> Result { + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 { + return Err(ENODEV); + } + let directory_base = DIRECTORY_BASE + low_bias; + let block_list_base = BLOCK_LIST_BASE + low_bias; + // A 1 MiB block is one native accessor's admitted size. This is the + // initial host policy, not a claim to reproduce every native override. + let geometry = g18_uma::g18_uma_geometry(0x8000_0000, block_bytes, + uscs, false, true, 14).ok_or(EINVAL)?; + let layout = g18_uma::g18_uma_sparse_directory(&geometry, uscs, 14).ok_or(EINVAL)?; + use crate::{g18::address_map as va, g18::uapi::render_policy as policy}; + if ideal && (guard || uscs != 12 || geometry.directory_bytes_per_usc != 0x158000 + || layout.initial_page_count != 13 || layout.page_bytes != 0x4000) { + return Err(EINVAL); + } + if directory_base.checked_add(layout.reservation_bytes).ok_or(EOVERFLOW)? > block_list_base { + return Err(EINVAL); + } + let fixed = |size| MappedObject::new_wc_in_range_with_guards(dev, uat.kernel_vm(), + high_range.clone(), size, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, + usize::from(guard)); + let mut directory = KVec::new(); + let mut directory_growth = KVec::new(); + let mut directory_growth_offsets = KVec::new(); + let mut aliases = KVec::new(); + for index in 0..layout.initial_page_count { + let offset = layout.page_offset(index).ok_or(EINVAL)?; + let end = if index + 1 == layout.initial_page_count { + layout.reservation_bytes + } else { + layout.page_offset(index + 1).ok_or(EINVAL)? + - if ideal { policy::IDEAL_GROWTH_BYTES } else { 0 } + }; + let guards: usize = ((end - offset - layout.page_bytes) / layout.page_bytes).try_into()?; + let mut page = fixed(mmu::UAT_PGSZ)?; + aliases.push(page.map_alias_at_with_guards(uat.kernel_lower_vm(), + directory_base + offset, mmu::PROT_GPU_SHARED_RW, guards)?, GFP_KERNEL)?; + if uat.kernel_vm().translate_iova(page.iova())? + != uat.kernel_lower_vm().translate_iova(directory_base + offset)? { + return Err(EFAULT); + } + directory.push(page, GFP_KERNEL)?; + } + if ideal { + // Original 8c8b8f8..8c8ba08 selects 21 tail pages per USC for + // 40 one-MiB blocks. Preserve the guarded middle of every stride. + for index in 0..12u64 { + let high = va::GRAPHICS_UMA_GROWTH + index * policy::IDEAL_GROWTH_BYTES; + let low = directory_base + index * geometry.directory_bytes_per_usc + + policy::IDEAL_GROWTH_OFFSET; + let mut extent = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, + policy::IDEAL_GROWTH_BYTES as usize, mmu::PROT_FW_SHARED_RW)?; + aliases.push(extent.map_alias_at(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..policy::IDEAL_GROWTH_BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset)? + != uat.kernel_lower_vm().translate_iova(low + offset)? { return Err(EFAULT); } + } + directory_growth.push(extent, GFP_KERNEL)?; + directory_growth_offsets.push(low - directory_base, GFP_KERNEL)?; + } + } + // Validate every PTE in the aperture, including the intentionally + // absent entries. Do this before a firmware owner can see any address. + let mut holes = 0usize; + for offset in (0..layout.reservation_bytes).step_by(mmu::UAT_PGSZ) { + let mapped = uat.kernel_lower_vm().translate_iova(directory_base + offset).is_ok(); + let expected = offset % geometry.directory_bytes_per_usc == 0 + || (ideal && offset < 12 * geometry.directory_bytes_per_usc + && offset % geometry.directory_bytes_per_usc >= policy::IDEAL_GROWTH_OFFSET); + if mapped != expected { return Err(EFAULT); } + if !mapped { holes += 1; } + } + // A missing PTE is not enough: the allocator must also reject a new + // owner inside both the first and last gap. On unexpected success, + // dropping the temporary alias unwinds it before returning an error. + if geometry.directory_bytes_per_usc > layout.page_bytes { + let last_hole = if ideal { 11 * geometry.directory_bytes_per_usc + layout.page_bytes } + else { layout.reservation_bytes - 2 * layout.page_bytes }; + for offset in [layout.page_bytes, last_hole] { + match directory[0].map_alias_at(uat.kernel_lower_vm(), + directory_base + offset, mmu::PROT_GPU_SHARED_RW) { + Err(error) if error == EBUSY || error == ENOSPC => {}, + Err(error) => return Err(error), + Ok(alias) => { drop(alias); return Err(EFAULT); }, + } + } + } + let block_bytes = usize::try_from(u64::from(geometry.block_count) * 4)?; + let block_bytes = block_bytes.checked_add(mmu::UAT_PGSZ - 1).ok_or(EOVERFLOW)? + & !(mmu::UAT_PGSZ - 1); + let mut block_list = fixed(block_bytes)?; + aliases.push(block_list.map_alias_at(uat.kernel_lower_vm(), block_list_base, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..block_bytes).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(block_list.iova() + offset as u64)? + != uat.kernel_lower_vm().translate_iova(block_list_base + offset as u64)? { + return Err(EFAULT); + } + } + let mut state = fixed(STATE_BYTES)?; + state.with_bytes_mut(|bytes| { + // Native initial hardware slot is unassigned. All other fields + // remain unpublished until the native pool-state encoder runs. + bytes[8..12].copy_from_slice(&u32::MAX.to_le_bytes()); + Ok(()) + })?; + // Native FList::init allocates eight bytes at 8c8d28c..8c8d298; + // serialize at 8c8c19c/1b0 publishes its GPU address/current value. + // A null address crashes b000 at 1e5c4 (stage28); zero is a valid + // initial value of a backed counter, never a valid counter address. + let mut preempt_counter = fixed(8)?; + preempt_counter.with_bytes_mut(|bytes| { + if bytes.len() != 8 || bytes.iter().any(|&value| value != 0) { + return Err(EINVAL); + } + Ok(()) + })?; + uat.kernel_vm().translate_iova(preempt_counter.iova())?; + crate::g18_kick_info!(dev.as_ref(), + "G18 UMA retirement counter: owned FW address={:#x} initial=0; unpublished\n", + preempt_counter.iova()); + crate::g18_kick_info!(dev.as_ref(), + "G18 UMA sparse owner: reserved={:#x} backing_pages={} holes={} bias={:#x} aliases=verified guards=verified unpublished\n", + layout.reservation_bytes, layout.initial_page_count, holes, layout.gpu_base_offset); + Ok(Self { + process_aliases: KVec::new(), + aliases, scratch_aliases: KVec::new(), scratch: KVec::new(), scratch_plan: None, + directory, directory_growth, block_list, state, preempt_counter, geometry, layout, encoded: false, + directory_growth_offsets, shared_compute_plan: None, + descriptor_loaded: false, + render: false, low_bias, hardware_slot, + block_stride: if low_bias == 0 { BLOCK_DATA_STRIDE } else { 0x300000 }, + }) + } + + /// Separate deterministic owner for H1. The firmware-memory constructor + /// selects this INSTEAD OF the compute pool; their low windows cannot + /// coexist. All compute constructor and backing policy remain unchanged. + pub(crate) fn new_render(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result { + use crate::g18::address_map as va; + let mut owner = Self::new_with_guard(dev, uat, + va::GRAPHICS_UMA_HIGH..va::GRAPHICS_UMA_HIGH + 0x40000, 12, false, + crate::g18::uapi::render_policy::ideal_enabled())?; + if owner.state.iova() != va::GRAPHICS_UMA_STATE + || owner.block_list.iova() != va::GRAPHICS_UMA_LIST + || owner.preempt_counter.iova() != va::GRAPHICS_UMA_COUNTER { return Err(EFAULT); } + owner.render = true; + Ok(owner) + } + + pub(crate) fn prepare_render_backing(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, + vm: &mmu::Vm) -> Result { + use crate::{g18::address_map as va, g18::uapi::render_policy as policy}; + let blocks = policy::uma_blocks(); + let directory_ready = if policy::ideal_enabled() { + blocks == 40 && self.directory_growth.len() == 12 + && self.directory_growth.iter().all(|extent| extent.logical_size == policy::IDEAL_GROWTH_BYTES as usize) + && self.geometry.directory_bytes_per_usc == 0x158000 + } else { + self.directory_growth.is_empty() + && g18_uma::g18_initial_directory_fits(blocks, 0x100000, 12, 3, 14) == Some(true) + }; + if !self.render || self.encoded || self.scratch_plan.is_some() || !directory_ready { + return Err(EINVAL); + } + let mut entries = KVec::new(); + for index in 0..blocks { + let low = policy::block_base() + u64::from(index) * BLOCK_DATA_STRIDE; + let high = va::GRAPHICS_UMA_BACKING + u64::from(index) * 0x100000; + let mut block = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, 0x100000, + mmu::PROT_FW_SHARED_RW)?; + let alias = block.map_alias_at_with_guards(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW, 1)?; + for offset in (0..0x100000).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset)? + != uat.kernel_lower_vm().translate_iova(low + offset)? { return Err(EFAULT); } + } + entries.push(g18_uma::g18_uma_block_entry(low, 0).ok_or(EINVAL)?, GFP_KERNEL)?; + self.scratch_aliases.push(alias, GFP_KERNEL)?; + self.scratch.push(block, GFP_KERNEL)?; + } + self.block_list.with_bytes_mut(|out| { + for (index, value) in entries.iter().enumerate() { + out[index*4..index*4+4].copy_from_slice(&value.to_le_bytes()); + } + Ok(()) + })?; + self.scratch_plan = Some(g18_uma::G18ComputeScratchPlan { + block_bytes: 0x100000, block_count: blocks, + register_pair: policy::resource_pairs()[0], chunk_counters: policy::uma_chunks(), + }); + self.alias_into_process(dev, uat, vm)?; + dev_info!(dev.as_ref(), "G18_RENDER_UMA backing={}MiB chunks={:?} resource_pairs={:?} ideal={} growth_extents={}; mapped in job context\n", + blocks, policy::uma_chunks(), policy::resource_pairs(), policy::ideal_enabled(), self.directory_growth.len()); + Ok(()) + } + + /// Retain the exact addition fixture's initial backing before publication. + /// A failed allocation/alias check leaves the prior empty pool untouched. + /// No firmware command, process root or hardware pool binding is published. + pub(crate) fn prepare_compute_backing( + &mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, high_range: Range, + plan: g18_uma::G18ComputeScratchPlan, + ) -> Result<()> { + if self.encoded || self.scratch_plan.is_some() { return Err(EBUSY); } + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 + || plan.block_bytes != self.geometry.block_bytes + || !((self.low_bias == 0 && plan.block_bytes == 0x100000 + && plan.block_count == 5 && plan.register_pair == [16, 6] + && plan.chunk_counters == [0, 350]) + || (self.low_bias == 0x40000000 && plan.block_bytes == 0x200000 + && plan.block_count == 4 && plan.register_pair == [30, 12] + && plan.chunk_counters == [0, 260])) + || self.geometry.pool_bytes != 0x8000_0000 + || self.geometry.directory_bytes != self.geometry.directory_bytes_per_usc * 12 + || g18_uma::g18_initial_directory_fits(plan.block_count, plan.block_bytes, 12, 3, 14) + != Some(true) { + return Err(EINVAL); + } + let mut backing = KVec::new(); + let mut aliases = KVec::new(); + let mut entries = KVec::new(); + for index in 0..plan.block_count { + let gpu = BLOCK_DATA_BASE + self.low_bias + u64::from(index) * self.block_stride; + entries.push(g18_uma::g18_uma_block_entry(gpu, 0).ok_or(EINVAL)?, GFP_KERNEL)?; + let mut block = MappedObject::new_wc_in_range( + dev, uat.kernel_vm(), high_range.clone(), plan.block_bytes as usize, + mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, + )?; + // GPU-address alignment, not physical contiguity: the native list + // stores GPU virtual addresses and the UAT retains each GEM page. + let alias = block.map_alias_at_with_guards( + uat.kernel_lower_vm(), gpu, mmu::PROT_GPU_SHARED_RW, 1, + )?; + for offset in (0..plan.block_bytes).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(block.iova() + offset)? + != uat.kernel_lower_vm().translate_iova(gpu + offset)? { + return Err(EFAULT); + } + } + if uat.kernel_lower_vm().translate_iova(gpu + plan.block_bytes).is_ok() { + return Err(EFAULT); + } + aliases.push(alias, GFP_KERNEL)?; + backing.push(block, GFP_KERNEL)?; + } + // All allocations and translations are validated before the first + // publication data changes. Nothing fallible follows this write. + self.block_list.with_bytes_mut(|bytes| { + if bytes.len() < entries.len() * 4 || bytes.iter().any(|&v| v != 0) { + return Err(EBUSY); + } + for (index, entry) in entries.iter().enumerate() { + bytes[index * 4..index * 4 + 4].copy_from_slice(&entry.to_le_bytes()); + } + Ok(()) + })?; + self.scratch_aliases = aliases; + self.scratch = backing; + self.scratch_plan = Some(plan); + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), + "G18 compute scratch: bytes={} blocks={} pair={:?}; retained translations verified unpublished\n", + plan.block_bytes * u64::from(plan.block_count), plan.block_count, plan.register_pair); + Ok(()) + } + + /// Alias the backed directory pages, block list and scratch blocks into the + /// job's VM at their unchanged GPU addresses. The USC walks the descriptor's + /// directory/block-list/block pointers in the job's context: stage27 took a + /// GMMU fault (vm_slot 3, Unmapped read of the block list 0x7004000000) + /// while these existed only in the kernel lower VM. Holes stay unmapped. + /// Call after the backing exists and before the job's roots are published. + pub(crate) fn alias_into_process( + &mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, + ) -> Result<()> { + let plan = self.scratch_plan.ok_or(EINVAL)?; + if !self.process_aliases.is_empty() { return Err(EBUSY); } + let block_list_bytes = (u64::from(self.geometry.block_count) * 4) + .checked_add(mmu::UAT_PGSZ as u64 - 1).ok_or(EOVERFLOW)? & !(mmu::UAT_PGSZ as u64 - 1); + let mut aliases = KVec::new(); + let mut ranges = KVec::new(); + for (index, page) in self.directory.iter_mut().enumerate() { + let gpu = DIRECTORY_BASE + self.low_bias + self.layout.page_offset(index as u32).ok_or(EINVAL)?; + let guards = if let Some(shared) = self.shared_compute_plan { + let commitment = crate::g18::uapi::compute_plan::G18ComputeDirectory::for_blocks( + shared.ideal.block_count).ok_or(EINVAL)?; + if index < 12 { + (self.geometry.directory_bytes_per_usc / self.layout.page_bytes) as usize + - commitment.tail_pages_per_usc as usize - 1 + } else { 0 } + } else { 0 }; + aliases.push(page.map_alias_at_with_guards(vm, gpu, mmu::PROT_GPU_SHARED_RW, guards)?, GFP_KERNEL)?; + ranges.push((gpu, mmu::UAT_PGSZ as u64), GFP_KERNEL)?; + } + for (index, extent) in self.directory_growth.iter_mut().enumerate() { + let gpu = DIRECTORY_BASE + self.low_bias + + *self.directory_growth_offsets.get(index).ok_or(EINVAL)?; + aliases.push(extent.map_alias_at(vm, gpu, mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + ranges.push((gpu, extent.logical_size as u64), GFP_KERNEL)?; + } + aliases.push(self.block_list.map_alias_at(vm, BLOCK_LIST_BASE + self.low_bias, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + ranges.push((BLOCK_LIST_BASE + self.low_bias, block_list_bytes), GFP_KERNEL)?; + let block_base = if self.render { crate::g18::uapi::render_policy::block_base() } else { BLOCK_DATA_BASE + self.low_bias }; + for (index, block) in self.scratch.iter_mut().enumerate() { + let gpu = block_base + index as u64 * self.block_stride; + aliases.push(block.map_alias_at_with_guards(vm, gpu, mmu::PROT_GPU_SHARED_RW, + usize::from(self.shared_compute_plan.is_some()))?, GFP_KERNEL)?; + ranges.push((gpu, plan.block_bytes), GFP_KERNEL)?; + } + // Every job-context page must reach the same memory as the kernel + // lower VM view that the descriptor and block list were encoded from. + let mut pages = 0usize; + for &(gpu, bytes) in ranges.iter() { + for offset in (0..bytes).step_by(mmu::UAT_PGSZ) { + if vm.translate_iova(gpu + offset)? + != uat.kernel_lower_vm().translate_iova(gpu + offset)? { + return Err(EFAULT); + } + pages += 1; + } + } + fence(Ordering::SeqCst); + self.process_aliases = aliases; + crate::g18_kick_info!(dev.as_ref(), + "G18 UMA pool aliased into the job VM: {} mappings, {} pages verified (directory {:#x}, block list {:#x}, blocks {:#x}+{}x{:#x})\n", + ranges.len(), pages, DIRECTORY_BASE + self.low_bias, BLOCK_LIST_BASE + self.low_bias, block_base, + self.scratch.len(), self.block_stride); + Ok(()) + } + + /// Prepare this boot's single compute pool, empty unless backing was + /// explicitly prepared first. Ownership must survive firmware access. + /// Runtime pool growth is not implemented. + pub(crate) fn prepare_binding(&mut self) -> Result { + if self.encoded { return Err(EBUSY); } + let counter_address = self.preempt_counter.iova(); + if counter_address == 0 || counter_address & 7 != 0 { return Err(EINVAL); } + let counter = self.preempt_counter.with_bytes_mut(|bytes| { + if bytes.len() != 8 { return Err(EINVAL); } + let value = u64::from_le_bytes(bytes.try_into().map_err(|_| EINVAL)?); + if value != 0 { return Err(EBUSY); } + Ok(value) + })?; + let encoded = if let Some(plan) = self.shared_compute_plan { + plan.pool_state(1, DIRECTORY_BASE + self.low_bias, + BLOCK_LIST_BASE + self.low_bias, counter_address, counter).ok_or(EINVAL)? + } else { g18_uma::encode_g18_pool_state(&g18_uma::G18UmaPoolState { + // Host identities are local to this one fixed-VA pool and boot. + // The statistics ID and FList generation have different meanings. + statistics_id: 1, + // Native halUMAGetGrowType returns 2; implicit engine init also + // stores 2. This records the protocol, not an implemented grow path. + grow_engine: 2, + priority: 2, + directory_gpu: DIRECTORY_BASE + self.low_bias + self.layout.gpu_base_offset, + directory_stride: self.geometry.directory_bytes_per_usc, + chunk_counters: self.scratch_plan.map_or([0, 0], |plan| plan.chunk_counters), + block_list_gpu: BLOCK_LIST_BASE + self.low_bias, + populated_blocks: self.scratch_plan.map_or(0, |plan| plan.block_count), + block_units_64k: (self.geometry.block_bytes >> 16).try_into()?, + // Native base configureDevice initializes this mode to zero. + sma_mode: 0, + preempt_counter_gpu: counter_address, + preempt_counter: counter, + compute: !self.render, + // Fresh Pool::init clears +0x1f0; prepareBufferResources writes 1. + policy_value: 1, + }) }; + self.state.with_bytes_mut(|bytes| { + if bytes.iter().enumerate().any(|(i, &v)| v != if (8..12).contains(&i) { 0xff } else { 0 }) { + return Err(EBUSY); + } + bytes.copy_from_slice(&encoded); + Ok(()) + })?; + fence(Ordering::SeqCst); + self.encoded = true; + Ok(G18UmaPoolBinding { state: self.state.iova(), generation: 0, slot: self.hardware_slot }) + } + + /// Native `prepareLocked` 0xfffffe0008c8f308..0x8c8f390 on a pool's first + /// use: store the hardware slot at state+0x08 (only when it differs), then + /// `halLoadUMADesc` packs state[..0x7c] with the four SMA words into + /// table[slot]. Plain host stores; native has no barrier or cache + /// maintenance here. The submit's tag15 names the pool only afterwards, so + /// call this before the binding is published. Refuses an occupied slot, a + /// foreign binding or a second load. + pub(crate) fn load_hardware_descriptor( + &mut self, + dev: &AsahiDrmDevice, + table: &mut MappedObject, + binding: G18UmaPoolBinding, + device_config: [u64; 4], + ) -> Result<[u64; g18_uma::G18_UMA_DESCRIPTOR_BYTES / 8]> { + if binding.generation != 0 { return Err(EINVAL); } + self.write_hardware_descriptor(dev, table, binding, device_config, [0; 16], false) + } + + /// Requires the caller's retired scheduler and old firmware binding check. + /// This only replaces our own still-retained descriptor; it cannot reclaim + /// an arbitrary occupied slot or reset an already encoded view in place. + pub(crate) fn replace_hardware_descriptor(&mut self, dev: &AsahiDrmDevice, + table: &mut MappedObject, binding: G18UmaPoolBinding, + previous: &mut Self, old: G18UmaPoolBinding, device_config: [u64; 4]) + -> Result<[u64; 16]> + { + if !previous.descriptor_loaded || previous.render || self.render + || old.state != previous.state.iova() || old.slot != previous.hardware_slot + || binding.slot != old.slot || binding.state == old.state + || old.generation.checked_add(1) != Some(binding.generation) + || binding.generation == u64::MAX { return Err(EINVAL); } + let state = previous.render_state()?; + if state[8..12] != old.slot.to_le_bytes() { return Err(EIO); } + let packed: [u8; g18_uma::G18_UMA_STATE_BYTES] = + state[..g18_uma::G18_UMA_STATE_BYTES].try_into().map_err(|_| EINVAL)?; + let expected = g18_uma::pack_g18_uma_descriptor(&packed, device_config); + self.write_hardware_descriptor(dev, table, binding, device_config, expected, true) + } + + fn write_hardware_descriptor(&mut self, dev: &AsahiDrmDevice, table: &mut MappedObject, + binding: G18UmaPoolBinding, device_config: [u64; 4], expected: [u64; 16], + retired_view: bool) + -> Result<[u64; 16]> + { + const DESCRIPTOR: usize = g18_uma::G18_UMA_DESCRIPTOR_BYTES; + if !self.encoded || self.descriptor_loaded { + dev_err!(dev.as_ref(), "G18_POOL_DESCRIPTOR_REFUSED owner encoded={} loaded={}\n", + self.encoded, self.descriptor_loaded); + return Err(EBUSY); + } + if binding.state != self.state.iova() || binding.slot != self.hardware_slot + || binding.generation == u64::MAX || device_config == [0; 4] { + return Err(EINVAL); + } + let slot = binding.slot as usize; + if slot >= UMA_TABLE_SLOTS { return Err(EINVAL); } + let mut packed = [0u8; g18_uma::G18_UMA_STATE_BYTES]; + self.state.with_bytes_mut(|bytes| { + // The encoded state must still carry the native unassigned slot. + if bytes.len() < STATE_BYTES || bytes[8..12] != u32::MAX.to_le_bytes() { + dev_err!(dev.as_ref(), "G18_POOL_DESCRIPTOR_REFUSED candidate slot is not unassigned\n"); + return Err(EBUSY); + } + packed.copy_from_slice(&bytes[..g18_uma::G18_UMA_STATE_BYTES]); + Ok(()) + })?; + packed[8..12].copy_from_slice(&binding.slot.to_le_bytes()); + let words = g18_uma::pack_g18_uma_descriptor(&packed, device_config); + let offset = slot * DESCRIPTOR; + table.with_bytes_mut(|out| { + if out.len() != UMA_TABLE_SLOTS * DESCRIPTOR { + return Err(EBUSY); + } + for (index, word) in expected.iter().enumerate() { + let at = offset + index * 8; + let actual = u64::from_le_bytes(out[at..at + 8].try_into().map_err(|_| EINVAL)?); + if actual == *word { continue; } + // Native FList prepareBufferResources resets these cursor + // fields separately from immutable identity/geometry: + // state+0x24..0x43 and +0x5c..0x7b. They may have advanced + // in the used descriptor. A retired replacement has its own + // fresh cursors; the first-load check still admits only zero. + if retired_view && matches!(index, 1 | 2 | 7 | 8 | 12..=15) { + crate::g18_kick_info!(dev.as_ref(), + "G18_POOL_DESCRIPTOR_CURSOR slot={} word={} host={:#x} retired={:#x}\n", + binding.slot, index, word, actual); + } else { + dev_err!(dev.as_ref(), + "G18_POOL_DESCRIPTOR_REFUSED slot={} word={} expected={:#x} actual={:#x} retired_view={}\n", + binding.slot, index, word, actual, retired_view); + return Err(EBUSY); + } + } + Ok(()) + })?; + // All checks passed. A later failure still aborts before tag15. + // Native order: state+0x08 first, then the sixteen descriptor words. + self.state.with_bytes_mut(|bytes| { + bytes[8..12].copy_from_slice(&binding.slot.to_le_bytes()); + Ok(()) + })?; + table.with_bytes_mut(|out| { + for (index, word) in words.iter().enumerate() { + let at = offset + index * 8; + out[at..at + 8].copy_from_slice(&word.to_le_bytes()); + } + Ok(()) + })?; + fence(Ordering::SeqCst); + self.descriptor_loaded = true; + crate::g18_kick_info!(dev.as_ref(), + "G18 UMA descriptor loaded: slot={} entry={:#x} state={:#x} state+0x08={} sma={:x?} words={:x?}; tag15 unpublished\n", + binding.slot, table.iova() + offset as u64, binding.state, binding.slot, + device_config, words); + Ok(words) + } + + pub(crate) fn directory_page_high(&self, index: usize) -> Option { + self.directory.get(index).map(MappedObject::iova) + } + + pub(crate) fn verify_render_descriptor(&mut self, table: &mut MappedObject, + device_config: [u64; 4]) -> Result<[u64; 16]> { + if !self.render || !self.descriptor_loaded { return Err(EINVAL); } + let state: [u8; g18_uma::G18_UMA_STATE_BYTES] = self.state.with_bytes_mut(|out| + out[..g18_uma::G18_UMA_STATE_BYTES].try_into().map_err(|_| EINVAL))?; + if u32::from_le_bytes(state[8..12].try_into().map_err(|_| EINVAL)?) != G18_UMA_HARDWARE_SLOT { return Err(EIO); } + let expected = g18_uma::pack_g18_uma_descriptor(&state, device_config); + let mut actual = [0u64; 16]; + for (i, value) in actual.iter_mut().enumerate() { *value = table.read_u64(table.iova() + i as u64 * 8)?; } + if actual != expected || actual == [0; 16] { return Err(EIO); } + Ok(actual) + } + + pub(crate) fn render_state(&mut self) -> Result<[u8; STATE_BYTES]> { + self.state.with_bytes_mut(|out| out.try_into().map_err(|_| EINVAL)) + } + + pub(crate) fn audit_render_mappings(&self, uat: &mmu::Uat, vm: &mmu::Vm) -> Result { + if !self.render || self.scratch.len() != crate::g18::uapi::render_policy::uma_blocks() as usize { return Err(EINVAL); } + let mut pages = 0; + let mut audit = |high: u64, low: u64, bytes: usize| -> Result { + for offset in (0..bytes).step_by(mmu::UAT_PGSZ) { + let pa = uat.kernel_vm().translate_iova(high + offset as u64)?; + if vm.translate_iova(low + offset as u64)? != pa + || uat.kernel_lower_vm().translate_iova(low + offset as u64)? != pa { return Err(EFAULT); } + pages += 1; + } + Ok(()) + }; + for (i, page) in self.directory.iter().enumerate() { + audit(page.iova(), DIRECTORY_BASE + self.low_bias + self.layout.page_offset(i as u32).ok_or(EINVAL)?, mmu::UAT_PGSZ)?; + } + for (i, extent) in self.directory_growth.iter().enumerate() { + audit(extent.iova(), DIRECTORY_BASE + i as u64 * self.geometry.directory_bytes_per_usc + + crate::g18::uapi::render_policy::IDEAL_GROWTH_OFFSET, extent.logical_size)?; + } + audit(self.block_list.iova(), BLOCK_LIST_BASE, self.block_list.logical_size)?; + for (i, block) in self.scratch.iter().enumerate() { + audit(block.iova(), crate::g18::uapi::render_policy::block_base() + i as u64 * BLOCK_DATA_STRIDE, block.logical_size)?; + } + Ok(pages) + } + + pub(crate) fn view(&self) -> G18UmaPoolView { + G18UmaPoolView { + directory_base: DIRECTORY_BASE, reservation_bytes: self.layout.reservation_bytes, + stride: self.geometry.directory_bytes_per_usc, page_count: self.directory.len(), + gpu_base_offset: self.layout.gpu_base_offset, state_high: self.state.iova(), + block_list_high: self.block_list.iova(), block_list_low: BLOCK_LIST_BASE, + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/pool_application.rs b/drivers/gpu/drm/asahi/g18/buffer/pool_application.rs new file mode 100644 index 0000000000000..3e1e012966a0c --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/pool_application.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +use super::*; +impl G18UmaPoolMemory { + pub(crate) fn verify_application_descriptor(&mut self, table: &mut MappedObject, + device_config: [u64;4], first: bool) -> Result<[u64;16]> { + if first {return self.verify_render_descriptor(table,device_config);} + if !self.render || !self.descriptor_loaded {return Err(EINVAL);} + let state: [u8;g18_uma::G18_UMA_STATE_BYTES]=self.state.with_bytes_mut(|out| + out[..g18_uma::G18_UMA_STATE_BYTES].try_into().map_err(|_|EINVAL))?; + let expected=g18_uma::pack_g18_uma_descriptor(&state,device_config); + let mut actual=[0;16]; + for (i,v) in actual.iter_mut().enumerate() {*v=table.read_u64(table.iova()+i as u64*8)?;} + // Directory address/stride, device masks, modes, block-list address + // and block geometry stay fixed. Allocation cursors/counters remain + // hardware-owned; never restore the initial row over their progress. + for i in [0,3,4,5,6,10] { + if actual[i]!=expected[i] {return Err(EIO);} + } + if actual[11]>u64::from(crate::g18::uapi::render_policy::UMA_BLOCKS) {return Err(EIO);} + Ok(actual) + } + + pub(crate) fn application_aliases(&mut self, uat: &mmu::Uat, vm: &mmu::Vm) + -> Result> { + if !self.render || self.scratch_plan.is_none() {return Err(EINVAL);} + let mut aliases=KVec::new(); + for (index,page) in self.directory.iter_mut().enumerate() { + let address=DIRECTORY_BASE+self.layout.page_offset(index as u32).ok_or(EINVAL)?; + aliases.push(page.map_alias_at(vm,address,mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?; + } + aliases.push(self.block_list.map_alias_at(vm,BLOCK_LIST_BASE,mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?; + for (index,block) in self.scratch.iter_mut().enumerate() { + aliases.push(block.map_alias_at(vm,BLOCK_DATA_BASE+index as u64*BLOCK_DATA_STRIDE, + mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?; + } + self.audit_render_mappings(uat,vm)?;Ok(aliases) + } +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/pool_owner_probe.rs b/drivers/gpu/drm/asahi/g18/buffer/pool_owner_probe.rs new file mode 100644 index 0000000000000..ed6d3cde1d966 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/pool_owner_probe.rs @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Opt-in allocation E2E on the real UAT, before firmware publication. +//! This never submits GPU work or claims shader/CTS qualification. + +use super::*; +use crate::g18::uapi::compute_plan::{G18ComputeDirectory, G18ComputePlan, G18_COMPUTE_BLOCK_BYTES}; + +struct Snapshot { + translations: KVec<(u64, u64, u64)>, + list: KVec, + state: [u8; STATE_BYTES], + plan: Option, +} + +impl G18UmaPoolMemory { + fn owner_snapshot(&mut self, uat: &mmu::Uat) -> Result { + let plan = self.shared_compute_plan; + let blocks = self.scratch_plan.ok_or(EINVAL)?.block_count; + let directory = G18ComputeDirectory::for_blocks(blocks).ok_or(EINVAL)?; + let mut translations = KVec::new(); + let mut capture = |high: u64, low: u64, size: usize| -> Result { + for offset in (0..size).step_by(mmu::UAT_PGSZ) { + let high = high + offset as u64; + let low = low + offset as u64; + let physical = uat.kernel_vm().translate_iova(high)?; + if uat.kernel_lower_vm().translate_iova(low)? != physical { return Err(EFAULT); } + translations.push((high, low, physical as u64), GFP_KERNEL)?; + } + Ok(()) + }; + let base = DIRECTORY_BASE + self.low_bias; + for (index, page) in self.directory.iter().enumerate() { + capture(page.iova(), base + self.layout.page_offset(index as u32).ok_or(EINVAL)?, + mmu::UAT_PGSZ)?; + } + for (extent, offset) in self.directory_growth.iter().zip(self.directory_growth_offsets.iter()) { + capture(extent.iova(), base + offset, extent.logical_size)?; + } + capture(self.block_list.iova(), BLOCK_LIST_BASE + self.low_bias, self.block_list.logical_size)?; + for (index, block) in self.scratch.iter().enumerate() { + capture(block.iova(), BLOCK_DATA_BASE + self.low_bias + index as u64 * self.block_stride, + block.logical_size)?; + } + let tail_bytes = u64::from(directory.tail_pages_per_usc) * self.layout.page_bytes; + for offset in (0..directory.reservation_bytes()).step_by(mmu::UAT_PGSZ) { + let in_stride = offset % self.geometry.directory_bytes_per_usc; + let expected = in_stride == 0 || (offset < 12 * self.geometry.directory_bytes_per_usc + && in_stride >= self.geometry.directory_bytes_per_usc - tail_bytes); + if uat.kernel_lower_vm().translate_iova(base + offset).is_ok() != expected { + return Err(EFAULT); + } + } + // The sparse holes must still belong to the allocator, not merely + // have absent PTEs. Probe both ends of the aperture when holes exist. + if directory.tail_pages_per_usc < 85 { + for offset in [self.layout.page_bytes, + 11 * self.geometry.directory_bytes_per_usc + self.layout.page_bytes] { + match self.directory[0].map_alias_at(uat.kernel_lower_vm(), base + offset, + mmu::PROT_GPU_SHARED_RW) { + Err(error) if error == EBUSY || error == ENOSPC => {}, + _ => return Err(EFAULT), + } + } + } + let list = self.block_list.with_bytes_mut(|bytes| { + let mut copy = KVec::with_capacity(bytes.len(), GFP_KERNEL)?; + for &byte in bytes.iter() { copy.push(byte, GFP_KERNEL)?; } + Ok(copy) + })?; + Ok(Snapshot { translations, list, state: self.render_state()?, plan }) + } + + fn unchanged(&mut self, uat: &mmu::Uat, before: &Snapshot) -> Result { + let after = self.owner_snapshot(uat)?; + if before.translations[..] != after.translations[..] || before.list[..] != after.list[..] + || before.state != after.state || before.plan != after.plan { return Err(EFAULT); } + Ok(()) + } + + /// E2E injection at the last allocation boundary: a complete replacement + /// is discarded before descriptor publication. Compare every old mapping, + /// state byte and list entry, then let the real preparation reuse the bank. + pub(crate) fn verify_replacement_rollback(&mut self, dev: &AsahiDrmDevice, + uat: &mmu::Uat, plan: G18ComputePlan) -> Result { + use crate::g18::address_map as va; + let before = self.owner_snapshot(uat)?; + let mut staged = self.replacement_shared_compute(dev, uat, + va::DYNAMIC_HIGH_START..va::DYNAMIC_HIGH_END, plan)?; + let _binding = staged.prepare_binding()?; + let staged_before = staged.owner_snapshot(uat)?; + let bias = staged.low_bias; + let retained = self.scratch.len().min(plan.ideal.block_count as usize); + drop(staged); + self.unchanged(uat, &before)?; + for &(high, low, _) in staged_before.translations.iter() { + if uat.kernel_vm().translate_iova(high).is_ok() + || uat.kernel_lower_vm().translate_iova(low).is_ok() { return Err(EFAULT); } + } + dev_info!(dev.as_ref(), + "G18_COMPUTE_POOL_ROLLBACK old_pages={} staged_pages={} retained_blocks={} released_bias={:#x} state=list=translations=unchanged unpublished=1\n", + before.translations.len(), staged_before.translations.len(), retained, bias); + Ok(()) + } + + #[inline(never)] + pub(crate) fn verify_shared_compute_owner(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result { + use crate::g18::address_map as va; + let high = va::DYNAMIC_HIGH_START..va::DYNAMIC_HIGH_END; + let mut words = [48,12,0,0,2304,24,0,0,2,1,0,0,0,0,0,0,1,0,0,128,4,0,0]; + let inverse = G18ComputePlan::from_descriptor(&words).ok_or(EINVAL)?; + words[4] = 48; + let small = G18ComputePlan::from_descriptor(&words).ok_or(EINVAL)?; + if inverse.ideal.bytes != 30 * 1024 * 1024 || inverse.ideal.block_count != 15 + || small.ideal.block_count != 2 { return Err(EINVAL); } + dev_info!(dev.as_ref(), "G18_COMPUTE_OWNER_E2E begin unpublished=1 gpu_kicks=0\n"); + let mut pool = Self::new_shared_compute(dev, uat, high.clone(), small)?; + let before = pool.owner_snapshot(uat)?; + // Refuse an actual collision after five additional blocks are staged. + let blocked_low = BLOCK_DATA_BASE + pool.low_bias + 7 * pool.block_stride; + let blocker = pool.directory[0].map_alias_at(uat.kernel_lower_vm(), blocked_low, + mmu::PROT_GPU_SHARED_RW)?; + match pool.prepare_shared_compute(dev, uat, high.clone(), inverse) { + Err(error) if error == EBUSY || error == ENOSPC => {}, + _ => return Err(EFAULT), + } + pool.unchanged(uat, &before)?; + drop(blocker); + for index in 2..15 { + if uat.kernel_lower_vm().translate_iova(BLOCK_DATA_BASE + pool.low_bias + + index * pool.block_stride).is_ok() { return Err(EFAULT); } + } + dev_info!(dev.as_ref(), "G18_COMPUTE_OWNER_E2E scratch_collision=pass previous_pages={}\n", + before.translations.len()); + + // In a fresh 64-MiB subarena, leave exactly thirteen guarded blocks + // and two guarded directory extents free. Third extent allocation + // must fail after two real guard splits, causing their rollback. + let constrained = high.end - 64 * 1024 * 1024..high.end; + let page = mmu::UAT_PGSZ as u64; + let free = 13 * (G18_COMPUTE_BLOCK_BYTES + page) + 2 * (21 * page + page); + let cut = constrained.start + free; + let guard_pages = ((constrained.end - cut) / page - 1) as usize; + let blocker = MappedObject::new_wc_in_range_with_guards(dev, uat.kernel_vm(), + cut..constrained.end, mmu::UAT_PGSZ, page, mmu::PROT_FW_SHARED_RW, guard_pages)?; + if blocker.iova() != cut { return Err(EFAULT); } + match pool.prepare_shared_compute(dev, uat, constrained.clone(), inverse) { + Err(error) if error == ENOSPC => {}, + _ => return Err(EFAULT), + } + pool.unchanged(uat, &before)?; + for at in (constrained.start..cut).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(at).is_ok() { return Err(EFAULT); } + } + drop(blocker); + dev_info!(dev.as_ref(), "G18_COMPUTE_OWNER_E2E directory_allocation_rollback=pass\n"); + + pool.prepare_shared_compute(dev, uat, high.clone(), inverse)?; + let grown = pool.owner_snapshot(uat)?; + for &(high, low, physical) in before.translations.iter() { + if uat.kernel_vm().translate_iova(high)? as u64 != physical + || uat.kernel_lower_vm().translate_iova(low)? as u64 != physical { return Err(EFAULT); } + } + let vm = uat.new_vm(0x267, 0x70_4000_0000..0x70_6000_0000)?.with_t8152_lifetime()?; + pool.alias_into_process(dev, uat, &vm)?; + for &(_, low, physical) in grown.translations.iter() { + if vm.translate_iova(low)? as u64 != physical { return Err(EFAULT); } + } + for usc in [0,11] { + let hole = DIRECTORY_BASE + pool.low_bias + usc * pool.geometry.directory_bytes_per_usc + page; + if vm.translate_iova(hole).is_ok() { return Err(EFAULT); } + match pool.directory[0].map_alias_at(&vm, hole, mmu::PROT_GPU_SHARED_RW) { + Err(error) if error == EBUSY || error == ENOSPC => {}, + _ => return Err(EFAULT), + } + } + // Publication state has the exact pure serializer output, and once + // encoded the allocation path must refuse further growth/reset. + let expected = inverse.pool_state(1, DIRECTORY_BASE + pool.low_bias, + BLOCK_LIST_BASE + pool.low_bias, pool.preempt_counter.iova(), 0).ok_or(EINVAL)?; + pool.prepare_binding()?; + if pool.render_state()? != expected { return Err(EFAULT); } + if pool.prepare_shared_compute(dev, uat, high.clone(), inverse) != Err(EBUSY) { + return Err(EFAULT); + } + dev_info!(dev.as_ref(), + "G18_COMPUTE_OWNER_E2E inverse_pool=pass bytes={} directory_pages=265 translated_pages={} process_aliases=pass state=pass\n", + inverse.ideal.bytes, grown.translations.len()); + drop(pool); + for &(_, low, _) in grown.translations.iter() { + if uat.kernel_lower_vm().translate_iova(low).is_ok() || vm.translate_iova(low).is_ok() { + return Err(EFAULT); + } + } + drop(vm); + // Reusing the exact low addresses catches leaked allocator reservations. + drop(Self::new_shared_compute(dev, uat, high.clone(), inverse)?); + Self::verify_guard_tiers(dev, uat, high)?; + dev_info!(dev.as_ref(), "G18_COMPUTE_OWNER_E2E pass gpu_kicks=0 hardware_descriptor_loaded=0\n"); + Ok(()) + } + + #[inline(never)] + fn verify_guard_tiers(dev: &AsahiDrmDevice, uat: &mmu::Uat, high: Range) -> Result { + let low = DIRECTORY_BASE + 0x40000000; + let mut anchor_object = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), high.clone(), + mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)?; + let mut anchor = anchor_object.map_alias_at_with_guards(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW, 85)?; + let original = uat.kernel_lower_vm().translate_iova(low)?; + let mut tails = KVec::with_capacity(3, GFP_KERNEL)?; + for pages in [21,22,42] { + let extent = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), high.clone(), + pages * mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)?; + let alias = extent.mapping.as_ref().ok_or(EINVAL)?.map_alias_into_guard(&mut anchor, + mmu::PROT_GPU_SHARED_RW)?; + for offset in (0..pages * mmu::UAT_PGSZ).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(extent.iova() + offset as u64)? + != uat.kernel_lower_vm().translate_iova(alias.iova() + offset as u64)? { + return Err(EFAULT); + } + } + tails.push((alias, extent), GFP_KERNEL)?; + } + match anchor_object.mapping.as_ref().ok_or(EINVAL)?.map_alias_into_guard(&mut anchor, + mmu::PROT_GPU_SHARED_RW) { + Err(error) if error == ENOSPC => {}, + _ => return Err(EFAULT), + } + while let Some((tail, extent)) = tails.pop() { + let address = tail.iova(); + anchor.rollback_guard_tail(tail)?; + for offset in (0..extent.logical_size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_lower_vm().translate_iova(address + offset as u64).is_ok() { + return Err(EFAULT); + } + } + if uat.kernel_lower_vm().translate_iova(low)? != original { return Err(EFAULT); } + for (retained, backing) in tails.iter() { + for offset in (0..backing.logical_size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_lower_vm().translate_iova(retained.iova() + offset as u64)? + != uat.kernel_vm().translate_iova(backing.iova() + offset as u64)? { + return Err(EFAULT); + } + } + } + match anchor_object.map_alias_at(uat.kernel_lower_vm(), address, mmu::PROT_GPU_SHARED_RW) { + Err(error) if error == EBUSY || error == ENOSPC => {}, + _ => return Err(EFAULT), + } + } + dev_info!(dev.as_ref(), "G18_COMPUTE_OWNER_E2E directory_tiers=21,43,85 rollback=pass\n"); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/pool_shared_compute.rs b/drivers/gpu/drm/asahi/g18/buffer/pool_shared_compute.rs new file mode 100644 index 0000000000000..f8e9a17cf7817 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/pool_shared_compute.rs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Transactional allocation for a fresh, unpublished shared compute pool. +//! Previously submitted firmware state cannot be grown or reset here. + +use super::*; +use crate::g18::uapi::compute_plan::{G18ComputeDirectory, G18ComputePlan, G18_COMPUTE_BLOCK_BYTES}; + +/// Reserve first, then append without any fallible operation after publication. +fn append_reserved(out: &mut KVec, values: KVec) { + let length = out.len() + values.len(); + assert!(out.capacity() >= length); + for (slot, value) in out.spare_capacity_mut().iter_mut().zip(values) { slot.write(value); } + // SAFETY: capacity was checked and every added slot was initialized once. + unsafe { out.set_len(length); } +} + +struct DirectoryTail { + mapping: Option, + backing: MappedObject, + offset: u64, + anchor: usize, +} + +/// Until commit, every tail restores its original anchor reservation on Drop. +/// Mapping teardown then removes its PTEs before releasing its backing. +struct DirectoryTransaction<'a> { + anchors: &'a mut KVec, + tails: KVec, +} + +impl DirectoryTransaction<'_> { + fn commit(mut self) -> KVec { + core::mem::replace(&mut self.tails, KVec::new()) + } +} + +impl Drop for DirectoryTransaction<'_> { + fn drop(&mut self) { + for tail in self.tails.iter_mut().rev() { + if let Some(mapping) = tail.mapping.take() { + // Each tail came from this exact anchor under exclusive access. + // Reverse order preserves adjacency; rollback allocates nothing. + assert!(self.anchors[tail.anchor].rollback_guard_tail(mapping).is_ok()); + } + } + } +} + +impl G18UmaPoolMemory { + pub(crate) fn new_shared_compute(dev: &AsahiDrmDevice, uat: &mmu::Uat, + high: Range, plan: G18ComputePlan) -> Result + { + let mut pool = Self::new_layout(dev, uat, high.clone(), 12, + true, false, 0x40000000, 1, G18_COMPUTE_BLOCK_BYTES)?; + pool.prepare_shared_compute(dev, uat, high, plan)?; + Ok(pool) + } + + /// A fresh FList view, retaining the previous scratch GEM objects while + /// giving the directory, list, state and counter independent ownership. + /// The caller must keep the old view until a verified firmware rebind. + pub(crate) fn replacement_shared_compute(&self, dev: &AsahiDrmDevice, uat: &mmu::Uat, + high: Range, plan: G18ComputePlan) -> Result + { + if self.render || !self.encoded || !self.descriptor_loaded + || self.hardware_slot != 1 || !self.process_aliases.is_empty() + { return Err(EBUSY); } + let bias = if self.low_bias == 0x2_4000_0000 { 0x3_4000_0000 } + else { 0x2_4000_0000 }; + let mut pool = Self::new_layout(dev, uat, high.clone(), 12, + true, false, bias, 1, G18_COMPUTE_BLOCK_BYTES)?; + pool.prepare_shared_compute_retaining(dev, uat, high, plan, Some(self))?; + Ok(pool) + } + + /// Increase an unpublished pool without moving any existing mapping. + /// Every backing object, alias and list entry is staged. A failure restores + /// all directory guards and leaves the previous plan and block list intact. + pub(crate) fn prepare_shared_compute(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, + high: Range, plan: G18ComputePlan) -> Result + { + self.prepare_shared_compute_retaining(dev, uat, high, plan, None) + } + + fn prepare_shared_compute_retaining(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, + high: Range, plan: G18ComputePlan, retained: Option<&Self>) -> Result + { + if self.encoded || self.descriptor_loaded || !self.process_aliases.is_empty() { + return Err(EBUSY); + } + let previous = self.shared_compute_plan.map_or(0, |p| p.ideal.block_count); + let new = plan.ideal; + let directory = G18ComputeDirectory::for_blocks(new.block_count).ok_or(EINVAL)?; + let old_tail = if previous == 0 { 0 } else { + G18ComputeDirectory::for_blocks(previous).ok_or(EINVAL)?.tail_pages_per_usc + }; + if self.render || ![0x40000000, 0x2_4000_0000, 0x3_4000_0000].contains(&self.low_bias) + || self.hardware_slot != 1 + || self.geometry.block_bytes != G18_COMPUTE_BLOCK_BYTES + || self.geometry.directory_bytes_per_usc != 0x158000 + || self.layout.reservation_bytes != directory.reservation_bytes() + || self.scratch.len() != previous as usize + || (previous == 0 && self.scratch_plan.is_some()) + || new.block_count < previous + || new.bytes != u64::from(new.block_count) * G18_COMPUTE_BLOCK_BYTES + || directory.tail_pages_per_usc < old_tail + || high.start >= high.end + { return Err(EINVAL); } + let added = (new.block_count - previous) as usize; + let extra_pages = directory.tail_pages_per_usc - old_tail; + let extent_count = if extra_pages == 0 { 0 } else { 12 }; + let extent_bytes = extra_pages as usize * mmu::UAT_PGSZ; + // Reject impossible address-space demands before any GEM allocations. + // Actual free ranges and collisions are still checked by the allocator. + let needed = (added as u64).checked_mul(G18_COMPUTE_BLOCK_BYTES).ok_or(EOVERFLOW)? + .checked_add((extent_count * extent_bytes) as u64).ok_or(EOVERFLOW)? + .checked_add(((added + extent_count) * mmu::UAT_PGSZ) as u64).ok_or(EOVERFLOW)?; + if needed > high.end - high.start { return Err(ENOSPC); } + let low_end = (BLOCK_DATA_BASE + self.low_bias) + .checked_add(u64::from(new.block_count) * self.block_stride).ok_or(EOVERFLOW)?; + if low_end > uat.kernel_lower_vm().kernel_range().end { return Err(ENOSPC); } + self.scratch.reserve(added, GFP_KERNEL)?; + self.scratch_aliases.reserve(added, GFP_KERNEL)?; + self.directory_growth.reserve(extent_count, GFP_KERNEL)?; + self.directory_growth_offsets.reserve(extent_count, GFP_KERNEL)?; + self.aliases.reserve(extent_count, GFP_KERNEL)?; + let mut backing = KVec::with_capacity(added, GFP_KERNEL)?; + let mut aliases = KVec::with_capacity(added, GFP_KERNEL)?; + let mut entries = KVec::with_capacity(added, GFP_KERNEL)?; + for index in previous..new.block_count { + let low = BLOCK_DATA_BASE + self.low_bias + u64::from(index) * self.block_stride; + let mut block = if let Some(old) = retained.and_then(|p| p.scratch.get(index as usize)) { + if !old.cpu_wc || old.object_offset != 0 + || old.logical_size != G18_COMPUTE_BLOCK_BYTES as usize { return Err(EINVAL); } + let mut object = crate::gem::ObjectRef::new(old.object.gem.clone()); + let mapping = object.map_into_range(uat.kernel_vm(), high.clone(), + mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, true)?; + let iova = mapping.iova(); + MappedObject { mapping: Some(mapping), object, iova, object_offset: 0, + logical_size: old.logical_size, cpu_wc: true } + } else { + MappedObject::new_wc_in_range(dev, uat.kernel_vm(), high.clone(), + G18_COMPUTE_BLOCK_BYTES as usize, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)? + }; + let alias = block.map_alias_at_with_guards(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW, 1)?; + for offset in (0..G18_COMPUTE_BLOCK_BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(block.iova() + offset)? + != uat.kernel_lower_vm().translate_iova(low + offset)? { return Err(EFAULT); } + if let Some(old) = retained.and_then(|p| p.scratch.get(index as usize)) { + if uat.kernel_vm().translate_iova(old.iova() + offset)? + != uat.kernel_vm().translate_iova(block.iova() + offset)? { return Err(EFAULT); } + } + } + if uat.kernel_lower_vm().translate_iova(low + G18_COMPUTE_BLOCK_BYTES).is_ok() { + return Err(EFAULT); + } + entries.push(g18_uma::g18_uma_block_entry(low, 0).ok_or(EINVAL)?, GFP_KERNEL)?; + aliases.push(alias, GFP_KERNEL)?; + backing.push(block, GFP_KERNEL)?; + } + let mut transaction = DirectoryTransaction { + anchors: &mut self.aliases, + tails: KVec::with_capacity(extent_count, GFP_KERNEL)?, + }; + for usc in 0..extent_count { + let extent = match MappedObject::new_wc_in_range(dev, uat.kernel_vm(), high.clone(), + extent_bytes, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW) { + Ok(extent) => extent, + Err(error) => { + dev_info!(dev.as_ref(), "G18_SHARED_COMPUTE_ROLLBACK directory_tails={} error={:?}\n", + transaction.tails.len(), error); + return Err(error); + } + }; + let offset = (usc as u64 + 1) * self.geometry.directory_bytes_per_usc + - u64::from(directory.tail_pages_per_usc) * self.layout.page_bytes; + let mapping = extent.mapping.as_ref().ok_or(EINVAL)?.map_alias_into_guard( + &mut transaction.anchors[usc], mmu::PROT_GPU_SHARED_RW)?; + let low = DIRECTORY_BASE + self.low_bias + offset; + // Retain rollback ownership before any fallible translation check. + transaction.tails.push(DirectoryTail { mapping: Some(mapping), backing: extent, + offset, anchor: usc }, GFP_KERNEL)?; + let tail = &transaction.tails[usc]; + if tail.mapping.as_ref().ok_or(EINVAL)?.iova() != low { return Err(EFAULT); } + for page in (0..extent_bytes).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(tail.backing.iova() + page as u64)? + != uat.kernel_lower_vm().translate_iova(low + page as u64)? { return Err(EFAULT); } + } + } + // vmap may fail, and list validation may fail: both precede mutation. + // No fallible operation follows the complete block-list update. + self.block_list.with_bytes_mut(|bytes| { + let start = previous as usize * 4; + let end = new.block_count as usize * 4; + if end > bytes.len() || bytes[start..].iter().any(|&v| v != 0) { return Err(EBUSY); } + for (index, value) in entries.iter().enumerate() { + bytes[start + index*4..start + index*4 + 4].copy_from_slice(&value.to_le_bytes()); + } + Ok(()) + })?; + let tails = transaction.commit(); + for mut tail in tails { + // All destination capacity was reserved before staging began. + self.aliases.push(tail.mapping.take().expect("owned directory tail"), GFP_KERNEL) + .expect("reserved alias capacity"); + self.directory_growth.push(tail.backing, GFP_KERNEL).expect("reserved backing capacity"); + self.directory_growth_offsets.push(tail.offset, GFP_KERNEL).expect("reserved offset capacity"); + } + append_reserved(&mut self.scratch_aliases, aliases); + append_reserved(&mut self.scratch, backing); + self.scratch_plan = Some(g18_uma::G18ComputeScratchPlan { + block_bytes: G18_COMPUTE_BLOCK_BYTES, block_count: new.block_count, + register_pair: new.register_pair.map(u64::from), chunk_counters: new.chunk_counters, + }); + self.shared_compute_plan = Some(plan); + fence(Ordering::SeqCst); + crate::g18_kick_info!(dev.as_ref(), + "G18_SHARED_COMPUTE_OWNER bytes={} blocks={} directory_pages={} previous_blocks={} pair={:?} counters={:?} unpublished=1\n", + new.bytes, new.block_count, directory.committed_pages(), previous, + new.register_pair, new.chunk_counters); + if let Some(old) = retained { + crate::g18_kick_info!(dev.as_ref(), + "G18_SHARED_COMPUTE_REPLACEMENT old_bias={:#x} new_bias={:#x} retained_blocks={} blocks={} unpublished=1\n", + old.low_bias, self.low_bias, old.scratch.len().min(new.block_count as usize), new.block_count); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g18/compat/dep_stats.rs b/drivers/gpu/drm/asahi/g18/compat/dep_stats.rs new file mode 100644 index 0000000000000..9eef0d51ccdc7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/dep_stats.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Live render-dependency resolution counters (`g17p_render_dep_stats`, +//! root-only, read-only sysfs). Counter-only: relaxed atomics, no printk. +//! +//! `g17p_live_render_dependency` turns a still-pending predecessor's native +//! token into an explicit kick dependency. Each failure is counted by +//! `SubmissionError` variant, with the last failure's predecessor kick word, +//! fence status and decoded QID. `high_qid` counts dependencies resolved on +//! a predecessor whose fragment QID is 64..127: its kick word carries QID bit +//! 6 in bit 46, the firmware's valid-queue mask-word selector (ubfx x6,x5, +//! #46,#1), which the resolver used to refuse as unmodeled bits. +use core::fmt::Write; +use core::sync::atomic::{AtomicI64, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +use crate::g18::fw::submission::SubmissionError; + +pub(crate) static RESOLVED: AtomicU64 = AtomicU64::new(0); +pub(crate) static HIGH_QID: AtomicU64 = AtomicU64::new(0); +static PREDECESSOR_FAILED: AtomicU64 = AtomicU64::new(0); +static MISSING_TOKEN: AtomicU64 = AtomicU64::new(0); +static UNMODELED_BITS: AtomicU64 = AtomicU64::new(0); +static TIMESTAMP_RANGE: AtomicU64 = AtomicU64::new(0); +static QID_RANGE: AtomicU64 = AtomicU64::new(0); +static OTHER: AtomicU64 = AtomicU64::new(0); +static LAST_WORD: AtomicU64 = AtomicU64::new(0); +static LAST_STATUS: AtomicI64 = AtomicI64::new(0); +static LAST_VARIANT: AtomicU64 = AtomicU64::new(0); + +/// Count one resolved dependency (Ok with a dependency). +#[inline] +pub(crate) fn note_resolved(native_word: u64) { + RESOLVED.fetch_add(1, Ordering::Relaxed); + if (native_word >> crate::g18::fw::submission::KICK_SELECTOR_BIT) & 1 != 0 { + HIGH_QID.fetch_add(1, Ordering::Relaxed); + } +} + +/// Count one resolver failure. Cold: only failing submissions reach it. +#[cold] +#[inline(never)] +pub(crate) fn note_error(error: SubmissionError, status: i32, native_word: u64) { + let (counter, variant) = match error { + SubmissionError::RenderBarrierPredecessorFailed => (&PREDECESSOR_FAILED, 1), + SubmissionError::RenderBarrierMissingToken => (&MISSING_TOKEN, 2), + SubmissionError::KickWordUnmodeledBits => (&UNMODELED_BITS, 3), + SubmissionError::KickTimestampOutOfRange => (&TIMESTAMP_RANGE, 4), + SubmissionError::KickDependencyQueueIdOutOfRange => (&QID_RANGE, 5), + _ => (&OTHER, 6), + }; + counter.fetch_add(1, Ordering::Relaxed); + LAST_WORD.store(native_word, Ordering::Relaxed); + LAST_STATUS.store(i64::from(status), Ordering::Relaxed); + LAST_VARIANT.store(variant, Ordering::Relaxed); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn stats_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let r = |c: &AtomicU64| c.load(Ordering::Relaxed); + let word = LAST_WORD.load(Ordering::Relaxed); + let _ = writeln!(out, + "resolved={} high_qid={} predecessor_failed={} missing_token={} unmodeled_bits={} \ + timestamp_range={} qid_range={} other={} last_variant={} last_status={} \ + last_word={:#x} last_qid={}", + r(&RESOLVED), r(&HIGH_QID), r(&PREDECESSOR_FAILED), r(&MISSING_TOKEN), + r(&UNMODELED_BITS), r(&TIMESTAMP_RANGE), r(&QID_RANGE), r(&OTHER), + r(&LAST_VARIANT), LAST_STATUS.load(Ordering::Relaxed), word, + (word >> crate::g18::fw::submission::KICK_QID_SHIFT) & u64::from(crate::g18::fw::submission::KICK_QID_MASK)); + out.used as core::ffi::c_int +} + +unsafe extern "C" fn stats_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + -1 // EPERM: read-only counters +} + +static STATS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(stats_set), get: Some(stats_get), free: None, +}; +#[link_section = "__param"] +#[used] +static STATS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_render_dep_stats" + } else { c"aurora.g17p_render_dep_stats" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&STATS_OPS), perm: 0o400, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/dvfs_parity.rs b/drivers/gpu/drm/asahi/g18/compat/dvfs_parity.rs new file mode 100644 index 0000000000000..3ec61a902813d --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/dvfs_parity.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Root-only runtime control `g17p_dvfs_parity` for the opt-in native +//! DVFS-input parity diagnostic. Default "0": no RegionC or message change. +//! +//! Write `MASK [HOLD_US [RAMP_US [CE_MAX]]]` (decimal or 0x hex): +//! bit0 work-tied CLPC deadline control effort (RegionC+0xe4), +//! bit1 native submit bracket (RegionC+0x7c/+0x80), +//! bit2 native-gated 0x87 note (requires bit1). +//! Reading prints the configuration, the live effort and event counters. +//! A change takes effect at the next submission or completion service. +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +use crate::g18::compat::g17p::{self as policy, Config}; + +static MASK: AtomicU32 = AtomicU32::new(policy::DEFAULT_CONFIG.mask); +static HOLD_NS: AtomicU64 = AtomicU64::new(policy::DEFAULT_CONFIG.hold_ns); +static RAMP_NS: AtomicU64 = AtomicU64::new(policy::DEFAULT_CONFIG.ramp_ns); +static CE_MAX: AtomicU32 = AtomicU32::new(policy::DEFAULT_CONFIG.ce_max); + +/// Counters for hardware verification. Never reset except by module load. +pub(crate) static CE_PUBLISHES: AtomicU64 = AtomicU64::new(0); +pub(crate) static CE_CLEARS: AtomicU64 = AtomicU64::new(0); +pub(crate) static CE_LIVE: AtomicU32 = AtomicU32::new(0); +pub(crate) static CE_LAST_SET_NS: AtomicU64 = AtomicU64::new(0); +pub(crate) static CE_LAST_CLEAR_NS: AtomicU64 = AtomicU64::new(0); +pub(crate) static RECHECKS: AtomicU64 = AtomicU64::new(0); +pub(crate) static BRACKET_OPENS: AtomicU64 = AtomicU64::new(0); +pub(crate) static KICKS_SENT: AtomicU64 = AtomicU64::new(0); +pub(crate) static KICKS_GATED: AtomicU64 = AtomicU64::new(0); +pub(crate) static STATS_66C_LAST: AtomicU32 = AtomicU32::new(0); +pub(crate) static STATS_66C_NONZERO: AtomicU64 = AtomicU64::new(0); +pub(crate) static PUBLISH_ERRORS: AtomicU64 = AtomicU64::new(0); + +pub(crate) fn config() -> Config { + Config { + mask: MASK.load(Ordering::Relaxed), + hold_ns: HOLD_NS.load(Ordering::Relaxed), + ramp_ns: RAMP_NS.load(Ordering::Relaxed), + ce_max: CE_MAX.load(Ordering::Relaxed), + } +} + +pub(crate) fn enabled(bit: u32) -> bool { + MASK.load(Ordering::Relaxed) & bit != 0 +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn parity_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 64]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let Ok(text) = core::str::from_utf8(&command[..len]) else { return -22; }; + let Some(parsed) = policy::parse(text.trim()) else { return -22; }; + // Order: parameters before the mask, so an enabling write never runs + // with stale timing values. + HOLD_NS.store(parsed.hold_ns, Ordering::Relaxed); + RAMP_NS.store(parsed.ramp_ns, Ordering::Relaxed); + CE_MAX.store(parsed.ce_max, Ordering::Relaxed); + MASK.store(parsed.mask, Ordering::Release); + kernel::pr_info!( + "G17P dvfs parity mask={:#x} hold_us={} ramp_us={} ce_max={:#x}\n", + parsed.mask, parsed.hold_ns / 1000, parsed.ramp_ns / 1000, parsed.ce_max + ); + 0 +} + +unsafe extern "C" fn parity_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let cfg = config(); + let _ = writeln!(out, "mask={:#x} hold_us={} ramp_us={} ce_max={:#x}", + cfg.mask, cfg.hold_ns / 1000, cfg.ramp_ns / 1000, cfg.ce_max); + let _ = writeln!(out, "ce_live={:#x} ce_publishes={} ce_clears={} rechecks={} publish_errors={}", + CE_LIVE.load(Ordering::Relaxed), CE_PUBLISHES.load(Ordering::Relaxed), + CE_CLEARS.load(Ordering::Relaxed), RECHECKS.load(Ordering::Relaxed), + PUBLISH_ERRORS.load(Ordering::Relaxed)); + let _ = writeln!(out, "ce_last_set_ns={} ce_last_clear_ns={} now_ns={}", + CE_LAST_SET_NS.load(Ordering::Relaxed), CE_LAST_CLEAR_NS.load(Ordering::Relaxed), + crate::g18::compat::timing_buffer::now()); + let _ = writeln!(out, "bracket_opens={} kicks_sent={} kicks_gated={} stats_66c_last={:#x} stats_66c_nonzero={}", + BRACKET_OPENS.load(Ordering::Relaxed), KICKS_SENT.load(Ordering::Relaxed), + KICKS_GATED.load(Ordering::Relaxed), STATS_66C_LAST.load(Ordering::Relaxed), + STATS_66C_NONZERO.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} + +static PARITY_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(parity_set), get: Some(parity_get), free: None, +}; +#[link_section = "__param"] +#[used] +static PARITY_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_dvfs_parity" + } else { c"aurora.g17p_dvfs_parity" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PARITY_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/e2e.rs b/drivers/gpu/drm/asahi/g18/compat/e2e.rs new file mode 100644 index 0000000000000..173d2571ee38d --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/e2e.rs @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Opt-in end-to-end timeline ring (v81 diagnostic, default OFF). +//! +//! `g17p_e2e` (root-only sysfs parameter) arms a preallocated ring of 64-byte +//! records. While disarmed, every hook is a single relaxed atomic load and a +//! branch; no production state, ordering, locking or firmware access changes. +//! While armed, `record` reserves one slot with an atomic increment and does +//! eight relaxed stores (the first word last, with Release). No allocation, +//! printk, sleeping lock or wait is ever taken from a hook. +//! +//! Record layout (u64 words): +//! w0: CNTPCT_EL0 at the record point (host 1 GHz on T8140); written last. +//! w1: kind | cpu << 8 | generation << 16 | aux << 32 +//! w2: identity (submission id, slot, ...; kind specific) +//! w3..w7: payload a..e (kind specific; see the KIND_* constants). +//! +//! Commands written to `g17p_e2e`: +//! "arm " start a new generation from slot 0 (mask bit0 base events, +//! bit1 mailbox send/IRQ events, bit2 read-only firmware power +//! word + GPC pstate samples, bit3 device segment reads, +//! bit4 v91 fine submit-path laps, kinds 27..31) +//! "stop" stop recording (slots stay readable) +//! "mark " insert a marker record (only while armed) +//! "seek " set the read cursor +//! Reading returns up to 24 records from the cursor and advances it. +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const OPT_BASE: u32 = 1 << 0; +pub(crate) const OPT_MAILBOX: u32 = 1 << 1; +pub(crate) const OPT_POWER: u32 = 1 << 2; +pub(crate) const OPT_SEGMENTS: u32 = 1 << 3; +/// v91: fine submit-path laps (K_FINE_*). Default off; with the ring armed +/// without this bit, each lap site is one relaxed load and a branch. +pub(crate) const OPT_FINE: u32 = 1 << 4; + +// Kernel event kinds. +/// submit ioctl entry (head = file.rs submit entry). id=submission, aux=queue id, +/// a=in_syncs | out_syncs << 32, b=cmdbuf bytes, c=tgid, d=cmdbuf copied, +/// e=syncs parsed (both CNTPCT). +pub(crate) const K_IOCTL_ENTER: u8 = 1; +/// render command parsed. id=submission, a=command count, +/// b=render_barrier | compute_barrier << 32, c=render wait_through (MAX none), +/// d=compute wait_through (MAX none), e=render sequence. +pub(crate) const K_IOCTL_PARSED: u8 = 2; +/// render job pushed to drm_sched. id=submission, a=dependencies, +/// b=checked (input) dependencies, c=firmware fences, d=render sequence. +pub(crate) const K_SCHED_PUSHED: u8 = 3; +/// submit ioctl exit. id=submission, a=errno (0 ok). +pub(crate) const K_IOCTL_EXIT: u8 = 4; +/// run_job (publish) entry. id=submission, a=input fences, b=firmware fences, +/// c=input fences already signalled at entry. +pub(crate) const K_RUN_JOB: u8 = 5; +/// input fences waited/checked. id=submission. +pub(crate) const K_DEPS_DONE: u8 = 6; +/// physical lease attached. id=submission, a=render slot. +pub(crate) const K_LEASE: u8 = 7; +/// publication lock/prepare times (CNTPCT). id=submission, a=work state +/// acquired, b=first Runtime lock requested, c=first Runtime lock acquired, +/// d=prepare begin (Runtime dropped), e=prepare end. head=Runtime reacquired. +pub(crate) const K_PUB_LOCKS: u8 = 8; +/// publication stage summary. id=submission, aux=slot|retained<<8|hit<<9|ok<<10, +/// a=runtime_lock_ns | state_polls << 32, b=handoff_ns | prepare_ns << 32, +/// c=restage_ns | reachability_ns << 32, d=flist_request_ns | tvb_blocks << 32, +/// e=doorbell CNTPCT. head=submit_translated_render return (Runtime held). +pub(crate) const K_PUB_STAGES: u8 = 9; +/// publication sub-stage nanoseconds sub_ns[0..10] packed pairwise into a..e, +/// sub_ns[10] in aux. id=submission. +pub(crate) const K_PUB_SUB: u8 = 10; +/// publish() return after Runtime release. id=submission, a=ok. +pub(crate) const K_PUB_EXIT: u8 = 11; +/// mailbox send. a=endpoint, b=message, c=RTKit instance. +pub(crate) const K_SEND: u8 = 13; +/// hard-IRQ mailbox receive. a=endpoint | processor << 8, b=message. +pub(crate) const K_IRQ: u8 = 14; +/// event-worker firmware-event service wrapper. a=entry, b=Runtime acquired, +/// c=Runtime released, d=deferred completions signalled. head=after signals. +pub(crate) const K_WORKER: u8 = 15; +/// one render observation iteration. id=slot, a=Runtime acquired, +/// b=observation taken (Runtime released), c=observe() done, +/// d=Runtime reacquired, e=commit done (before release), aux=signalled|ok<<16. +pub(crate) const K_OBS_LOOP: u8 = 16; +/// completion proven under Runtime. id=submission, aux=slot, a=primary +/// notification ingress, b=secondary ingress, c=service-pass entry, +/// d=pass start (GPU 24 MHz), e=pass end (GPU 24 MHz). +pub(crate) const K_OBSERVED: u8 = 17; +/// G17PRenderCompletion::complete. id=submission, a=entry, b=UAPI timestamps +/// written, c=lease dropped, d=VM job dropped, e=hardware fence signalled. +/// head=aggregate member completed. +pub(crate) const K_COMPLETE: u8 = 18; +/// aggregate (output syncobj) fence signalled. id=submission, a=before signal, +/// aux=errno. head=after dma_fence_signal returned. +pub(crate) const K_AGG_SIGNAL: u8 = 19; +/// marker. id=user value. +pub(crate) const K_MARK: u8 = 20; +/// device identity at retirement. id=pair slot << 32 | ordinal, a=TA QID | +/// FRG QID << 32, b=observed TA stamp | FRG stamp << 32, c=expected stamp, +/// d=pass start, e=pass end. +pub(crate) const K_OBS_DEVICE: u8 = 21; +/// descriptor segments (opt bit3). id=pair slot << 32 | ordinal, a..d=TA +/// start/end, FRG start/end (24 MHz), e=read ok. +pub(crate) const K_SEGMENTS: u8 = 22; +/// read-only power samples (opt bit2). id=site (1 doorbell, 2 observed), +/// aux=slot, a=firmware power-up count, b=firmware power state word, +/// c=GPC performance state (MMIO, low 4 bits), d=read error mask. +pub(crate) const K_POWER: u8 = 23; +/// v84 feed-perf actuator change. aux=new pct (0 = withdrawn), id=score, +/// a=previous pct, b=starved passes, c=fed passes (cumulative). +pub(crate) const K_FEED_PERF: u8 = 26; +/// v91 ioctl laps (opt bit4), recorded after the scheduler push. id=submission, +/// aux=validation requests | proof hit << 8 | batch walks << 9 | pending cap +/// << 16. a=queue submit entry, b=output fences split, c=VM guard + parser + +/// pending vector, d=translate begin, e=translate end. +pub(crate) const K_FINE_IOCTL: u8 = 27; +/// v91 ioctl laps, continued. id=submission. a=parse finish + VM job guards, +/// b=aggregate fence + output syncobjs installed, c=timestamp destinations, +/// d=hardware fence + completion + dependency vectors, e=packet built. +pub(crate) const K_FINE_QUEUE: u8 = 28; +/// v91 locked reservation laps inside take_render_restage (Runtime held). +/// id=pair slot, a=entry, b=recovery gate done, c=dependency snapshot done, +/// d=VM handoff done, e=manager reservation done (graph taken). +pub(crate) const K_FINE_RESERVE: u8 = 29; +/// v91 restage_command laps (unlocked). id=pair slot << 32 | ordinal. +/// a=entry, b=render-state/AXFB/status/timestamp encode done, c=PM scene + +/// objects selected, d=TA descriptor + retained fields done, e=3D descriptor, +/// tail and MCache done. head=support record + fence done. +pub(crate) const K_FINE_RESTAGE: u8 = 30; +/// v91 SUB_CONFIG laps (Runtime held). id=pair slot << 32 | ordinal. +/// a=entry, b=3D QoS published, c=3D consistency validated, d=TA QoS +/// published, e=TA consistency validated. head=pair configure done. +pub(crate) const K_FINE_CONFIG: u8 = 31; +/// v94 retained-graph reachability revalidation (opt bit4), one per render +/// publication. id=render descriptor storage IOVA (per pair graph), +/// aux=path | reason << 4 | generation << 8 (path 0 exact, 1 lock-free walk, +/// 2 locked walk, 3 full walk; reason for a full walk 1 no proof, 2 no key, +/// 3 refused, 4 touched, 5 key moved), a=entry, b=app changes walked | kernel +/// changes walked << 32, c=app exec lock held (0 unlocked), d=kernel exec +/// lock held (0 unlocked), e=walks done (all CNTPCT). head=exit. +pub(crate) const K_FINE_REACH: u8 = 32; + +const CAPACITY: usize = 1 << 17; +#[repr(C, align(64))] +struct Slot { w: [AtomicU64; 8] } +static SLOTS: [Slot; CAPACITY] = [const { Slot { w: [const { AtomicU64::new(0) }; 8] } }; CAPACITY]; +static ARMED: AtomicU32 = AtomicU32::new(0); +static GENERATION: AtomicU32 = AtomicU32::new(0); +static NEXT: AtomicUsize = AtomicUsize::new(0); +static BOUND: AtomicUsize = AtomicUsize::new(0); +static CURSOR: AtomicUsize = AtomicUsize::new(0); +static DROPPED: AtomicU64 = AtomicU64::new(0); + +#[inline(always)] +pub(crate) fn armed() -> bool { ARMED.load(Ordering::Relaxed) != 0 } +#[inline(always)] +pub(crate) fn option(bit: u32) -> bool { ARMED.load(Ordering::Relaxed) & bit != 0 } + +/// Host CNTPCT_EL0 (same register the existing timing buffer uses). +#[inline(always)] +pub(crate) fn now() -> u64 { crate::g18::compat::timing_buffer::arch_counter() } + +/// Only reads the counter while armed; otherwise returns 0. +#[inline(always)] +pub(crate) fn stamp() -> u64 { if armed() { now() } else { 0 } } + +/// v91: counter for a fine lap (option bit4), otherwise 0. +#[inline(always)] +pub(crate) fn fine() -> u64 { if option(OPT_FINE) { now() } else { 0 } } + +#[inline(always)] +fn cpu() -> u64 { + // SAFETY: raw_smp_processor_id has no preconditions; a stale value after + // migration is acceptable for a diagnostic label. + u64::from(unsafe { bindings::raw_smp_processor_id() }) & 0xff +} + +pub(crate) fn record_at(head: u64, kind: u8, aux: u32, id: u64, payload: [u64; 5]) { + if !armed() { return; } + let generation = GENERATION.load(Ordering::Relaxed); + let index = NEXT.fetch_add(1, Ordering::Relaxed); + if index >= CAPACITY { DROPPED.fetch_add(1, Ordering::Relaxed); return; } + let slot = &SLOTS[index]; + slot.w[1].store(u64::from(kind) | cpu() << 8 | (u64::from(generation) & 0xffff) << 16 + | u64::from(aux) << 32, Ordering::Relaxed); + slot.w[2].store(id, Ordering::Relaxed); + for (word, value) in slot.w[3..].iter().zip(payload) { word.store(value, Ordering::Relaxed); } + slot.w[0].store(head.max(1), Ordering::Release); +} + +#[inline(always)] +pub(crate) fn record(kind: u8, aux: u32, id: u64, payload: [u64; 5]) { + if !armed() { return; } + record_at(now(), kind, aux, id, payload); +} + +// Read-only power samples (option bit2). Mappings are created on the first +// arm with the bit set and kept until module unload (never torn down while a +// hook could still use them). Same physical words and access method as the +// existing `g17p_diag_knobs` read (memremap WB + dc ivac + load), and the +// same always-on GPC register as `g17p_gpc_sample`. +static POWERUPS_PAGE: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); +static STATE_PAGE: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); +static GPC_REG: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); +const POWERUPS_LINK: u64 = 0xffff_fc00_0012_4ba0; +const STATE_LINK: u64 = 0xffff_fc00_0017_71f8; +const GPC_STATE_PHYS: u64 = 0x4_80e0_1000; +const PAGE: u64 = crate::pgtable::UAT_PGSZ as u64; + +fn map_power_sources() -> bool { + let map_page = |slot: &AtomicPtr, link: u64| -> bool { + if !slot.load(Ordering::Acquire).is_null() { return true; } + let physical = crate::g18::compat::g17p::g17p_gfx_link_to_physical(link); + // SAFETY: same page as the existing bounded diag_knobs probe; WB + // read-only use; retained until module unload. + let ptr = unsafe { bindings::memremap(physical & !(PAGE - 1), PAGE as usize, + bindings::MEMREMAP_WB as _) }; + if ptr.is_null() { return false; } + slot.store(ptr.cast::(), Ordering::Release); + true + }; + let ok = map_page(&POWERUPS_PAGE, POWERUPS_LINK) && map_page(&STATE_PAGE, STATE_LINK); + if GPC_REG.load(Ordering::Acquire).is_null() { + // SAFETY: the always-on 4-byte register already sampled by + // `g17p_gpc_sample`; retained until module unload. + let base = unsafe { bindings::ioremap(GPC_STATE_PHYS as bindings::phys_addr_t, 4) }; + if base.is_null() { return false; } + GPC_REG.store(base.cast::(), Ordering::Release); + } + ok +} + +fn read_firmware_word(page: &AtomicPtr, link: u64) -> Option { + let base = page.load(Ordering::Acquire); + if base.is_null() { return None; } + let physical = crate::g18::compat::g17p::g17p_gfx_link_to_physical(link); + let offset = (physical & (PAGE - 1)) as usize; + // SAFETY: `base` maps the whole containing page for the module lifetime; + // `offset` is 8-aligned (both links are) and below the page size. The + // invalidate touches only AP cachelines of this read-only mapping, + // exactly like LiveFirmwareU64Probe::observe_mask. + unsafe { + let address = base.add(offset); + core::arch::asm!("dsb osh", "dc ivac, {a}", "dsb osh", a = in(reg) address, + options(nostack, preserves_flags)); + Some(core::ptr::read_volatile(address.cast::())) + } +} + +/// Opt-in bit2 sample at a named site. Read-only; never called otherwise. +pub(crate) fn power_sample(site: u64, slot: u8) { + if !option(OPT_POWER) { return; } + let powerups = read_firmware_word(&POWERUPS_PAGE, POWERUPS_LINK); + let state = read_firmware_word(&STATE_PAGE, STATE_LINK); + let gpc = { + let base = GPC_REG.load(Ordering::Acquire); + // SAFETY: live 4-byte MMIO mapping retained until module unload. + if base.is_null() { None } else { Some(u64::from(unsafe { bindings::readl(base.cast()) })) } + }; + let errors = u64::from(powerups.is_none()) | u64::from(state.is_none()) << 1 + | u64::from(gpc.is_none()) << 2; + record(K_POWER, u32::from(slot), site, + [powerups.unwrap_or(0), state.unwrap_or(0), gpc.unwrap_or(0) & 0xf, errors, 0]); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} + +fn bounded_command(value: *const core::ffi::c_char, out: &mut [u8; 32]) -> Option { + let mut len = 0; + while len < out.len() { + // SAFETY: the parameter core supplies a NUL-terminated value; at most + // 32 bytes are examined and never beyond the first NUL. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { return Some(len); } + out[len] = byte; + len += 1; + } + None +} + +unsafe extern "C" fn e2e_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 32]; + let Some(len) = bounded_command(value, &mut command) else { return -22; }; + let Ok(text) = core::str::from_utf8(&command[..len]) else { return -22; }; + let text = text.trim(); + let (verb, argument) = match text.split_once(' ') { + Some((verb, argument)) => (verb, argument.trim()), + None => (text, ""), + }; + let hex = |argument: &str| u64::from_str_radix(argument.strip_prefix("0x").unwrap_or(argument), 16).ok(); + match verb { + "arm" => { + let Some(mask) = hex(if argument.is_empty() { "1" } else { argument }) else { return -22; }; + let mask = (mask as u32) | OPT_BASE; + if mask & OPT_POWER != 0 && !map_power_sources() { return -12; } + ARMED.store(0, Ordering::Relaxed); + GENERATION.fetch_add(1, Ordering::Relaxed); + DROPPED.store(0, Ordering::Relaxed); + CURSOR.store(0, Ordering::Relaxed); + NEXT.store(0, Ordering::Relaxed); + ARMED.store(mask, Ordering::Release); + } + "stop" => { + ARMED.store(0, Ordering::Release); + BOUND.store(NEXT.load(Ordering::Relaxed).min(CAPACITY), Ordering::Relaxed); + } + "mark" => { + let Some(value) = hex(argument) else { return -22; }; + record(K_MARK, 0, value, [0; 5]); + } + "seek" => { + let Some(index) = hex(argument) else { return -22; }; + CURSOR.store((index as usize).min(CAPACITY), Ordering::Relaxed); + } + _ => return -22, + } + 0 +} + +unsafe extern "C" fn e2e_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4095) }; + let mut out = Output { bytes, used: 0 }; + let armed = ARMED.load(Ordering::Acquire); + let bound = if armed != 0 { NEXT.load(Ordering::Relaxed).min(CAPACITY) } else { BOUND.load(Ordering::Relaxed) }; + let generation = GENERATION.load(Ordering::Relaxed) as u64 & 0xffff; + let mut cursor = CURSOR.load(Ordering::Relaxed); + let _ = writeln!(out, "s {:x} {:x} {:x} {:x} {:x} {}", armed, generation, cursor, bound, + DROPPED.load(Ordering::Relaxed), crate::g18::compat::timing_buffer::arch_frequency()); + for _ in 0..24 { + if cursor >= bound { break; } + let slot = &SLOTS[cursor]; + let head = slot.w[0].load(Ordering::Acquire); + let meta = slot.w[1].load(Ordering::Relaxed); + if head == 0 || (meta >> 16) & 0xffff != generation { + // Reserved but not yet published (or a stale late writer). + let _ = writeln!(out, "u {:x}", cursor); + } else { + let _ = write!(out, "r {:x} {:x} {:x}", cursor, head, meta); + for word in &slot.w[2..] { let _ = write!(out, " {:x}", word.load(Ordering::Relaxed)); } + let _ = writeln!(out); + } + cursor += 1; + } + CURSOR.store(cursor, Ordering::Relaxed); + out.used as core::ffi::c_int +} + +static E2E_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(e2e_set), get: Some(e2e_get), free: None, +}; +#[link_section = "__param"] +#[used] +static E2E_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_e2e" + } else { c"aurora.g17p_e2e" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&E2E_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/g17p.rs b/drivers/gpu/drm/asahi/g18/compat/g17p.rs new file mode 100644 index 0000000000000..c1b2663123477 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/g17p.rs @@ -0,0 +1,574 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Module 1134 G17P items read by carried translation and diagnostic code. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. +use core::ops::Range; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +#[cfg(CONFIG_DEV_COREDUMP)] +use core::ptr::NonNull; +#[cfg(CONFIG_DEV_COREDUMP)] +use kernel::{bindings, ffi::c_void}; +use kernel::{addr::PhysicalAddr, prelude::*}; +use crate::pgtable::{UAT_PGMSK, UAT_PGSZ}; +// diagnostic_accounting_clocks names the probe as `pgtable::LiveFirmwareU64Probe`. +use crate::g18::compat::g17p as pgtable; + +// Pinned J700 T8140 GFX A000 scheduler root used by the exact EP type-4 +// thunk. The extracted image has SHA-256 +// 3e2821bb40d3abffe30488a06ede5f523cfa47544d6a0ad7654a08be6b5dc30f. +// Its __DATA vmaddr/size are 0xfffffc000004c000/0x134000, exactly matching +// the J700 gfx-data carveout size. These addresses are diagnostic only. +const G17P_GFX_LINK_DATA_BASE: u64 = 0xffff_fc00_0004_c000; + +const G17P_GFX_LINK_DATA_END: u64 = 0xffff_fc00_0018_0000; + +const G17P_GFX_PHYSICAL_DATA_BASE: u64 = 0x0000_0100_01d7_c000; + +/// Translate a GFX firmware LINK address into the physical address the host +/// can map. Same arithmetic the render status-page probes use. +pub(crate) fn g17p_gfx_link_to_physical(link: u64) -> u64 { + G17P_GFX_PHYSICAL_DATA_BASE + (link - G17P_GFX_LINK_DATA_BASE) +} + +fn g17p_binding_row_address(base: u64, id: u32, stride: u64) -> Option { + if stride == 0 || stride & 7 != 0 { return None; } + let start = base.checked_add(u64::from(id).checked_mul(stride)?)?; + let end = start.checked_add(stride)?; + if start < G17P_GFX_LINK_DATA_BASE || start & 7 != 0 || end > G17P_GFX_LINK_DATA_END { + return None; + } + Some(start) +} + +/// Explicit diagnostic read of retained GFX accounting clocks. No MMIO and no +/// firmware writes. Caller requires a frozen, nonempty G17 timing capture. +/// Sample timestamps can advance independently; frequency/epoch fields must +/// remain stable throughout the read or the result is rejected. +pub(crate) fn diagnostic_accounting_clocks() -> Result<[u64; 8]> { + const OFFSETS: [u64; 8] = [ + 0x124ad8, 0x124af8, 0x124b18, 0x124b38, + 0x6e598, 0x6e5b0, 0x6e5d0, 0x179320, + ]; + let read = |index: usize| -> Result { + let link = 0xffff_fc00_0000_0000u64.checked_add(OFFSETS[index]).ok_or(EOVERFLOW)?; + let start = g17p_binding_row_address(link, 0, 8).ok_or(EINVAL)?; + pgtable::LiveFirmwareU64Probe::new(g17p_gfx_link_to_physical(start)) + .and_then(|probe| probe.observe_mask(u64::MAX, 1)) + .map(|sample| sample.first) + }; + let mut values = [0; 8]; + for index in 0..8 { values[index] = read(index)?; } + for index in 4..8 { + if read(index)? != values[index] { return Err(EAGAIN); } + } + Ok(values) +} + +/// Platform-derived SKSM scratch offsets within the mapped accelerator +/// register window. +/// +/// Apple's host loads these three values from accelerator object fields +/// `+0x10730`, `+0x10734`, and `+0x10738`. Those are object offsets, not SGX +/// MMIO offsets. This type carries the loaded values and no physical base. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17SksmScratchGeometry { + pub(crate) write0_base: u32, + pub(crate) write1_base: u32, + pub(crate) stride: u32, +} + +/// The two offsets used by the fixed G17P scratch FIFO slot. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17SksmScratchOffsets { + pub(crate) write0: u32, + pub(crate) write1: u32, +} + +impl G17SksmScratchGeometry { + /// Apply the exact host equation for fixed FIFO slot 9. + /// + /// The slot is independent of the SKSM queue ID carried in each write. + pub(crate) const fn fifo_offsets( + self, + ) -> core::result::Result { + const FIFO_SLOT: u32 = 9; + + let fifo_offset = match self.stride.checked_mul(FIFO_SLOT) { + Some(offset) => offset, + None => return Err(ConstructionError::SksmScratchOffsetOverflow), + }; + let write0 = match self.write0_base.checked_add(fifo_offset) { + Some(offset) => offset, + None => return Err(ConstructionError::SksmScratchOffsetOverflow), + }; + let write1 = match self.write1_base.checked_add(fifo_offset) { + Some(offset) => offset, + None => return Err(ConstructionError::SksmScratchOffsetOverflow), + }; + + Ok(G17SksmScratchOffsets { write0, write1 }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ConstructionError { + WrongChip, + WrongDeclaredIdentity, + WrongHardwareIdentity, + WrongUatGeometry, + UndecodableIdentity, + WrongDecodedIdentity, + WrongSubmissionBackend, + NullUatOwner, + SplitUatOwner, + WrongRoleTopology, + WrongRootPlacement, + WrongSharedClusterSize, + WrongSksmScratchGeometry, + WrongSksmQueueGeometry, + SksmScratchOffsetOverflow, +} + +pub(crate) const T8140_TIMESTAMP_ALIAS_RANGE: Range = + crate::g18::compat::g17p::CLIENT_LOW_VA_START..0x71_0000_0000; + +/// Two GiB boundary inside the existing four GiB runtime reservation. In +/// addition to descriptor arrays, each independently executing CL graph owns +/// a 23-MiB private-memory pool. Keep room for every advertised queue without +/// reducing concurrency; the remaining 2 GiB hold per-VM private aliases. +pub(crate) const CLIENT_LOW_VA_START: u64 = 0x70_8000_0000; + +pub(crate) const QID_COUNT: usize = 128; + +/// Device-wide QID gauges for `g17p_queue_slots`, indexed by [`Engine`]. +/// Kept by every table mutation and undone by `Drop`, so replacing the table +/// at processor stop leaves only the new table's leases counted. +pub(crate) static LEASED: [AtomicU32; 3] = [AtomicU32::new(0), AtomicU32::new(0), AtomicU32::new(0)]; + +pub(crate) static PUBLISHED: [AtomicU32; 3] = [AtomicU32::new(0), AtomicU32::new(0), AtomicU32::new(0)]; + +/// Most QIDs leased at once since load. +pub(crate) static HIGH_WATER: AtomicU32 = AtomicU32::new(0); + +/// `acquire` calls that found no free QID (the ENOSPC of queue creation and +/// of a new render pair). +pub(crate) static EXHAUSTED: AtomicU64 = AtomicU64::new(0); + +/// New compute graphs refused to keep [`COMPUTE_RENDER_RESERVE`] QIDs free. +pub(crate) static RESERVE_REFUSED: AtomicU64 = AtomicU64::new(0); + +/// QIDs a new compute graph may never take: kept for render pairs. +/// +/// A published QID is only returned at processor stop, and a quarantined or +/// exit-killed compute graph is never reused, so compute faults consume the +/// table one QID at a time. Without a reserve the last free QID goes to a +/// compute graph, and from then on every client that needs a new render +/// pair -- every new desktop client -- fails its first render with ENOSPC. +/// With it, exhaustion fails the next compute queue creation instead. 16 is +/// eight render pairs. +pub(crate) const COMPUTE_RENDER_RESERVE: usize = 16; + +/// Total QIDs currently leased, all engines. +pub(crate) fn leased_total() -> u32 { + LEASED.iter().map(|count| count.load(Ordering::Relaxed)).sum() +} + +/// bit0: publish a work-tied CLPC deadline control effort at RegionC+0xe4. +pub(crate) const PARITY_CE: u32 = 1 << 0; + +/// bit1: native submit-in-progress bracket order (+0x7c, then +0x80). +pub(crate) const PARITY_BRACKET: u32 = 1 << 1; + +/// bit2: native-gated 0x87 note inside the bracket (requires bit1). +pub(crate) const PARITY_KICK: u32 = 1 << 2; + +pub(crate) const PARITY_MASK_ALL: u32 = PARITY_CE | PARITY_BRACKET | PARITY_KICK; + +/// 1.0 in the kext's 16.16 effort encoding; larger values are rejected by +/// `updateCLPCDeadlineInterfaceCE(u32)` (`cmp w1, #0x10, lsl #12; b.ls`). +pub(crate) const CE_ONE: u32 = 0x1_0000; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Config { + pub(crate) mask: u32, + pub(crate) hold_ns: u64, + pub(crate) ramp_ns: u64, + pub(crate) ce_max: u32, +} + +pub(crate) const DEFAULT_HOLD_US: u64 = 16_000; + +pub(crate) const MIN_HOLD_US: u64 = 100; + +pub(crate) const MAX_HOLD_US: u64 = 1_000_000; + +pub(crate) const MAX_RAMP_US: u64 = 5_000_000; + +pub(crate) const DEFAULT_CONFIG: Config = Config { + // v84: native CLPC deadline-effort parity (bit0) is the production + // default (v83 hardware-qualified). Bracket/kick (bits 1/2) stay opt-in. + mask: PARITY_CE, + hold_ns: DEFAULT_HOLD_US * 1000, + ramp_ns: 0, + ce_max: CE_ONE, +}; + +fn parse_number(text: &str) -> Option { + match text.strip_prefix("0x") { + Some(hex) => u64::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + } +} + +/// Parse `MASK [HOLD_US [RAMP_US [CE_MAX]]]`. Omitted fields keep defaults. +pub(crate) fn parse(text: &str) -> Option { + let mut fields = text.split_ascii_whitespace(); + let mask = parse_number(fields.next()?)?; + if mask > u64::from(PARITY_MASK_ALL) { + return None; + } + let mut config = DEFAULT_CONFIG; + config.mask = mask as u32; + if config.mask & PARITY_KICK != 0 && config.mask & PARITY_BRACKET == 0 { + // Native only sends the note from inside updateSubmitInProgress(true). + return None; + } + if let Some(hold) = fields.next() { + let hold = parse_number(hold)?; + if !(MIN_HOLD_US..=MAX_HOLD_US).contains(&hold) { + return None; + } + config.hold_ns = hold * 1000; + } + if let Some(ramp) = fields.next() { + let ramp = parse_number(ramp)?; + if ramp > MAX_RAMP_US { + return None; + } + config.ramp_ns = ramp * 1000; + } + if let Some(max) = fields.next() { + let max = parse_number(max)?; + if max == 0 || max > u64::from(CE_ONE) { + return None; + } + config.ce_max = max as u32; + } + if fields.next().is_some() { + return None; + } + Some(config) +} + +#[cfg(CONFIG_DEV_COREDUMP)] +struct PhysPageMapping { + ptr: NonNull, +} + +/// A bounded read-only probe for one qword in live firmware-owned physical +/// memory. +/// +/// Construction maps the containing page once. [`Self::observe_mask`] then +/// invalidates only the sampled AP cacheline before each load, so a short +/// firmware transition can be observed without remapping, writing, or holding +/// any firmware lock. +#[cfg(CONFIG_DEV_COREDUMP)] +pub(crate) struct LiveFirmwareU64Probe { + page: PhysPageMapping, + offset: usize, +} + +/// Result of one bounded live-mask observation. +#[cfg(CONFIG_DEV_COREDUMP)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct LiveFirmwareMaskObservation { + pub(crate) first: u64, + pub(crate) last: u64, + pub(crate) observed_or: u64, + pub(crate) mask_seen: bool, + pub(crate) polls: usize, +} + +#[cfg(CONFIG_DEV_COREDUMP)] +impl LiveFirmwareU64Probe { + const MAX_POLLS: usize = 256; + + pub(crate) fn new(physical: PhysicalAddr) -> Result { + if physical & 7 != 0 { + return Err(EINVAL); + } + let page_mask = UAT_PGMSK as u64; + let page = PhysPageMapping::new(physical & !page_mask)?; + let offset = (physical & page_mask) as usize; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > UAT_PGSZ { + return Err(EINVAL); + } + Ok(Self { page, offset }) + } + + /// Observe one mask for at most `polls` cache-invalidated qword loads. + /// DIAGNOSTIC. Set bits in the probed qword; returns (before, after). + pub(crate) fn set_bits(&self, bits: u64) -> Result<(u64, u64)> { + self.page.set_bits_u64(self.offset, bits) + } + + /// DIAGNOSTIC (v93r). `set_bits` on a freshly invalidated line, then + /// clean it to the point of coherency so the coprocessor sees the bits. + pub(crate) fn set_bits_visible(&self, bits: u64) -> Result<(u64, u64)> { + self.page.invalidate_live_range(self.offset, size_of::())?; + let result = self.page.set_bits_u64(self.offset, bits)?; + self.page.clean_written_range(self.offset, size_of::())?; + Ok(result) + } + + /// DIAGNOSTIC. Read `out.len()` consecutive qwords from the probed qword + /// onwards, within its page, after invalidating their AP cachelines. Not + /// an atomic snapshot of the run. + pub(crate) fn read_run(&self, out: &mut [u64]) -> Result { + let size = out.len().checked_mul(size_of::()).ok_or(EOVERFLOW)?; + self.page.invalidate_live_range(self.offset, size)?; + for (index, value) in out.iter_mut().enumerate() { + *value = self.page.read_u64(self.offset + index * size_of::())?; + } + Ok(()) + } + + pub(crate) fn observe_mask( + &self, + mask: u64, + polls: usize, + ) -> Result { + if mask == 0 || polls == 0 || polls > Self::MAX_POLLS { + return Err(EINVAL); + } + + let mut first = 0; + let mut last = 0; + let mut observed_or = 0; + let mut mask_seen = false; + let mut completed = 0; + for index in 0..polls { + self.page.invalidate_live_range(self.offset, size_of::())?; + let value = self.page.read_u64(self.offset)?; + if index == 0 { + first = value; + } + last = value; + observed_or |= value; + completed = index + 1; + if value & mask != 0 { + mask_seen = true; + break; + } + core::hint::spin_loop(); + } + + Ok(LiveFirmwareMaskObservation { + first, + last, + observed_or, + mask_seen, + polls: completed, + }) + } +} + +#[cfg(CONFIG_DEV_COREDUMP)] +impl PhysPageMapping { + fn new(phys: PhysicalAddr) -> Result { + if phys & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + let ptr = unsafe { + // SAFETY: the caller supplies a page-aligned physical address + // obtained from a live firmware UAT PTE. The mapping is used only + // for bounded reads and is released by Drop. + bindings::memremap( + phys, + UAT_PGSZ, + bindings::MEMREMAP_WB as _, + ) + }; + let ptr = NonNull::new(ptr).ok_or_else(|| { + pr_err!("UAT crash walker: memremap failed for {:#x}\n", phys); + ENOMEM + })?; + Ok(Self { ptr }) + } + + fn invalidate_live_range(&self, offset: usize, size: usize) -> Result { + let end = offset.checked_add(size).ok_or(EOVERFLOW)?; + if size == 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + let ctr_el0: u64; + unsafe { + // SAFETY: CTR_EL0 is readable at EL1 and reports DminLine as + // log2(words). The resulting line size controls only bounded cache + // maintenance on this read-only mapping. + core::arch::asm!( + "mrs {ctr}, ctr_el0", + ctr = out(reg) ctr_el0, + options(nomem, nostack, preserves_flags), + ); + } + let line_size = 4usize + .checked_shl(((ctr_el0 >> 16) & 0xf) as u32) + .ok_or(EOVERFLOW)?; + if !line_size.is_power_of_two() { + return Err(EINVAL); + } + let start_address = self.ptr.as_ptr() as usize + offset; + let end_address = self.ptr.as_ptr() as usize + end; + let mut address = start_address & !(line_size - 1); + unsafe { + // SAFETY: the page remains mapped for the whole loop. `dc ivac` + // invalidates AP cachelines only; this diagnostic never writes the + // firmware-owned page. The barriers order invalidation before the + // atomic loads in `copy_atomic()`. + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + while address < end_address { + core::arch::asm!( + "dc ivac, {address}", + address = in(reg) address, + options(nostack, preserves_flags), + ); + address = address.checked_add(line_size).ok_or(EOVERFLOW)?; + } + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + } + Ok(()) + } + + fn read_u64(&self, offset: usize) -> Result { + let end = offset.checked_add(8).ok_or(EOVERFLOW)?; + if offset & 7 != 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + let pointer = unsafe { + // SAFETY: offset is qword-aligned and bounded to the mapped page. + self.ptr.as_ptr().cast::().add(offset).cast::() + }; + Ok(unsafe { &*pointer }.load(Ordering::Acquire)) + } + + /// DIAGNOSTIC. Set bits in one firmware-DRAM qword. + /// + /// The b1 image gates KTrace emission on class bytes at 0x17780c/0x17780d: + /// bit 0 is the only one this driver has ever had set, which is why the + /// halt-decision records (0x909 with the scan's own inputs, 0x113 at the + /// thread-4 wake, 0x105, 0x110) have never appeared in a capture. Arming + /// them is a read-modify-write of a debug-enable byte the firmware only + /// reads; it steers no scheduling decision. + fn set_bits_u64(&self, offset: usize, bits: u64) -> Result<(u64, u64)> { + let end = offset.checked_add(8).ok_or(EOVERFLOW)?; + if offset & 7 != 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + let pointer = unsafe { + // SAFETY: offset is qword-aligned and bounded to the mapped page, + // exactly as read_u64 checks above. + self.ptr.as_ptr().cast::().add(offset).cast::() + }; + let cell = unsafe { &*pointer }; + let before = cell.load(Ordering::Acquire); + cell.store(before | bits, Ordering::Release); + let after = cell.load(Ordering::Acquire); + Ok((before, after)) + } + + /// Clean AP cachelines covering a range this mapping has just written. + /// + /// The mapping is write-back, so a store can sit in an AP cacheline the + /// coprocessor's fetch path will not see. `invalidate_live_range()` above + /// is the read-side mirror of this; keep the two symmetric. + fn clean_written_range(&self, offset: usize, size: usize) -> Result { + let end = offset.checked_add(size).ok_or(EOVERFLOW)?; + if size == 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + let ctr_el0: u64; + unsafe { + // SAFETY: CTR_EL0 is readable at EL1 and reports DminLine as + // log2(words); the value bounds cache maintenance only. + core::arch::asm!( + "mrs {ctr}, ctr_el0", + ctr = out(reg) ctr_el0, + options(nomem, nostack, preserves_flags), + ); + } + let line_size = 4usize + .checked_shl(((ctr_el0 >> 16) & 0xf) as u32) + .ok_or(EOVERFLOW)?; + if !line_size.is_power_of_two() { + return Err(EINVAL); + } + let start_address = self.ptr.as_ptr() as usize + offset; + let end_address = self.ptr.as_ptr() as usize + end; + let mut address = start_address & !(line_size - 1); + unsafe { + // SAFETY: the page stays mapped for the whole loop. `dc cvac` + // cleans AP cachelines to the point of coherency; the barriers + // order the preceding stores before the clean and the clean before + // anything the coprocessor is subsequently told to do. + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + while address < end_address { + core::arch::asm!( + "dc cvac, {address}", + address = in(reg) address, + options(nostack, preserves_flags), + ); + address = address.checked_add(line_size).ok_or(EOVERFLOW)?; + } + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + } + Ok(()) + } + + fn write_atomic(&self, offset: usize, data: &[u8]) -> Result { + let end = offset.checked_add(data.len()).ok_or(EOVERFLOW)?; + if (offset | data.len()) & 7 != 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + for (index, source) in data.chunks_exact(size_of::()).enumerate() { + let at = offset + index * size_of::(); + let pointer = unsafe { + // SAFETY: `at` is qword-aligned and bounded to the mapped page + // by the check above. + self.ptr.as_ptr().cast::().add(at).cast::() + }; + let mut qword = [0u8; size_of::()]; + qword.copy_from_slice(source); + unsafe { &*pointer }.store(u64::from_le_bytes(qword), Ordering::Release); + } + self.clean_written_range(offset, data.len()) + } + + fn copy_atomic(&self, offset: usize, out: &mut [u8]) -> Result { + let end = offset.checked_add(out.len()).ok_or(EOVERFLOW)?; + if (offset | out.len()) & 7 != 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + for (index, destination) in out.chunks_exact_mut(size_of::()).enumerate() { + destination.copy_from_slice( + &self + .read_u64(offset + index * size_of::())? + .to_le_bytes(), + ); + } + Ok(()) + } +} + +#[cfg(CONFIG_DEV_COREDUMP)] +impl Drop for PhysPageMapping { + fn drop(&mut self) { + unsafe { + // SAFETY: ptr was returned by memremap and has not been unmapped. + bindings::memunmap(self.ptr.as_ptr()); + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/compat/gem_stats.rs b/drivers/gpu/drm/asahi/g18/compat/gem_stats.rs new file mode 100644 index 0000000000000..8f81b2d02a575 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/gem_stats.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Live GEM object accounting (`gem_stats`, root-only, read-only sysfs). +//! +//! Every shmem object this driver creates is counted from creation until its +//! last reference drops, split by user and kernel ownership. Once a GEM object +//! is mapped into a GPU VM its pages stay pinned (and unevictable) until the +//! object is freed, so `user_bytes + kernel_bytes` is an upper bound on the +//! GPU share of /proc/meminfo `Unevictable`, and a value that keeps growing +//! after clients exit identifies a reference leak in the driver. The +//! `*_created` / `*_freed` totals show allocation churn. +use core::fmt::Write; +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +static USER_OBJECTS: AtomicU64 = AtomicU64::new(0); +static USER_BYTES: AtomicU64 = AtomicU64::new(0); +static USER_CREATED: AtomicU64 = AtomicU64::new(0); +static KERNEL_OBJECTS: AtomicU64 = AtomicU64::new(0); +static KERNEL_BYTES: AtomicU64 = AtomicU64::new(0); +static KERNEL_CREATED: AtomicU64 = AtomicU64::new(0); + +fn counters(kernel: bool) -> (&'static AtomicU64, &'static AtomicU64, &'static AtomicU64) { + if kernel { + (&KERNEL_OBJECTS, &KERNEL_BYTES, &KERNEL_CREATED) + } else { + (&USER_OBJECTS, &USER_BYTES, &USER_CREATED) + } +} + +/// Count one new GEM object. +pub(crate) fn note_new(kernel: bool, size: usize) { + let (objects, bytes, created) = counters(kernel); + objects.fetch_add(1, Ordering::Relaxed); + bytes.fetch_add(size as u64, Ordering::Relaxed); + created.fetch_add(1, Ordering::Relaxed); +} + +/// Count one freed GEM object. +pub(crate) fn note_free(kernel: bool, size: usize) { + let (objects, bytes, _) = counters(kernel); + objects.fetch_sub(1, Ordering::Relaxed); + bytes.fetch_sub(size as u64, Ordering::Relaxed); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn stats_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let r = |c: &AtomicU64| c.load(Ordering::Relaxed); + let uc = r(&USER_CREATED); + let kc = r(&KERNEL_CREATED); + let (uo, ko) = (r(&USER_OBJECTS), r(&KERNEL_OBJECTS)); + let _ = writeln!(out, + "user_objects={} user_bytes={} user_created={} user_freed={} \ + kernel_objects={} kernel_bytes={} kernel_created={} kernel_freed={}", + uo, r(&USER_BYTES), uc, uc.wrapping_sub(uo), + ko, r(&KERNEL_BYTES), kc, kc.wrapping_sub(ko)); + out.used as core::ffi::c_int +} + +unsafe extern "C" fn stats_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + -1 // EPERM: read-only counters +} + +static STATS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(stats_set), get: Some(stats_get), free: None, +}; +#[link_section = "__param"] +#[used] +static STATS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"gem_stats" + } else { c"aurora.gem_stats" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&STATS_OPS), perm: 0o400, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/host_path.rs b/drivers/gpu/drm/asahi/g18/compat/host_path.rs new file mode 100644 index 0000000000000..2403560feae6c --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/host_path.rs @@ -0,0 +1,780 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! v84 host refill-path controls. +//! +//! `g17p_host_path` (root-only sysfs, runtime-writable bitmask): +//! bit0 EARLY_PUBLISH: a render whose submit ioctl has no unsignalled input, +//! prefix or firmware dependency, whose queue has no job still inside +//! the DRM scheduler, and for which a physical lease and a preparation +//! epoch are available without waiting, is published (doorbell rung) +//! from the submit ioctl. The DRM scheduler job is still created and +//! pushed; its run_job returns the already-published hardware fence, so +//! timeouts, credits, teardown and cancellation keep their existing +//! owners. Anything else takes the unchanged scheduler path. +//! bit1 FEED_PERF: GPU-feed CPU performance request (see below). +//! bit2 EARLY_SHALLOW (v87): bit0's early publication, taken only while the +//! queue has fewer than `g17p_early_depth` (default 2) of its own renders +//! outstanding, counting this one. The count is the queue's existing +//! pruned `render_frontier` (unsignalled hardware fences). A deeper +//! queue takes exactly the v84 scheduler path (bit0 off). +//! bit3 STATE_WORD (v87): publication and event-service recovery gates read +//! only the firmware recovery *state* word (the only field they test) +//! instead of the three-word handshake snapshot. Same predicate. +//! bit4 SETTLE_ONCE (v87): within one Runtime hold of the publish +//! reservation (warm checks + restage take), test the open-recovery +//! gate once instead of once per helper. The final gate immediately +//! before the doorbell (submit_translated_render) is unchanged. +//! bit5 RING_CACHE (v87): firmware-event ring drains skip re-reading the two +//! host-authored main-config ring pointers once they have been checked +//! equal to their derivation; revalidated every RING_REVALIDATE drains. +//! bit6 WORKER_TAIL (v87): event-worker bookkeeping after the drain (DVFS +//! parity, teardown service, back-pressure, recovery/poll/requeue +//! decisions) runs under one Runtime acquisition instead of six. +//! bit7 OBS_SKIP (v87): the render observation loop is skipped when, under +//! the drain's own Runtime hold, no pending slot has both type-1 masks. +//! bit8 VM_STATUS_MIRROR (v87): GET_PARAMS group 3 registration of a +//! userspace word that mirrors the sticky VM status (see g17_status). +//! bit9 VALIDATE_PROOF (v91): the submit ioctl's UAPI address validation +//! (one GPUVM exec lock per range before) records the ranges during +//! translation and proves them with ONE exec lock, or with none when +//! the queue's previous proof covers them at the VM's unchanged +//! mapping-validation epoch. Any failed proof reruns the unchanged +//! per-range translation, so errors are exactly those of v90. +//! bit10 PENDING_EXACT (v91): the ioctl's pending-command vector grows to +//! the commands actually present instead of preallocating 64 x ~1.2 KB +//! (a kmalloc_large page allocation and free on every submission). +//! bit11 WORD_STORES (v91): descriptor/record encoders store naturally +//! aligned 16/32/64-bit fields with one store instead of byte stores +//! (identical bytes; fewer write-combined stores). +//! bit12 PCORE_NUDGE (v92, DIAGNOSTIC, default off): while the feed-perf +//! request is applied, a submit ioctl that runs on a lower-capacity CPU +//! moves its own thread onto the highest-capacity cluster once, by a +//! transient affinity restriction that is restored immediately; later +//! wakeups then prefer that (idle) previous CPU. It transiently rewrites +//! a user-visible attribute, so it exists only to measure the value of +//! big-core placement (what CLPC's P-core recommendation and, on Linux, +//! a UCLAMP_TASK kernel would provide) before any kernel change. +//! bit13 FW_DEPS (v93, default off until lab-qualified): an input syncobj +//! whose fence (after unwrapping timeline chain links) consists only of +//! already-signalled fences and aggregates of this same queue's recent +//! fully enqueued ioctls is not waited on the host. Instead the +//! command's logical render/compute prefixes are raised to cover those +//! ioctls, which the existing prefix machinery turns into firmware +//! barrier tokens (render->render at publish, compute<->render through +//! publication fences plus tokens, compute->compute through the QID's +//! implicit parent), and each matched aggregate is status-checked at +//! publication. Anything else (another queue, another VM or client, a +//! foreign or errored fence, a fence array, an ioctl that did not fully +//! enqueue, a chain deeper than FW_DEP_LINKS_MAX) keeps the unchanged +//! host wait for that whole sync. Same VM by construction, so a faulted +//! predecessor's existing fail-outstanding/quarantine paths release the +//! firmware waiter. +//! bit14 AGG_FIRST (v93, default off until lab-qualified): a completion +//! signals the ioctl's user-visible aggregate fence before its per- +//! command hardware fence (whose inline callbacks are drm_sched job-done +//! bookkeeping). Timestamp writes, sticky status, lease release and +//! VM-job release still precede both: userspace reads timestamps and +//! recycles BOs/VAs as soon as the aggregate signals. +//! bit15 EARLY_DEPS (v93, default off until lab-qualified): early +//! publication also admits a render whose host dependencies are all +//! already signalled and whose remaining dependencies are firmware +//! status checks (FW_DEPS aggregates, prefix hardware fences); it still +//! declines any unsignalled host dependency, a scheduler backlog or an +//! unavailable lease, so the ioctl never waits for a dependency. +//! bit16 WAIT_UTIL_HINT: tag accepted feeder work for 8 ms while the +//! existing starvation detector requests CPU performance. A blocking +//! syncobj wait uses a scheduler capacity hint of 512 only until it +//! returns. User clamps, cgroup limits and CPU affinity remain intact. +//! Default on; clearing this bit is the same-kernel A/B control. +//! +//! v92 per-cluster feed levels: `g17p_feed_perf_pct_p` / `g17p_feed_perf_pct_e` +//! (0 = inherit `g17p_feed_perf_pct`) set the floor of the highest-capacity +//! (P) and the other (E) cpufreq policies separately while the GPU is starved. +//! Native IOGPU reports every command buffer to CLPC (IOPerfControl workSubmit +//! at IOGPUBlockFence::init, workBegin, workEnd with GPU data), and CLPC raises +//! the submitting thread group's CPU performance only while it gates the GPU. +//! The submitting thread and the ioctl run on E cores ~90% of the time and the +//! completion worker on CPU0 (P), so the two clusters bound different parts +//! of the host loop and need not share one level. +//! +//! GPU-feed CPU performance request. Native analogue: IOGPU reports GPU work +//! to CLPC (`IOGPUFenceMachine::notifyCLPCIOPerfControl`) so CPU performance +//! follows a GPU-bound feed loop. On Linux a dma-fence waiter's sleep is not +//! iowait, so schedutil sees ~10% utilisation and keeps both clusters at their +//! lowest OPP while the GPU idles waiting for that same CPU work (measured v81: +//! 91-99.7% lowest OPP, GPU idle 47-52% at depth 1/2). +//! +//! Detector (hot path, atomics only): every retired render reports its GPU pass +//! envelope. Decayed sums track GPU busy time and GPU idle time in short gaps +//! (0 < gap < MAX_REFILL: the GPU waited for the host refill path). A long gap +//! (paced client or idle desktop) resets the loop streak. The request is armed +//! while a closed feed loop keeps the GPU idle >= 20% of the time; it lingers +//! LINGER after the last such pass and is then withdrawn. GPU-bound +//! (overlapped) streams never arm it and no request exists at idle. +//! Actuator (process context, one work item): a cpufreq freq_qos MIN request on +//! each policy at `g17p_feed_perf_pct` percent of that policy's maximum. +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const EARLY_PUBLISH: u32 = 1 << 0; +pub(crate) const FEED_PERF: u32 = 1 << 1; +pub(crate) const EARLY_SHALLOW: u32 = 1 << 2; +pub(crate) const STATE_WORD: u32 = 1 << 3; +pub(crate) const SETTLE_ONCE: u32 = 1 << 4; +pub(crate) const RING_CACHE: u32 = 1 << 5; +pub(crate) const WORKER_TAIL: u32 = 1 << 6; +pub(crate) const OBS_SKIP: u32 = 1 << 7; +pub(crate) const VM_STATUS_MIRROR: u32 = 1 << 8; +pub(crate) const VALIDATE_PROOF: u32 = 1 << 9; +pub(crate) const PENDING_EXACT: u32 = 1 << 10; +pub(crate) const WORD_STORES: u32 = 1 << 11; +pub(crate) const PCORE_NUDGE: u32 = 1 << 12; +pub(crate) const FW_DEPS: u32 = 1 << 13; +pub(crate) const AGG_FIRST: u32 = 1 << 14; +pub(crate) const EARLY_DEPS: u32 = 1 << 15; +pub(crate) const WAIT_UTIL_HINT: u32 = 1 << 16; +/// Read only producer cursors when no completion/retirement decision is made. +pub(crate) const PRODUCER_READS: u32 = 1 << 17; +/// Count the retained graph as a PB owner, preserving its binding token. +pub(crate) const PB_OWNER_REF: u32 = 1 << 18; +/// Native pool sharing. Captured at logical-context creation, never changes a +/// live pool. Clearing this bit is a same-kernel per-pair allocation control. +pub(crate) const SHARED_FLIST: u32 = 1 << 19; +const ALL_BITS: u32 = (1 << 20) - 1; + +/// Production default for this candidate. v84-final had FEED_PERF only; v87 +/// added bits 2-8 and v91 bits 9-11. `g17p_host_path=0x1fe` is exactly the +/// v90 behaviour and `=2` the v84-final behaviour, for same-boot A/B. +/// v93 bits 13-15 stay off. The scoped wait hint adds bit16 to the default; +/// `0xffe` disables only that hint for same-kernel comparison. `0xeffe` +/// enables the older three probes without the scoped hint. +// Producer-only reads extend 0x10ffe to 0x30ffe; clearing bit17 restores +// the original full snapshots for an otherwise identical comparison. +const DEFAULT_MASK: u32 = FEED_PERF | EARLY_SHALLOW | STATE_WORD | SETTLE_ONCE + | RING_CACHE | WORKER_TAIL | OBS_SKIP | VM_STATUS_MIRROR + | VALIDATE_PROOF | PENDING_EXACT | WORD_STORES | WAIT_UTIL_HINT | PRODUCER_READS | PB_OWNER_REF | SHARED_FLIST; +static MASK: AtomicU32 = AtomicU32::new(DEFAULT_MASK); +static FEED_PCT: AtomicU32 = AtomicU32::new(50); +/// v92 per-cluster overrides of FEED_PCT (0 = inherit). +static FEED_PCT_P: AtomicU32 = AtomicU32::new(0); +static FEED_PCT_E: AtomicU32 = AtomicU32::new(0); +/// EARLY_SHALLOW depth: early publication while fewer than this many renders +/// of the queue are outstanding, counting the one being submitted. +static EARLY_DEPTH: AtomicU32 = AtomicU32::new(2); +const EARLY_DEPTH_MAX: u32 = 64; +/// RING_CACHE: re-read and re-check the main-config ring pointers on at +/// least every this many drains of one role. +pub(crate) const RING_REVALIDATE: u32 = 256; + +#[inline(always)] +pub(crate) fn enabled(bit: u32) -> bool { MASK.load(Ordering::Relaxed) & bit != 0 } + +/// v87 counters (relaxed; observability only, printed by `g17p_host_path`). +pub(crate) static EARLY_TAKEN: AtomicU64 = AtomicU64::new(0); +pub(crate) static SETTLE_SKIPPED: AtomicU64 = AtomicU64::new(0); +pub(crate) static RING_CACHE_HITS: AtomicU64 = AtomicU64::new(0); +pub(crate) static TAIL_MERGED: AtomicU64 = AtomicU64::new(0); +pub(crate) static OBS_SKIPPED: AtomicU64 = AtomicU64::new(0); +pub(crate) static MIRRORS: AtomicU64 = AtomicU64::new(0); +pub(crate) static PROOF_HITS: AtomicU64 = AtomicU64::new(0); +pub(crate) static PROOF_BATCHES: AtomicU64 = AtomicU64::new(0); +pub(crate) static PROOF_FALLBACKS: AtomicU64 = AtomicU64::new(0); +pub(crate) static NUDGES: AtomicU64 = AtomicU64::new(0); +pub(crate) static NUDGE_FAILS: AtomicU64 = AtomicU64::new(0); +/// v93 FW_DEPS input syncs: converted to firmware prefixes, kept as host +/// waits, or dropped because already signalled without error. +pub(crate) static FW_DEP_CONVERTED: AtomicU64 = AtomicU64::new(0); +pub(crate) static FW_DEP_HOST: AtomicU64 = AtomicU64::new(0); +pub(crate) static FW_DEP_SATISFIED: AtomicU64 = AtomicU64::new(0); +/// v93 EARLY_DEPS: early publications that carried dependencies. +pub(crate) static EARLY_WITH_DEPS: AtomicU64 = AtomicU64::new(0); +/// v93 AGG_FIRST completions (aggregate member completed before hw fence). +pub(crate) static AGG_FIRST_TAKEN: AtomicU64 = AtomicU64::new(0); + +#[inline(always)] +pub(crate) fn count(counter: &AtomicU64) { counter.fetch_add(1, Ordering::Relaxed); } + +/// EARLY_SHALLOW gate. `outstanding_before` is the queue's unsignalled +/// renders excluding this one; admit while the total stays within `depth`. +#[inline(always)] +pub(crate) const fn early_shallow_admits(outstanding_before: usize, depth: u32) -> bool { + outstanding_before < depth as usize +} + +/// Whether a render submission may attempt early publication: bit0 alone +/// (v84, any depth) or bit2 within the depth. Dependencies always decline. +/// With v93 EARLY_DEPS the caller passes whether any host dependency is +/// still unsignalled instead of whether one exists. +#[inline(always)] +pub(crate) fn early_publish_allowed(outstanding_before: usize, has_dependencies: bool) -> bool { + if has_dependencies { return false; } + let mask = MASK.load(Ordering::Relaxed); + if mask & EARLY_PUBLISH != 0 { return true; } + if mask & EARLY_SHALLOW == 0 { return false; } + // No counter here: a declined (deep) submission must stay the v84 path. + early_shallow_admits(outstanding_before, EARLY_DEPTH.load(Ordering::Relaxed)) +} + +/// v93 FW_DEPS: longest timeline chain walked per input sync; ring size. +pub(crate) const FW_DEP_LINKS_MAX: u32 = 16; +pub(crate) const FW_DEP_RECENT: usize = 8; + +/// One unwrapped input-fence leaf, as seen by `fw_dep_step`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum FwDepLeaf { + /// Signalled without error: satisfies nothing further. + Signalled, + /// Signalled with an error: the host path must consume it. + Failed, + /// Unsignalled; identity is the fence address (compared, never used). + Pending(usize), + /// Nested container, fence array, missing leaf or over-long chain. + Unsupported, +} + +/// A recent fully enqueued ioctl of this queue: aggregate identity and the +/// inclusive logical sequence ranges of its commands (0 = none of that kind). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct FwDepRecent { + pub(crate) identity: usize, + pub(crate) render_last: u64, + pub(crate) compute_last: u64, +} + +/// Accumulated firmware expression of one input sync. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct FwDepResolution { + /// Highest render / compute logical sequence that must retire first. + pub(crate) render_through: u64, + pub(crate) compute_through: u64, + /// Bit i: recent entry i supplied a pending leaf (status-checked later). + pub(crate) used: u32, + /// Leaves seen (Signalled included); 0 means an empty walk. + pub(crate) leaves: u32, +} + +/// Fold one leaf into `resolution`. False means the sync is not provably +/// expressible in firmware and must keep its original host wait. +pub(crate) fn fw_dep_step( + resolution: &mut FwDepResolution, leaf: FwDepLeaf, recent: &[Option], +) -> bool { + resolution.leaves += 1; + let identity = match leaf { + FwDepLeaf::Signalled => return true, + FwDepLeaf::Failed | FwDepLeaf::Unsupported => return false, + FwDepLeaf::Pending(identity) => identity, + }; + if identity == 0 { return false; } + let Some(index) = recent.iter().position(|entry| + entry.is_some_and(|entry| entry.identity == identity)) else { return false; }; + let Some(entry) = recent[index] else { return false; }; + resolution.render_through = resolution.render_through.max(entry.render_last); + resolution.compute_through = resolution.compute_through.max(entry.compute_last); + resolution.used |= 1u32 << index; + true +} + +/// Raise an optional logical prefix (`None` = independent) to cover `through`. +#[inline(always)] +pub(crate) const fn fw_dep_raise(prefix: Option, through: u64) -> Option { + if through == 0 { return prefix; } + match prefix { + Some(limit) if limit >= through => Some(limit), + _ => Some(through), + } +} + +/// RING_CACHE decision for one role. `entry` is (cursor, ring, drains left) +/// of the last successful check; `expected` is this drain's derivation. +/// Returns true when the two pointer reads may be skipped (and consumes one +/// drain of the budget); false means read and check them, then `ring_checked`. +#[inline(always)] +pub(crate) fn ring_cache_hit(entry: &mut (u64, u64, u32), expected: (u64, u64), enabled: bool) -> bool { + if !enabled || entry.2 == 0 || entry.0 != expected.0 || entry.1 != expected.1 { + return false; + } + entry.2 -= 1; + true +} + +/// Record a successful full pointer check for `ring_cache_hit`. +#[inline(always)] +pub(crate) fn ring_checked(entry: &mut (u64, u64, u32), checked: (u64, u64)) { + *entry = (checked.0, checked.1, RING_REVALIDATE); +} + +// --------------------------------------------------------------------------- +// Feed-perf detector state. GPU pass stamps are 24 MHz firmware ticks. +const MAX_REFILL_TICKS: u64 = 24_000 * 2; // 2 ms: longer gaps are paced/idle +const STREAK_ON: u32 = 8; +const STREAK_MAX: u32 = 64; +const LINGER_NS: u64 = 8_000_000; +const POLL_MS: i64 = 2; + +static LAST_END: AtomicU64 = AtomicU64::new(0); +/// Exponentially decayed (1/8 per pass) GPU idle and busy ticks over recent +/// passes. Idle counts only short gaps: the GPU waited for the host. +static IDLE_EWMA: AtomicU64 = AtomicU64::new(0); +static BUSY_EWMA: AtomicU64 = AtomicU64::new(0); +/// Consecutive passes without a long (paced) gap. +static STREAK: AtomicU32 = AtomicU32::new(0); +static UNTIL_NS: AtomicU64 = AtomicU64::new(0); +/// A worker owns the actuator (running or queued). +static ACTIVE: AtomicBool = AtomicBool::new(false); +/// The detector asked for a worker; the event worker queues it. +static PENDING: AtomicBool = AtomicBool::new(false); +static UNBINDING: AtomicBool = AtomicBool::new(false); +static STARVED: AtomicU64 = AtomicU64::new(0); +static FED: AtomicU64 = AtomicU64::new(0); +static BOOSTS: AtomicU64 = AtomicU64::new(0); +static BOOSTED_NS: AtomicU64 = AtomicU64::new(0); + +fn now_ns() -> u64 { + ::ktime_get() as u64 +} + +fn decay_add(sum: &AtomicU64, add: u64) -> u64 { + let mut next = 0; + let _ = sum.fetch_update(Ordering::AcqRel, Ordering::Acquire, |v| { + next = v - v / 8 + add; + Some(next) + }); + next +} + +/// Hot path: one retired render's GPU pass envelope [start, end] (24 MHz). +/// Atomics only. Concurrent callers may interleave; the estimate tolerates it. +pub(crate) fn note_render_pass(start: u64, end: u64) { + if !enabled(FEED_PERF) || end <= start { return; } + let previous = LAST_END.fetch_max(end, Ordering::AcqRel); + let (idle, busy, paced) = if previous == 0 { + (0, end - start, true) + } else if start >= previous { + let gap = start - previous; + if gap < MAX_REFILL_TICKS { (gap, end - start, false) } else { (0, end - start, true) } + } else if end > previous { + (0, end - previous, false) // overlapped: only the new busy tail counts + } else { + (0, 0, false) + }; + let idle_sum = decay_add(&IDLE_EWMA, idle); + let busy_sum = decay_add(&BUSY_EWMA, busy); + let streak = if paced { + STREAK.store(0, Ordering::Release); + 0 + } else { + STREAK.fetch_update(Ordering::AcqRel, Ordering::Acquire, + |s| Some((s + 1).min(STREAK_MAX))).unwrap_or(0) + 1 + }; + // Host-starved feed loop: a closed loop (no paced gaps) in which the GPU + // is idle at least 20% of the time waiting for the host. + if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { + STARVED.fetch_add(1, Ordering::Relaxed); + UNTIL_NS.store(now_ns() + LINGER_NS, Ordering::Release); + if !ACTIVE.swap(true, Ordering::AcqRel) { + PENDING.store(true, Ordering::Release); + } + } else { + FED.fetch_add(1, Ordering::Relaxed); + } +} + +/// Called by the firmware-event worker after each pass (process context). +pub(crate) fn take_pending() -> bool { PENDING.swap(false, Ordering::AcqRel) } + +// --------------------------------------------------------------------------- +// Actuator. Only the single feed-perf work item (and unbind after that item is +// disabled) touches REQS, so accesses are serialised by construction. +const MAX_POLICIES: usize = 8; +struct Reqs { + policy: [AtomicPtr; MAX_POLICIES], + max_khz: [AtomicU32; MAX_POLICIES], + req: core::cell::UnsafeCell<[bindings::freq_qos_request; MAX_POLICIES]>, + count: AtomicU32, + registered: AtomicBool, + applied_pct: AtomicU32, + /// v92: this policy has the highest maximum frequency (the P cluster). + big: [AtomicBool; MAX_POLICIES], + /// v92: value last written to each request (skips redundant updates). + applied_khz: [AtomicU32; MAX_POLICIES], + /// v92: CPUs (bit per CPU id, ids < 64) of the highest-capacity policy. + big_cpus: AtomicU64, +} +// SAFETY: `req` is only accessed by the serialised actuator (see above). +unsafe impl Sync for Reqs {} +static REQS: Reqs = Reqs { + policy: [const { AtomicPtr::new(core::ptr::null_mut()) }; MAX_POLICIES], + max_khz: [const { AtomicU32::new(0) }; MAX_POLICIES], + // SAFETY: freq_qos_request is plain C data; all-zero is its unregistered state. + req: core::cell::UnsafeCell::new(unsafe { core::mem::zeroed() }), + count: AtomicU32::new(0), + registered: AtomicBool::new(false), + applied_pct: AtomicU32::new(0), + big: [const { AtomicBool::new(false) }; MAX_POLICIES], + applied_khz: [const { AtomicU32::new(0) }; MAX_POLICIES], + big_cpus: AtomicU64::new(0), +}; + +/// v92: the floor (kHz, 0 = withdrawn) one policy gets for request level +/// `base` percent. A big policy uses the P override, others the E override; +/// an override of 0 inherits `base`. Level 0 always withdraws. +pub(crate) const fn cluster_floor_khz(max_khz: u32, big: bool, base: u32, pct_p: u32, pct_e: u32) -> u32 { + if base == 0 { return 0; } + let over = if big { pct_p } else { pct_e }; + let pct = if over != 0 { over } else { base }; + let pct = if pct > 100 { 100 } else { pct }; + ((max_khz as u64 * pct as u64) / 100) as u32 +} + +/// v92: index of the policy with the highest maximum frequency (first on a +/// tie), or None when there are fewer than two distinct maxima (no big/little +/// distinction to make). +pub(crate) fn big_policy(max_khz: &[u32]) -> Option { + let top = *max_khz.iter().max()?; + if max_khz.iter().all(|&k| k == top) { return None; } + max_khz.iter().position(|&k| k == top) +} + +fn register() -> bool { + if REQS.registered.load(Ordering::Acquire) { return true; } + // SAFETY: plain read of a kernel global set at boot. + let cpus = unsafe { bindings::nr_cpu_ids }; + let mut count = 0usize; + let mut policy_cpus = [0u64; MAX_POLICIES]; + for cpu in 0..cpus { + // SAFETY: cpufreq_cpu_get tolerates any cpu index and takes a reference. + let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; + if policy.is_null() { continue; } + let known = (0..count).position(|i| REQS.policy[i].load(Ordering::Relaxed) == policy); + if let (Some(index), true) = (known, cpu < 64) { policy_cpus[index] |= 1u64 << cpu; } + if known.is_none() && count < MAX_POLICIES { + // SAFETY: `policy` is referenced; the request slot is unregistered + // and lives in static storage for the module lifetime. + let ret = unsafe { + let slot = (*REQS.req.get()).as_mut_ptr().add(count); + bindings::freq_qos_add_request(&mut (*policy).constraints, slot, + bindings::freq_qos_req_type_FREQ_QOS_MIN, + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32) + }; + if ret >= 0 { + REQS.policy[count].store(policy, Ordering::Relaxed); + // SAFETY: referenced policy; cpuinfo is immutable after init. + REQS.max_khz[count].store(unsafe { (*policy).cpuinfo.max_freq }, Ordering::Relaxed); + REQS.applied_khz[count].store(bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32, Ordering::Relaxed); + if cpu < 64 { policy_cpus[count] |= 1u64 << cpu; } + count += 1; + } + } + // SAFETY: balances cpufreq_cpu_get above. The request keeps its own + // pointer to the policy's constraints, as cpufreq cooling devices do. + unsafe { bindings::cpufreq_cpu_put(policy) }; + } + let mut maxima = [0u32; MAX_POLICIES]; + for (i, slot) in maxima.iter_mut().enumerate().take(count) { + *slot = REQS.max_khz[i].load(Ordering::Relaxed); + } + let big = big_policy(&maxima[..count]); + for i in 0..count { REQS.big[i].store(Some(i) == big, Ordering::Relaxed); } + // CPU ids >= 64 are not represented; the nudge then stays disabled. + REQS.big_cpus.store(match big { Some(i) if cpus <= 64 => policy_cpus[i], _ => 0 }, Ordering::Relaxed); + REQS.count.store(count as u32, Ordering::Release); + REQS.registered.store(count != 0, Ordering::Release); + count != 0 +} + +fn apply(pct: u32) { + if pct != 0 && !register() { return; } + if !REQS.registered.load(Ordering::Acquire) { return; } + let (pct_p, pct_e) = (FEED_PCT_P.load(Ordering::Relaxed), FEED_PCT_E.load(Ordering::Relaxed)); + for i in 0..REQS.count.load(Ordering::Acquire) as usize { + let max = REQS.max_khz[i].load(Ordering::Relaxed); + let floor = cluster_floor_khz(max, REQS.big[i].load(Ordering::Relaxed), pct, pct_p, pct_e); + let value = if floor == 0 { bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 } else { floor }; + // v92: per-cluster levels may change while applied; write on change only. + if REQS.applied_khz[i].swap(value, Ordering::Relaxed) == value { continue; } + // SAFETY: slot `i` was registered by `register` and is only touched here. + unsafe { bindings::freq_qos_update_request((*REQS.req.get()).as_mut_ptr().add(i), value as i32) }; + } + let before = REQS.applied_pct.swap(pct, Ordering::Relaxed); + if before == pct { return; } + let now = now_ns(); + if before == 0 && pct != 0 { + BOOSTS.fetch_add(1, Ordering::Relaxed); + BOOST_SINCE.store(now, Ordering::Relaxed); + } else if before != 0 && pct == 0 { + BOOSTED_NS.fetch_add(now.saturating_sub(BOOST_SINCE.load(Ordering::Relaxed)), Ordering::Relaxed); + } + crate::g18::compat::e2e::record(crate::g18::compat::e2e::K_FEED_PERF, pct, STREAK.load(Ordering::Relaxed) as u64, + [before as u64, STARVED.load(Ordering::Relaxed), FED.load(Ordering::Relaxed), 0, 0]); +} +static BOOST_SINCE: AtomicU64 = AtomicU64::new(0); + +/// v92 PCORE_NUDGE (diagnostic, opt-in): see the module documentation. +/// Called at the start of the submit ioctl (process context, sleepable, no +/// driver lock held). Only while the feed request is applied, only from a CPU +/// outside the big cluster, and only when the thread may run there at all. +pub(crate) fn nudge_feeding_thread() { + if !enabled(PCORE_NUDGE) || REQS.applied_pct.load(Ordering::Relaxed) == 0 { return; } + let big = REQS.big_cpus.load(Ordering::Relaxed); + // SAFETY: raw_smp_processor_id has no preconditions; a stale value only + // makes this diagnostic skip or nudge once more. + let cpu = unsafe { bindings::raw_smp_processor_id() } as u32; + if big == 0 || cpu >= 64 || big & (1u64 << cpu) != 0 { return; } + let task = kernel::task::Task::current_raw(); + // SAFETY: `task` is the current task, alive for this call. Its effective + // mask changes only through set_cpus_allowed_ptr; a concurrent + // sched_setaffinity() from another thread of this process during these two + // calls could be overwritten by the restore (documented diagnostic limit). + let saved = unsafe { core::ptr::read(core::ptr::addr_of!((*task).cpus_mask)) }; + let mut target = saved; + target.bits[0] &= big as usize; + for word in target.bits.iter_mut().skip(1) { *word = 0; } + if target.bits[0] == 0 { return; } + // SAFETY: valid current task and masks; both calls may sleep (ioctl context). + let moved = unsafe { bindings::set_cpus_allowed_ptr(task, &target) }; + if moved != 0 { count(&NUDGE_FAILS); return; } + // SAFETY: as above; restores the exact effective mask read before. + let restored = unsafe { bindings::set_cpus_allowed_ptr(task, &saved) }; + if restored != 0 { count(&NUDGE_FAILS); } else { count(&NUDGES); } +} + +/// Capacity hint for a client actively replenishing a CPU-starved GPU. +/// A per-file expiry and a scoped syncobj wait keep this out of idle/non-GPU +/// work. No task attribute, affinity or saved userspace clamp is rewritten. +pub(crate) fn feeder_wait_util_min() -> u32 { + if !cfg!(CONFIG_UCLAMP_TASK) { return 0; } + if enabled(WAIT_UTIL_HINT) && wanted() + && REQS.applied_pct.load(Ordering::Relaxed) != 0 { 512 } else { 0 } +} + +fn wanted() -> bool { + enabled(FEED_PERF) && !UNBINDING.load(Ordering::Acquire) + && now_ns() < UNTIL_NS.load(Ordering::Acquire) +} + +/// Body of the feed-perf work item. Holds the request while wanted, polling +/// the linger deadline; withdraws it and exits otherwise. +pub(crate) fn run_actuator() { + loop { + if wanted() { + apply(FEED_PCT.load(Ordering::Relaxed).clamp(1, 100)); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(POLL_MS)); + continue; + } + apply(0); + ACTIVE.store(false, Ordering::SeqCst); + // Close the race with a detector that saw ACTIVE still set. + if wanted() && !ACTIVE.swap(true, Ordering::AcqRel) { continue; } + return; + } +} + +/// Unbind, before the work item is disabled: makes a running actuator exit. +pub(crate) fn begin_shutdown() { UNBINDING.store(true, Ordering::Release); } + +/// Unbind: after the work item is disabled, withdraw and remove requests. +pub(crate) fn shutdown() { + UNBINDING.store(true, Ordering::Release); + if !REQS.registered.load(Ordering::Acquire) { return; } + for i in 0..REQS.count.load(Ordering::Acquire) as usize { + // SAFETY: registered slot; the actuator work item is disabled. + unsafe { bindings::freq_qos_remove_request((*REQS.req.get()).as_mut_ptr().add(i)) }; + REQS.policy[i].store(core::ptr::null_mut(), Ordering::Relaxed); + } + REQS.registered.store(false, Ordering::Release); +} + +// --------------------------------------------------------------------------- +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} + +fn parse_u32(value: *const core::ffi::c_char) -> Option { + if value.is_null() { return None; } + let mut command = [0u8; 16]; + let mut len = 0; + while len < command.len() { + // SAFETY: NUL-terminated parameter string; at most 16 bytes examined. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return None; } + let text = core::str::from_utf8(&command[..len]).ok()?.trim(); + match text.strip_prefix("0x") { + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + } +} + +unsafe extern "C" fn mask_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(mask) = parse_u32(value) else { return -22; }; + if mask & !ALL_BITS != 0 { return -22; } + MASK.store(mask, Ordering::Release); + if mask & FEED_PERF == 0 { UNTIL_NS.store(0, Ordering::Release); STREAK.store(0, Ordering::Release); } + 0 +} +unsafe extern "C" fn mask_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1000) }; + let mut out = Output { bytes, used: 0 }; + let _ = writeln!(out, "{:#x} pct={} applied={} streak={} idle={} busy={} starved={} fed={} boosts={} boosted_ns={} active={} policies={} depth={} early={} settle_skip={} ring_hits={} tail={} obs_skip={} mirrors={} proof_hits={} proof_batches={} proof_fallbacks={} pct_p={} pct_e={} big_cpus={:#x} applied_khz={}/{} nudges={} nudge_fails={} fwdeps={}/{}/{} early_deps={} agg_first={}", + MASK.load(Ordering::Relaxed), FEED_PCT.load(Ordering::Relaxed), REQS.applied_pct.load(Ordering::Relaxed), + STREAK.load(Ordering::Relaxed), IDLE_EWMA.load(Ordering::Relaxed), BUSY_EWMA.load(Ordering::Relaxed), + STARVED.load(Ordering::Relaxed), FED.load(Ordering::Relaxed), + BOOSTS.load(Ordering::Relaxed), BOOSTED_NS.load(Ordering::Relaxed), ACTIVE.load(Ordering::Relaxed), + REQS.count.load(Ordering::Relaxed), EARLY_DEPTH.load(Ordering::Relaxed), + EARLY_TAKEN.load(Ordering::Relaxed), + SETTLE_SKIPPED.load(Ordering::Relaxed), RING_CACHE_HITS.load(Ordering::Relaxed), + TAIL_MERGED.load(Ordering::Relaxed), OBS_SKIPPED.load(Ordering::Relaxed), + MIRRORS.load(Ordering::Relaxed), PROOF_HITS.load(Ordering::Relaxed), + PROOF_BATCHES.load(Ordering::Relaxed), PROOF_FALLBACKS.load(Ordering::Relaxed), + FEED_PCT_P.load(Ordering::Relaxed), FEED_PCT_E.load(Ordering::Relaxed), + REQS.big_cpus.load(Ordering::Relaxed), REQS.applied_khz[0].load(Ordering::Relaxed), + REQS.applied_khz[1].load(Ordering::Relaxed), NUDGES.load(Ordering::Relaxed), + NUDGE_FAILS.load(Ordering::Relaxed), FW_DEP_CONVERTED.load(Ordering::Relaxed), + FW_DEP_HOST.load(Ordering::Relaxed), FW_DEP_SATISFIED.load(Ordering::Relaxed), + EARLY_WITH_DEPS.load(Ordering::Relaxed), AGG_FIRST_TAKEN.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} +unsafe extern "C" fn pct_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(pct) = parse_u32(value) else { return -22; }; + if pct == 0 || pct > 100 { return -22; } + FEED_PCT.store(pct, Ordering::Release); + 0 +} +unsafe extern "C" fn pct_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 64) }; + let mut out = Output { bytes, used: 0 }; + let _ = writeln!(out, "{}", FEED_PCT.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} +static MASK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(mask_set), get: Some(mask_get), free: None, +}; +static PCT_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(pct_set), get: Some(pct_get), free: None, +}; +unsafe extern "C" fn depth_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(depth) = parse_u32(value) else { return -22; }; + if depth == 0 || depth > EARLY_DEPTH_MAX { return -22; } + EARLY_DEPTH.store(depth, Ordering::Release); + 0 +} +unsafe extern "C" fn depth_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 64) }; + let mut out = Output { bytes, used: 0 }; + let _ = writeln!(out, "{}", EARLY_DEPTH.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} +static DEPTH_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(depth_set), get: Some(depth_get), free: None, +}; +unsafe extern "C" fn pct_p_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(pct) = parse_u32(value) else { return -22; }; + if pct > 100 { return -22; } + FEED_PCT_P.store(pct, Ordering::Release); + 0 +} +unsafe extern "C" fn pct_e_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(pct) = parse_u32(value) else { return -22; }; + if pct > 100 { return -22; } + FEED_PCT_E.store(pct, Ordering::Release); + 0 +} +fn pct_get_from(counter: &AtomicU32, buf: *mut core::ffi::c_char) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 64) }; + let mut out = Output { bytes, used: 0 }; + let _ = writeln!(out, "{}", counter.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} +unsafe extern "C" fn pct_p_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + pct_get_from(&FEED_PCT_P, buf) +} +unsafe extern "C" fn pct_e_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + pct_get_from(&FEED_PCT_E, buf) +} +static PCT_P_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(pct_p_set), get: Some(pct_p_get), free: None, +}; +static PCT_E_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(pct_e_set), get: Some(pct_e_get), free: None, +}; +#[link_section = "__param"] +#[used] +static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_host_path" + } else { c"aurora.g17p_host_path" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&MASK_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +#[link_section = "__param"] +#[used] +static PCT_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_feed_perf_pct" + } else { c"aurora.g17p_feed_perf_pct" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PCT_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +#[link_section = "__param"] +#[used] +static DEPTH_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_early_depth" + } else { c"aurora.g17p_early_depth" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&DEPTH_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +#[link_section = "__param"] +#[used] +static PCT_P_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_feed_perf_pct_p" + } else { c"aurora.g17p_feed_perf_pct_p" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PCT_P_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +#[link_section = "__param"] +#[used] +static PCT_E_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_feed_perf_pct_e" + } else { c"aurora.g17p_feed_perf_pct_e" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PCT_E_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/mem_path.rs b/drivers/gpu/drm/asahi/g18/compat/mem_path.rs new file mode 100644 index 0000000000000..e658033088d7c --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/mem_path.rs @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! v94 memory-path controls and stall counters (`g17p_mem_path`). +//! +//! Root-only sysfs, runtime-writable bitmask; default 0 is exactly v93f. +//! bit0 ASYNC_CLOSED_DRAIN: GEM objects whose handles closed while the VM +//! had accepted jobs are parked (their mappings and backing retained) +//! until the VM's job count reaches zero. v93f then unmapped and freed +//! all of them inline in the last job's `T8140VmJobGuard` drop, which +//! runs inside that job's completion before its output fence signals, +//! and it held the VM's `draining` gate for the whole drain, stalling +//! every new submission (`retain_t8140_job`) and every new map +//! (`wait_for_user_map_admission`). With this bit, when the parked work +//! is only closed objects on a healthy VM (not quarantined or closed, no +//! deferred unmaps or close ranges), the list captured at that idle +//! instant is released on the unbound workqueue without the gate. Jobs +//! accepted later cannot name a closed handle, and a reused VA has +//! already been remapped by GPUVM, so no accepted job can reach what the +//! worker unmaps. Quarantine and VM close keep the gated path. +//! bit1 MEM_STATS: relaxed counters and maxima for the paths above, GEM +//! creation, VM_BIND phases and admission waits (printed on read). +//! bit2 COMPUTE_TS_POOL: compute completion-timestamp pages are recycled +//! per queue instead of allocating, zeroing, GPU-mapping and later +//! unmapping a fresh GEM page for every compute command. +//! bit3 REACH_LOCKLESS: the retained-graph reachability revalidation that +//! every render publication runs (`first_unreachable_cached`) walks +//! the app and kernel VM change histories without taking either +//! GPUVM exec lock (a drm_exec transaction with a 16 KiB kvmalloc and +//! the VM reservation ww_mutex). Writers record the history lock-free +//! while the bit is set; a refused lock-free walk (writer active, +//! history lost, entry not yet recorded) takes the v93f locked walk, +//! which, with this bit, also serves a publication that found a writer +//! in progress instead of a full page-table walk. +//! MEM_STATS additionally counts VM mapping changes per class and the +//! reachability paths taken (exact / lock-free walk / locked walk / full). +//! Boot-only `g17p_gem_huge` (asahi.rs) is reported here too. +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const ASYNC_CLOSED_DRAIN: u32 = 1 << 0; +pub(crate) const MEM_STATS: u32 = 1 << 1; +pub(crate) const COMPUTE_TS_POOL: u32 = 1 << 2; +pub(crate) const REACH_LOCKLESS: u32 = 1 << 3; +const ALL_BITS: u32 = (1 << 4) - 1; + +static MASK: AtomicU32 = AtomicU32::new(0); +/// Set at probe when the huge GEM mount was created. +pub(crate) static HUGE_MOUNT: AtomicU32 = AtomicU32::new(0); + +#[inline(always)] +pub(crate) fn enabled(bit: u32) -> bool { MASK.load(Ordering::Relaxed) & bit != 0 } + +/// CNTPCT (1 GHz on Neo) when MEM_STATS is set, else 0. +#[inline(always)] +pub(crate) fn stamp() -> u64 { + if enabled(MEM_STATS) { crate::g18::compat::timing_buffer::arch_counter() } else { 0 } +} + +/// One counted event with a duration: count, total and maximum. +pub(crate) struct Stat { count: AtomicU64, total: AtomicU64, max: AtomicU64 } +impl Stat { + pub(crate) const fn new() -> Self { + Self { count: AtomicU64::new(0), total: AtomicU64::new(0), max: AtomicU64::new(0) } + } + /// Record `now - start` when `start` came from `stamp()` (0 = disabled). + #[inline(always)] + pub(crate) fn since(&self, start: u64) { + if start == 0 { return; } + self.add(crate::g18::compat::timing_buffer::arch_counter().saturating_sub(start)); + } + pub(crate) fn add(&self, ticks: u64) { + self.count.fetch_add(1, Ordering::Relaxed); + self.total.fetch_add(ticks, Ordering::Relaxed); + self.max.fetch_max(ticks, Ordering::Relaxed); + } + fn reset(&self) { + self.count.store(0, Ordering::Relaxed); + self.total.store(0, Ordering::Relaxed); + self.max.store(0, Ordering::Relaxed); + } + fn print(&self, out: &mut Output<'_>, name: &str) { + let _ = write!(out, " {}={}/{}/{}", name, self.count.load(Ordering::Relaxed), + self.total.load(Ordering::Relaxed), self.max.load(Ordering::Relaxed)); + } +} + +/// Inline (v93f) and asynchronous closed-object drains, per drain. +pub(crate) static DRAIN_INLINE: Stat = Stat::new(); +pub(crate) static DRAIN_ASYNC: Stat = Stat::new(); +/// Objects released by either drain. +pub(crate) static DRAIN_OBJECTS: AtomicU64 = AtomicU64::new(0); +/// Objects parked at close because jobs were active. +pub(crate) static PARKED: AtomicU64 = AtomicU64::new(0); +/// Waits for the VM `draining` gate: new submissions and new maps. +pub(crate) static RETAIN_WAIT: Stat = Stat::new(); +pub(crate) static ADMISSION_WAIT: Stat = Stat::new(); +/// VM_BIND ioctl: whole call, validation+backing (pages) phase, commit phase. +pub(crate) static BIND_CALL: Stat = Stat::new(); +pub(crate) static BIND_PREPARE: Stat = Stat::new(); +pub(crate) static BIND_COMMIT: Stat = Stat::new(); +pub(crate) static BIND_OPS: AtomicU64 = AtomicU64::new(0); +pub(crate) static BIND_BYTES: AtomicU64 = AtomicU64::new(0); +/// GEM_CREATE ioctl. +pub(crate) static GEM_CREATE: Stat = Stat::new(); +pub(crate) static GEM_BYTES: AtomicU64 = AtomicU64::new(0); +/// COMPUTE_TS_POOL: pages reused, and freshly allocated. +pub(crate) static TS_REUSED: AtomicU64 = AtomicU64::new(0); +pub(crate) static TS_ALLOCATED: AtomicU64 = AtomicU64::new(0); + +/// Retained-graph reachability (per render publication): exact epoch match, +/// lock-free history walk, locked history walk, full page-table walk. +pub(crate) static REACH_EXACT: Stat = Stat::new(); +pub(crate) static REACH_LOCKLESS_WALK: Stat = Stat::new(); +pub(crate) static REACH_LOCKED_WALK: Stat = Stat::new(); +pub(crate) static REACH_FULL: Stat = Stat::new(); +/// History entries examined by either walk. +pub(crate) static REACH_WALKED: AtomicU64 = AtomicU64::new(0); +/// Why a full walk ran: no cached proof for the generation, a writer was in +/// progress (no key), the history walk refused, a change touched the graph. +pub(crate) static REACH_FULL_NO_PROOF: AtomicU64 = AtomicU64::new(0); +pub(crate) static REACH_FULL_NO_KEY: AtomicU64 = AtomicU64::new(0); +pub(crate) static REACH_FULL_REFUSED: AtomicU64 = AtomicU64::new(0); +pub(crate) static REACH_FULL_TOUCHED: AtomicU64 = AtomicU64::new(0); +/// Mapping-validation epoch bumps: kernel VM; user VM inside the kernel's +/// timestamp-alias window (compute completion pages); user VM elsewhere. +pub(crate) static VM_CHG_KERNEL: AtomicU64 = AtomicU64::new(0); +pub(crate) static VM_CHG_TS_ALIAS: AtomicU64 = AtomicU64::new(0); +pub(crate) static VM_CHG_USER: AtomicU64 = AtomicU64::new(0); + +/// Called from `VmInner::note_mapping_validation_change` when MEM_STATS is set. +pub(crate) fn note_vm_change(kernel: bool, start: u64) { + let counter = if kernel { + &VM_CHG_KERNEL + } else if crate::g18::compat::g17p::T8140_TIMESTAMP_ALIAS_RANGE.contains(&start) { + &VM_CHG_TS_ALIAS + } else { + &VM_CHG_USER + }; + counter.fetch_add(1, Ordering::Relaxed); +} + +#[inline(always)] +pub(crate) fn count(counter: &AtomicU64, value: u64) { + if enabled(MEM_STATS) { counter.fetch_add(value, Ordering::Relaxed); } +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} + +fn parse<'a>(value: *const core::ffi::c_char) -> Option<&'a str> { + if value.is_null() { return None; } + // SAFETY: the parameter core supplies a NUL-terminated string. + let text = unsafe { core::ffi::CStr::from_ptr(value) }.to_str().ok()?; + Some(text.trim()) +} + +unsafe extern "C" fn mask_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(text) = parse(value) else { return -22; }; + if text == "reset" { + for stat in [&DRAIN_INLINE, &DRAIN_ASYNC, &RETAIN_WAIT, &ADMISSION_WAIT, &BIND_CALL, + &BIND_PREPARE, &BIND_COMMIT, &GEM_CREATE, &REACH_EXACT, &REACH_LOCKLESS_WALK, + &REACH_LOCKED_WALK, &REACH_FULL] { + stat.reset(); + } + for counter in [&DRAIN_OBJECTS, &PARKED, &BIND_OPS, &BIND_BYTES, &GEM_BYTES, &TS_REUSED, + &TS_ALLOCATED, &REACH_WALKED, &REACH_FULL_NO_PROOF, &REACH_FULL_NO_KEY, + &REACH_FULL_REFUSED, &REACH_FULL_TOUCHED, &VM_CHG_KERNEL, &VM_CHG_TS_ALIAS, + &VM_CHG_USER] { + counter.store(0, Ordering::Relaxed); + } + return 0; + } + let parsed = match text.strip_prefix("0x") { + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + }; + let Some(mask) = parsed else { return -22; }; + if mask & !ALL_BITS != 0 { return -22; } + MASK.store(mask, Ordering::Release); + 0 +} + +unsafe extern "C" fn mask_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes, used: 0 }; + let _ = write!(out, "{:#x} huge_mount={} parked={} drained_objects={} bind_ops={} bind_bytes={} gem_bytes={} ts_reused={} ts_allocated={} (stats: count/total_ns/max_ns)", + MASK.load(Ordering::Relaxed), HUGE_MOUNT.load(Ordering::Relaxed), + PARKED.load(Ordering::Relaxed), DRAIN_OBJECTS.load(Ordering::Relaxed), + BIND_OPS.load(Ordering::Relaxed), BIND_BYTES.load(Ordering::Relaxed), + GEM_BYTES.load(Ordering::Relaxed), TS_REUSED.load(Ordering::Relaxed), + TS_ALLOCATED.load(Ordering::Relaxed)); + DRAIN_INLINE.print(&mut out, "drain_inline"); + DRAIN_ASYNC.print(&mut out, "drain_async"); + RETAIN_WAIT.print(&mut out, "retain_wait"); + ADMISSION_WAIT.print(&mut out, "admission_wait"); + BIND_CALL.print(&mut out, "bind_call"); + BIND_PREPARE.print(&mut out, "bind_prepare"); + BIND_COMMIT.print(&mut out, "bind_commit"); + GEM_CREATE.print(&mut out, "gem_create"); + let _ = write!(out, " vm_chg_kernel={} vm_chg_ts_alias={} vm_chg_user={} reach_walked={} full_no_proof={} full_no_key={} full_refused={} full_touched={}", + VM_CHG_KERNEL.load(Ordering::Relaxed), VM_CHG_TS_ALIAS.load(Ordering::Relaxed), + VM_CHG_USER.load(Ordering::Relaxed), REACH_WALKED.load(Ordering::Relaxed), + REACH_FULL_NO_PROOF.load(Ordering::Relaxed), REACH_FULL_NO_KEY.load(Ordering::Relaxed), + REACH_FULL_REFUSED.load(Ordering::Relaxed), REACH_FULL_TOUCHED.load(Ordering::Relaxed)); + REACH_EXACT.print(&mut out, "reach_exact"); + REACH_LOCKLESS_WALK.print(&mut out, "reach_lockless"); + REACH_LOCKED_WALK.print(&mut out, "reach_locked"); + REACH_FULL.print(&mut out, "reach_full"); + let _ = writeln!(out); + out.used as core::ffi::c_int +} + +static MASK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(mask_set), get: Some(mask_get), free: None, +}; +#[link_section = "__param"] +#[used] +static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_mem_path" + } else { c"aurora.g17p_mem_path" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&MASK_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/mod.rs b/drivers/gpu/drm/asahi/g18/compat/mod.rs new file mode 100644 index 0000000000000..e8f8c8e6b009e --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/mod.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Module 1134 diagnostic controls kept for its module-parameter interface. + +#[allow(dead_code)] +pub(crate) mod dep_stats; +#[allow(dead_code)] +pub(crate) mod dvfs_parity; +#[allow(dead_code)] +pub(crate) mod e2e; +#[allow(dead_code)] +pub(crate) mod gem_stats; +#[allow(dead_code)] +pub(crate) mod host_path; +#[allow(dead_code)] +pub(crate) mod mem_path; +#[allow(dead_code)] +pub(crate) mod queue_slots; +#[allow(dead_code)] +pub(crate) mod recovery_isolation; +#[allow(dead_code)] +pub(crate) mod recovery_path; +#[allow(dead_code)] +pub(crate) mod stackwatch; +#[allow(dead_code)] +pub(crate) mod timing_buffer; +#[allow(dead_code)] +pub(crate) mod g17p; diff --git a/drivers/gpu/drm/asahi/g18/compat/queue_slots.rs b/drivers/gpu/drm/asahi/g18/compat/queue_slots.rs new file mode 100644 index 0000000000000..0de236b4ab872 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/queue_slots.rs @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Firmware queue-slot accounting (`g17p_queue_slots`, root-only, read-only +//! sysfs). Counter-only: relaxed atomics, no printk. +//! +//! Every render pair and compute queue leases QIDs from one 128-entry table +//! (`g17_channel`). A published QID is kept until processor stop, and a +//! quarantined or exit-killed compute graph is never handed out again, so a +//! boot can run out of QIDs while memory is fine. Queue creation then fails +//! with ENOSPC, and so does a new render pair (`render-warm-pairs`). The +//! `qids` fields are exact (kept by the table itself); the `compute`, +//! `teardowns` and `render_quarantined` fields are a snapshot taken on every +//! teardown turn and on every queue creation. +//! +//! Every ENOSPC from one of these pools is counted by pool and named in +//! dmesg (the first one, then each power of two), so a failed QUEUE_CREATE +//! or submit says which pool ran out. +use core::fmt::Write; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam, prelude::*}; + +use crate::g18::compat::g17p::{self as g17_channel, QID_COUNT}; + +static COMPUTE_GRAPHS: AtomicU32 = AtomicU32::new(0); +static COMPUTE_OWNED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_RELEASED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_QUARANTINED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_EXIT_KILLED: AtomicU32 = AtomicU32::new(0); +static RETIRED_GRAPHS: AtomicU32 = AtomicU32::new(0); +static TEARDOWNS: AtomicU32 = AtomicU32::new(0); +static RENDER_QUARANTINED: AtomicU32 = AtomicU32::new(0); + +/// Pools whose exhaustion returns ENOSPC to a client. +#[derive(Clone, Copy)] +pub(crate) enum Pool { + /// A render slot's first use leases two QIDs (`render-warm-pairs`). + QidRenderPair, + /// A new compute graph leases one QID (queue creation / first compute). + QidCompute, + /// A new logical queue context (QUEUE_CREATE): its only 128-wide pool + /// is the QoS hardware-buffer IDs, held by logical queues and installed + /// graph bindings. + QueueContextQos, + /// Any other submit-path ENOSPC (TSQ render slots, retained scenes, + /// compute item ring), named by its submit stage in the log line. + SubmitOther, +} + +const POOL_COUNT: usize = 4; +const POOL_NAMES: [&str; POOL_COUNT] = ["qid-render-pair", "qid-compute", "queue-context-qos", "submit-other"]; +static POOL_ENOSPC: [AtomicU64; POOL_COUNT] = + [AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0)]; +/// Submits that failed with ENOSPC, whichever pool. +static SUBMIT_ENOSPC: AtomicU64 = AtomicU64::new(0); +static SUBMIT_FAILURES: AtomicU64 = AtomicU64::new(0); +static CONTEXT_UNBOUND_REFUSALS: AtomicU64 = AtomicU64::new(0); + +pub(crate) fn note_submit_failure() { + SUBMIT_FAILURES.fetch_add(1, Ordering::Relaxed); +} + +pub(crate) fn note_context_unbound_refusal() { + CONTEXT_UNBOUND_REFUSALS.fetch_add(1, Ordering::Relaxed); +} + +fn log_enospc(pool: &str, stage: &str, count: u64) { + // First one, then each power of two: a stuck pool cannot flood dmesg. + if count.is_power_of_two() { + pr_warn!( + "G17P ENOSPC pool={} stage={} (#{}): qids leased={}/{} compute={} compute-quarantined={} \ + reserve-refused={}\n", + pool, stage, count, g17_channel::leased_total(), QID_COUNT, + g17_channel::LEASED[2].load(Ordering::Relaxed), + COMPUTE_QUARANTINED.load(Ordering::Relaxed), + g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), + ); + } +} + +/// Count and name one ENOSPC from `pool`, and return it. +#[cold] +pub(crate) fn enospc(pool: Pool) -> Error { + let count = POOL_ENOSPC[pool as usize].fetch_add(1, Ordering::Relaxed) + 1; + log_enospc(POOL_NAMES[pool as usize], "-", count); + ENOSPC +} + +/// A submit failed with ENOSPC at `stage`. Pools tagged at their source +/// were already counted and named; this names the rest by stage. +#[cold] +pub(crate) fn note_submit_enospc(stage: &str) { + SUBMIT_ENOSPC.fetch_add(1, Ordering::Relaxed); + if stage != "render-warm-pairs" { + let other = POOL_ENOSPC[Pool::SubmitOther as usize].fetch_add(1, Ordering::Relaxed) + 1; + log_enospc(POOL_NAMES[Pool::SubmitOther as usize], stage, other); + } +} + +/// [`enospc`] for a call that may fail for other reasons too. +pub(crate) fn note_error(pool: Pool, error: Error) { + if error == ENOSPC { + enospc(pool); + } +} + +/// Snapshot of the runtime's queue-slot holders. +pub(crate) struct Snapshot { + pub(crate) compute_graphs: usize, + pub(crate) compute_owned: usize, + pub(crate) compute_released: usize, + pub(crate) compute_quarantined: usize, + pub(crate) compute_exit_killed: usize, + pub(crate) retired_graphs: usize, + pub(crate) teardowns: usize, + pub(crate) render_quarantined: usize, +} + +pub(crate) fn note(snapshot: Snapshot) { + let set = |gauge: &AtomicU32, value: usize| { + gauge.store(u32::try_from(value).unwrap_or(u32::MAX), Ordering::Relaxed) + }; + set(&COMPUTE_GRAPHS, snapshot.compute_graphs); + set(&COMPUTE_OWNED, snapshot.compute_owned); + set(&COMPUTE_RELEASED, snapshot.compute_released); + set(&COMPUTE_QUARANTINED, snapshot.compute_quarantined); + set(&COMPUTE_EXIT_KILLED, snapshot.compute_exit_killed); + set(&RETIRED_GRAPHS, snapshot.retired_graphs); + set(&TEARDOWNS, snapshot.teardowns); + set(&RENDER_QUARANTINED, snapshot.render_quarantined); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn slots_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let r = |c: &AtomicU32| c.load(Ordering::Relaxed); + let e = |pool: usize| POOL_ENOSPC[pool].load(Ordering::Relaxed); + let leased = g17_channel::leased_total(); + let published: u32 = g17_channel::PUBLISHED.iter().map(r).sum(); + let _ = writeln!(out, + "qids leased={} free={} published={} tiling={} fragment={} compute={} high_water={} exhausted={} \ + reserve={} reserve_refused={} \ + compute graphs={} owned={} released={} quarantined={} exit_killed={} retired_graphs={} \ + teardowns={} render_quarantined={} \ + enospc qid-render-pair={} qid-compute={} queue-context-qos={} submit-other={} submits={}", + leased, (QID_COUNT as u32).saturating_sub(leased), published, + r(&g17_channel::LEASED[0]), r(&g17_channel::LEASED[1]), r(&g17_channel::LEASED[2]), + r(&g17_channel::HIGH_WATER), g17_channel::EXHAUSTED.load(Ordering::Relaxed), + g17_channel::COMPUTE_RENDER_RESERVE, g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), + r(&COMPUTE_GRAPHS), r(&COMPUTE_OWNED), r(&COMPUTE_RELEASED), r(&COMPUTE_QUARANTINED), + r(&COMPUTE_EXIT_KILLED), r(&RETIRED_GRAPHS), r(&TEARDOWNS), r(&RENDER_QUARANTINED), + e(0), e(1), e(2), e(3), SUBMIT_ENOSPC.load(Ordering::Relaxed)); + let _ = writeln!(out, "submit failures={} context-unbound-refusals={}", + SUBMIT_FAILURES.load(Ordering::Relaxed), + CONTEXT_UNBOUND_REFUSALS.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} + +unsafe extern "C" fn slots_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + -1 // EPERM: read-only counters +} + +static SLOTS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(slots_set), get: Some(slots_get), free: None, +}; +#[link_section = "__param"] +#[used] +static SLOTS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_queue_slots" + } else { c"aurora.g17p_queue_slots" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&SLOTS_OPS), perm: 0o400, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/recovery_isolation.rs b/drivers/gpu/drm/asahi/g18/compat/recovery_isolation.rs new file mode 100644 index 0000000000000..22d7995b6a000 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/recovery_isolation.rs @@ -0,0 +1,644 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Recovery isolation (`g17p_recovery_isolation`): after a firmware +//! stamp-blamed recovery only the guilty client loses. +//! +//! Root-only sysfs, runtime-writable bitmask; also settable at load. Writing +//! 0 restores the accepted Y behaviour exactly (no completion is ever marked, +//! no admission wait is taken). Writing "reset" clears the counters. +//! +//! Hardware evidence (saved kernel logs, 2026-09-30/10-01): a stamp-blamed +//! recovery cancels in-flight work of EVERY client, and Y then poisons each +//! affected VM's sticky status, which Mesa turns into VK_ERROR_DEVICE_LOST: +//! * mmuf2 01:10:21 -- blamed qid 10 (Hyprland, VM 15). Xwayland's compute +//! qid 7 (VM 17) retired zero-duration -> `fail_while_owned(ENODATA)` +//! -> VM 17 status -61 -> every map refused -> Xwayland died. +//! * hypr-crash-0224 02:24:31 -- blamed qid 5 (another client). Hyprland's +//! parked 3D (qid 14) never ran after the recovery; its 2 s timeout +//! retired it unexecuted -> `complete(Err(EIO))` -> VM 43 status -5 -> +//! Hyprland aborted. +//! +//! Recovery-info entry layout. Native decode (static RE of the J700 kext, +//! `AGXArmFirmware::processFirmwareInitiatedRecovery` in +//! neo-stability-20260930/native/kext25.dis @fffffe00088a8398): it walks all +//! 256 8-byte entries, skips those without bit0, takes qid = bits 4..10 +//! (`ubfx w24, w22, #4, #7`), looks up the command by (qid, word1) and stores +//! a per-command error code: 5 when bit1 is clear, otherwise 3 (or 0xb when +//! the restart reason is 3). In public macOS logs 5 is +//! kIOGPUCommandBufferCallbackErrorInnocentVictim, 3 Hang, 0xb PageFault +//! (names from macOS log strings, NOT from this binary). So bit1 = GUILTY, +//! and native fails every listed non-guilty command with InnocentVictim -- +//! per command buffer, no replay seen in this function. Linux captures agree: +//! bit1 is set exactly on the queue the host's blamed stamp names in 4/4 +//! recoveries. bit2 is set on queues the firmware cancelled, bit3 on TA +//! queues (captures; native only forwards bit2 into its restart report). +//! +//! bit0 INNOCENT: classify every in-flight completion at each recovery. +//! A VM owns none of the guilty queues (blamed stamp + entries with +//! bit1) -> its cancelled work fails with an error fence ONLY: no +//! sticky VM status, no VM-wide fan-out. Its compute queue, once the +//! existing firmware-retirement witness proves the cancelled kicks +//! retired, returns to service instead of staying quarantined for the +//! boot. Unresolvable blame (a guilty qid whose owner cannot be found) +//! disables isolation for that recovery: today's behaviour. +//! bit1 ADMISSION_WAIT: a VM whose status is clean but whose mappings are +//! transiently blocked by retained (quarantined) work waits up to +//! ADMISSION_WAIT_MS for the release instead of refusing VM_BIND, and +//! a compute submit waits up to SUBMIT_WAIT_MS for the retirement +//! witness of its own spared queue. Poisoned VMs still refuse at once. +//! (iso3) A render submit whose owner has every pair quarantined +//! waits up to ADMISSION_WAIT_MS for the reclaim that returns the +//! pair, instead of Y's immediate EIO, which poisons the VM through +//! `fail_unpublished`. Field case: boot d324fcdd 05:38:44, spared VM +//! 2559's pair (slot 11) was quarantined at .370767, its next render +//! failed at .371211 and poisoned the VM, and the reclaim returned the +//! pair at .373928. An owner with a healthy pair never waits here. +//! bit2 SETTLE_CANCELLED: spared renders the firmware listed as CANCELLED +//! (bit2, not bit1) are settled at the end of the recovery through the +//! same exact-once settle the blamed slot uses, instead of waiting for +//! the 2 s scheduler watchdog. A group firmware has not provably +//! retired keeps its pair quarantined (mappings retained) until the +//! reclaim witness. Listed-only entries (bit2 clear) are left alone. +//! bit3 GUILTY_GRACE: the guilty VM is also spared, at most once per +//! GRACE_WINDOW_NS per VM; a second guilty recovery inside the window +//! poisons it as today. Opt-in: this drops the guilty submission +//! silently instead of reporting device loss. +//! bit4 RESUME_DOORBELL (opt-in, A/B): when spared render work the firmware +//! did not cancel is still pending after the recovery, send ONE paired +//! render doorbell (the message every render submission ends with; +//! it names no queue) so firmware rescans its rings. 02:24 evidence: +//! Hyprland's parked 3D (entry flags 0x1, not cancelled) never ran in +//! the 2 s after the recovery. Whether a doorbell activates it is the +//! question this bit answers; it is not established. +//! +//! bit6 REPLAY (iso4, default on): Vulkan has no per-command-buffer error +//! (native Metal reports InnocentVictim per command buffer; a Vulkan +//! client can only be told "device lost" or be handed wrong results), +//! so a spared command that provably never started is published again +//! instead of failing. Compute: a spared zero-duration queue's kicks +//! are kept across the retirement witness; only if EVERY kept kick's +//! own completion words still show start == end (firmware retired it +//! without executing, or never reached it) after the witness, and the +//! queue was returned to service, are they published again, oldest +//! first, before any newer submission of that queue (the compute +//! backend waits up to SUBMIT_WAIT_MS for the witness). Fences stay +//! unsignalled and CLASS_SPARED throughout, so nothing on this path +//! can poison the VM; a refused or failed replay fails that kick and +//! all its successors exactly as before (ENODATA, spared). A command +//! is replayed at most MAX_REPLAYS times, and each replay grants its +//! scheduler job one fresh timeout period (the recovery consumed the +//! old one). A replayed front kick is never treated as an earlier +//! recovery's collateral: if a later recovery blames its QID, it ran on +//! its own after the replay and its owner is guilty (no replay loop). +//! A kick that a render already named as its native parent is not +//! replayed (see `ComputeReplayQueue::native_consumer`). +//! Renders are NOT replayed yet; spared renders failed after +//! firmware retirement are counted by whether their pass started +//! (`spared-render-started/-unstarted`) to size that work. +//! Evidence level: "start == end means never executed" is the driver's +//! existing zero-duration interpretation (`g17p_zero_duration`), and +//! kicks of one QID run in publication order, so a zero-duration front +//! kick implies none behind it ran. It is not independently decoded +//! from native firmware. Every in-flight kick snapshotted at recovery +//! in the saved logs (199 snapshots) had page words [0, 0]. +//! bit0 also covers inherited errors (iso4): a consumer whose dependency +//! failed fails with ECANCELED instead of the dependency's EIO / +//! ETIMEDOUT / ENODEV, so it is not charged as its own VM's device +//! loss. Since c209c3dc5, consumers of a producer with host-copied +//! timestamps wait for its completion fence, so they always see a +//! failed producer's error; a spared render retired by firmware +//! completes with EIO, which used to poison every consumer's VM. +//! +//! v2 (iso2): a recovery that blames a queue whose OLDEST in-flight work was +//! already spared by an earlier recovery treats it as that recovery's +//! collateral and spares its owner again (counter `collateral`); and the +//! first job guard of a submission takes exactly Y's path unless Y's path +//! refuses it (no extra locking on the hot path). +//! +//! v3 (iso3) bit5 ATTRIBUTION (default on): the firmware's blame is the +//! first set bit of the hardware-slot fault masks (B1 `FUN_5450`, see +//! `PRIMARY_STATUS_B_RECOVERY_SLOT_MASK_OFFSET`), scanned TA first (inferred +//! from captures), not the slot that stalled. +//! With two or more sources, if the blamed slot's pass provably never +//! started (its tiler never stamped the pass start, and no DM1 progress +//! record exists for it), the culprit is re-attributed to the other sources +//! whose pass provably started (stamped pass start). If none did, no client +//! is guilty. A blamed source +//! with any execution evidence, a single-source fault and every page fault +//! (reason 3) keep the firmware's blame. Field case: 06:02:19 boot d324fcdd, +//! mask 0x11: slot 0 = quickshell TA qid 9 (pass start 0, no record) was +//! blamed; slot 4 = Minecraft 3D qid 27 (pass started, DM1 record state 2) +//! had stalled. +//! +//! bit7 VICTIM_DEFER (t-1179, default on): a HOST scheduler timeout is +//! not a firmware recovery, so nothing above classifies it. Field case +//! (boot of 2026-10-06 06:47:56, census J700-003): a vkd3d-proton test +//! (owner 18911, qid 72, vm 25834) stopped retiring compute; its 2 s +//! watchdog quarantined it. gnome-shell's compute kick (owner 3, qid 4, +//! vm 5) had been published right behind it (submissions 3131223 and +//! 3131225) and never ran: global GPU progress moved by one in those +//! 6 s. Its own watchdog fired at 06:48:02, poisoned VM 5 with -110 and +//! failed its render slots; firmware retired BOTH queues at 06:48:05, +//! three seconds later, but VM 5 refused every later map and the +//! compositor aborted. So a compute timeout whose queue's oldest kick +//! was published after an unfinished kick of ANOTHER VM's queue is +//! treated as head-of-line blocked: its scheduler timeout is renewed, +//! at most VICTIM_MAX_RENEWALS times, and nothing else changes (a +//! firmware recovery meanwhile classifies its kicks as usual). If the +//! blocker is still unretired when the budget runs out, its in-flight +//! kicks are classified CLASS_SPARED before the quarantine: their +//! fences fail, the VM status is untouched and the queue returns after +//! the retirement witness. Without a blocker, or once the blocker +//! retired, a timeout is the queue's own and keeps today's behaviour +//! (VM poisoned). A queue quarantined without retirement tickets +//! never counts as a blocker: its kicks stay listed until processor +//! stop, whatever firmware did. Kicks of different QIDs share the +//! compute pipe in publication order -- the field case shows that +//! blocking on G17P and the same client pair shows it on G13 (a 3 s +//! kick delayed another process's tiny kick by 2.998 s), but it is not +//! decoded from firmware -- so the renewal is +//! bounded, and the oldest unretired kick never has a blocker: a stall +//! always charges at least that queue. +//! +//! Memory safety is unchanged in every mode: packets, VM-job guards and +//! quarantined pairs are retained and released only by the existing +//! witnesses (exact retirement, firmware drain/reclaim, processor stop). +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +#[cfg(not(test))] +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const INNOCENT: u32 = 1 << 0; +pub(crate) const ADMISSION_WAIT: u32 = 1 << 1; +pub(crate) const SETTLE_CANCELLED: u32 = 1 << 2; +pub(crate) const GUILTY_GRACE: u32 = 1 << 3; +pub(crate) const RESUME_DOORBELL: u32 = 1 << 4; +pub(crate) const ATTRIBUTION: u32 = 1 << 5; +pub(crate) const REPLAY: u32 = 1 << 6; +pub(crate) const VICTIM_DEFER: u32 = 1 << 7; +const ALL_BITS: u32 = INNOCENT | ADMISSION_WAIT | SETTLE_CANCELLED | GUILTY_GRACE | RESUME_DOORBELL + | ATTRIBUTION | REPLAY | VICTIM_DEFER; +/// Correctness/containment fixes with no hot-path cost are on; the guilty +/// grace changes the guilty client's semantics and stays opt-in. +pub(crate) const DEFAULT_MASK: u32 = INNOCENT | ADMISSION_WAIT | SETTLE_CANCELLED | ATTRIBUTION | REPLAY + | VICTIM_DEFER; + +pub(crate) const ADMISSION_WAIT_MS: u32 = 1000; +/// iso4 REPLAY: publications of one command beyond its first. +pub(crate) const MAX_REPLAYS: u8 = 3; +pub(crate) const SUBMIT_WAIT_MS: u32 = 200; +pub(crate) const GRACE_WINDOW_NS: u64 = 60_000_000_000; +/// bit7: scheduler periods (2 s each) a head-of-line blocked compute job may +/// wait for its blocker beyond its own. The field blocker retired 9 s after +/// its own timeout; 8 extra periods cover that with margin. +pub(crate) const VICTIM_MAX_RENEWALS: u8 = 8; + +static MASK: AtomicU32 = AtomicU32::new(DEFAULT_MASK); + +#[inline(always)] +pub(crate) fn enabled(bit: u32) -> bool { MASK.load(Ordering::Relaxed) & bit != 0 } + +#[inline(always)] +pub(crate) fn mask() -> u32 { MASK.load(Ordering::Relaxed) } + +// ---- REPLAY decision (bit6) --------------------------------------------------- + +/// Host state of one quarantined compute queue at its retirement witness. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct ComputeReplayQueue { + /// Every lost kick was spared and its fence is still unsignalled. + pub(crate) spared_deferred: bool, + /// The queue will be returned to service by this witness. + pub(crate) spared_quarantine: bool, + /// The owner has released (closed) the queue. + pub(crate) released: bool, + /// The queue still belongs to the witness ticket's owner. + pub(crate) owner_matches: bool, + /// A render named one of these kicks as its native parent. Firmware + /// retired the cancelled kick, which satisfies that barrier, so the + /// render may already have run without the kick's output; a replay would + /// then write after its consumer read. + pub(crate) native_consumer: bool, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum ComputeReplay { + /// Not a spared deferred queue, or the bits are off: unchanged path. + NotCandidate, + /// A candidate whose kicks cannot all be proven unexecuted: they fail + /// exactly as before (ENODATA, spared). + Refused, + /// Every kick provably never executed: keep them and publish again, + /// oldest first, once the queue is back in service. + Replay, +} + +/// A kick's own completion words, read after the retirement witness, prove +/// it never executed when start == end: firmware's zero-duration retirement +/// shape, or both words still at the zeroed allocation value. +pub(crate) const fn never_executed(words: [u64; 2]) -> bool { words[0] == words[1] } + +/// `kicks` yields, oldest first, each kick's completion words, or None when +/// its fence is already signalled or the words are unreadable. +pub(crate) fn compute_replay_decision(mask: u32, queue: ComputeReplayQueue, kicks: I) -> ComputeReplay +where I: IntoIterator> { + if mask & (REPLAY | INNOCENT) != (REPLAY | INNOCENT) + || !queue.spared_deferred || !queue.spared_quarantine + || queue.released || !queue.owner_matches + { + return ComputeReplay::NotCandidate; + } + if queue.native_consumer { + return ComputeReplay::Refused; + } + let mut any = false; + for words in kicks { + any = true; + match words { + Some(words) if never_executed(words) => {} + _ => return ComputeReplay::Refused, + } + } + if any { ComputeReplay::Replay } else { ComputeReplay::NotCandidate } +} + +/// An error a consumer inherits from a failed dependency (an input fence or +/// a firmware-ordered parent) is not the consumer's own device loss. With +/// INNOCENT set, the device-loss codes EIO, ETIMEDOUT and ENODEV become +/// ECANCELED for the consumer: its job fails, its VM keeps running. Errnos +/// are negative, as `Error::to_errno` returns them. +pub(crate) const fn inherited_errno(mask: u32, errno: i32) -> i32 { + const EIO: i32 = -5; + const ENODEV: i32 = -19; + const ETIMEDOUT: i32 = -110; + const ECANCELED: i32 = -125; + if mask & INNOCENT != 0 && (errno == EIO || errno == ETIMEDOUT || errno == ENODEV) { + ECANCELED + } else { + errno + } +} + +/// One more publication of a command that has been replayed `replays` times. +pub(crate) const fn replay_budget_left(replays: u8) -> bool { replays < MAX_REPLAYS } + +/// Per-completion classification, written only while servicing a recovery. +pub(crate) const CLASS_NONE: u8 = 0; +/// Cancelled by another client's recovery (or graced): fail the fence only. +pub(crate) const CLASS_SPARED: u8 = 1; +/// Owned by the guilty VM: today's behaviour. +pub(crate) const CLASS_GUILTY: u8 = 2; + +pub(crate) const ENTRY_VALID: u8 = 1 << 0; +pub(crate) const ENTRY_GUILTY: u8 = 1 << 1; +pub(crate) const ENTRY_CANCELLED: u8 = 1 << 2; + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct RecoveryEntry { + pub(crate) qid: u8, + pub(crate) flags: u8, + pub(crate) stamp: u32, +} + +/// Decode one recovery-info table entry (see the module comment). +pub(crate) const fn decode_entry(word0: u32, word1: u32) -> Option { + if word0 & 1 == 0 { return None; } + // Native: ubfx #4, #7. + let qid = (word0 >> 4) & 0x7f; + Some(RecoveryEntry { qid: qid as u8, flags: (word0 & 0xf) as u8, stamp: word1 >> 8 }) +} + +/// QID masks of a decoded table plus the blamed stamp: +/// (guilty, listed, cancelled). The blamed stamp's QID counts as guilty. +pub(crate) fn qid_masks(entries: &[RecoveryEntry], blamed: Option) -> (u128, u128, u128) { + let mut guilty = 0u128; + let mut listed = 0u128; + let mut cancelled = 0u128; + for entry in entries { + let bit = 1u128 << (entry.qid & 0x7f); + listed |= bit; + if entry.flags & ENTRY_GUILTY != 0 { guilty |= bit; } + if entry.flags & ENTRY_CANCELLED != 0 { cancelled |= bit; } + } + if let Some(qid) = blamed.filter(|qid| *qid < 128) { guilty |= 1u128 << qid; } + (guilty, listed, cancelled) +} + +/// Class of one VM for one recovery. Returns (class, grace consumed). +pub(crate) const fn classify(guilty: bool, grace: bool, now_ns: u64, last_grace_ns: u64) -> (u8, bool) { + if !guilty { return (CLASS_SPARED, false); } + if grace && (last_grace_ns == 0 || now_ns.saturating_sub(last_grace_ns) >= GRACE_WINDOW_NS) { + return (CLASS_SPARED, true); + } + (CLASS_GUILTY, false) +} + +// ---- head-of-line victim deferral (bit7) ------------------------------------ + +/// The oldest unretired kick of one OTHER compute queue at a host timeout. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct ComputeHead { + /// VM of that queue's owner. + pub(crate) vm: u64, + /// Global publication order of the kick (`COMPUTE_PUBLICATION`). + pub(crate) published: u64, + /// Its completion record was already latched (finished, not retired yet). + pub(crate) finished: bool, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum VictimTimeout { + /// No older unfinished kick of another VM: the timeout is this queue's own. + Own, + /// Head-of-line blocked: renew the scheduler timeout once more. + Defer, + /// Still blocked after the renewal budget: quarantine spared. + Spare, +} + +/// `own_vm`/`own_published`: the timed-out queue's VM and its oldest +/// unretired kick's publication order; `renewals`: periods already granted. +/// Returns the decision and the publication order of the oldest blocker. +pub(crate) fn victim_timeout(mask: u32, own_vm: u64, own_published: u64, renewals: u8, others: I) + -> (VictimTimeout, Option) +where I: IntoIterator { + if mask & VICTIM_DEFER == 0 { return (VictimTimeout::Own, None); } + let blocker = others.into_iter() + .filter(|head| head.vm != own_vm && !head.finished && head.published < own_published) + .map(|head| head.published) + .min(); + match blocker { + None => (VictimTimeout::Own, None), + Some(_) if renewals < VICTIM_MAX_RENEWALS => (VictimTimeout::Defer, blocker), + // Sparing needs the INNOCENT machinery (no-poison fail, witness return). + Some(_) if mask & INNOCENT != 0 => (VictimTimeout::Spare, blocker), + Some(_) => (VictimTimeout::Own, blocker), + } +} + +/// Global publication order of async compute kicks. Written only under the +/// runtime lock at publication; a plain counter, never reset. +pub(crate) static COMPUTE_PUBLICATION: AtomicU64 = AtomicU64::new(0); + +// ---- fault-source attribution (bit5) --------------------------------------- + +/// One hardware slot of the recovery's fault mask, sampled while firmware is +/// halted: the slot's cached key (qid << 40 | stamp), its DM1 progress-monitor +/// state (0 = never written), and whether its pass had started (None = not a +/// render pass the host can resolve). +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct FaultSource { + pub(crate) slot: u8, + pub(crate) qid: Option, + pub(crate) stamp: u64, + pub(crate) monitor_state: u32, + pub(crate) started: Option, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Attribution { + /// Keep the firmware's guilty set (blamed stamp + table bit1). + Firmware, + /// The blamed slot never ran; these QIDs' owners are guilty instead. + Reattributed(u128), + /// The blamed slot never ran and no other source shows execution. + NoCulprit, +} + +pub(crate) const fn decode_slot_key(key: u64) -> (Option, u64) { + if key == 0 { return (None, 0); } + let qid = (key >> 40) & 0xff; + (if qid < 128 { Some(qid as u8) } else { None }, key & 0xff_ffff_ffff) +} + +/// `blamed_qid` is the QID of the firmware's stamp-blamed request. Across 13 +/// saved recoveries the blamed slot is the lowest faulting slot of the first +/// data master, in TA, 3D, CDM order, that has one (e.g. 06:02:19 mask 0x11 +/// blamed TA slot 0 over 3D slot 4; 03:18:16 mask 0x3 blamed 3D slot 1 over +/// compute slot 0) -- an inference from captures, so the blamed source is +/// matched by QID, never by bit position. +pub(crate) fn attribute(reason: u32, blamed_qid: Option, mask: u32, sources: &[FaultSource]) -> Attribution { + // Page faults name the faulting VM directly; one source leaves no choice. + if reason == 3 || mask.count_ones() < 2 { return Attribution::Firmware; } + let Some(blamed_qid) = blamed_qid else { return Attribution::Firmware; }; + let Some(blamed) = sources.iter().find(|source| source.qid == Some(blamed_qid)) else { + return Attribution::Firmware; + }; + let blamed_slot = blamed.slot; + if blamed.started != Some(false) || blamed.monitor_state != 0 { + return Attribution::Firmware; + } + let mut qids = 0u128; + // A culprit must have provably run: its pass start is stamped. A DM1 + // progress record alone is not used for a culprit (records are historical + // DRAM and slots are shared across data masters, so a record may be stale). + for source in sources.iter().filter(|source| source.slot != blamed_slot) { + if source.started != Some(true) { continue; } + if let Some(qid) = source.qid { qids |= 1u128 << (qid & 0x7f); } + } + if qids != 0 { Attribution::Reattributed(qids) } else { Attribution::NoCulprit } +} + +pub(crate) const MAX_SOURCES: usize = 16; +static SRC_MASK: AtomicU32 = AtomicU32::new(0); +static SRC_REASON: AtomicU32 = AtomicU32::new(0); +static SRC_KEY: [AtomicU64; MAX_SOURCES] = [const { AtomicU64::new(0) }; MAX_SOURCES]; +static SRC_MONITOR: [AtomicU32; MAX_SOURCES] = [const { AtomicU32::new(0) }; MAX_SOURCES]; + +/// Store the halted-firmware sample (`keys`/`monitors` indexed by slot). +pub(crate) fn store_sources(reason: u32, mask: u32, keys: &[u64; MAX_SOURCES], monitors: &[u32; MAX_SOURCES]) { + for slot in 0..MAX_SOURCES { + SRC_KEY[slot].store(keys[slot], Ordering::Relaxed); + SRC_MONITOR[slot].store(monitors[slot], Ordering::Relaxed); + } + SRC_REASON.store(reason, Ordering::Relaxed); + SRC_MASK.store(mask, Ordering::Release); +} + +pub(crate) fn clear_sources() { SRC_MASK.store(0, Ordering::Release); } + +/// (reason, mask, sources without `started`) of the last stored sample. +pub(crate) fn sources(out: &mut [FaultSource; MAX_SOURCES]) -> (u32, u32, usize) { + let mask = SRC_MASK.load(Ordering::Acquire); + let reason = SRC_REASON.load(Ordering::Relaxed); + let mut used = 0; + for slot in 0..MAX_SOURCES { + if mask & (1u32 << slot) == 0 { continue; } + let (qid, stamp) = decode_slot_key(SRC_KEY[slot].load(Ordering::Relaxed)); + out[used] = FaultSource { + slot: slot as u8, qid, stamp, + monitor_state: SRC_MONITOR[slot].load(Ordering::Relaxed), started: None, + }; + used += 1; + } + (reason, mask, used) +} + +// ---- recovery-info snapshot ------------------------------------------------ + +pub(crate) const MAX_ENTRIES: usize = 64; +static ENTRY_WORDS: [AtomicU64; MAX_ENTRIES] = [const { AtomicU64::new(0) }; MAX_ENTRIES]; +static ENTRY_COUNT: AtomicU32 = AtomicU32::new(0); + +/// Forget the previous recovery's table before reading the new one. +pub(crate) fn clear_entries() { ENTRY_COUNT.store(0, Ordering::Release); } + +/// Store the raw (word0, word1) of every VALID entry, in table order. +pub(crate) fn store_entries(raw: &[(u32, u32)]) { + let count = raw.len().min(MAX_ENTRIES); + for (slot, &(word0, word1)) in ENTRY_WORDS.iter().zip(raw.iter()) { + slot.store(u64::from(word0) | u64::from(word1) << 32, Ordering::Relaxed); + } + ENTRY_COUNT.store(count as u32, Ordering::Release); +} + +/// Decoded entries of the last stored table. +pub(crate) fn entries(out: &mut [RecoveryEntry; MAX_ENTRIES]) -> usize { + let count = (ENTRY_COUNT.load(Ordering::Acquire) as usize).min(MAX_ENTRIES); + let mut used = 0; + for slot in ENTRY_WORDS.iter().take(count) { + let raw = slot.load(Ordering::Relaxed); + if let Some(entry) = decode_entry(raw as u32, (raw >> 32) as u32) { + out[used] = entry; + used += 1; + } + } + used +} + +// ---- counters ---------------------------------------------------------------- + +pub(crate) static RECOVERIES: AtomicU64 = AtomicU64::new(0); +pub(crate) static UNRESOLVED: AtomicU64 = AtomicU64::new(0); +pub(crate) static SPARED_RENDERS: AtomicU64 = AtomicU64::new(0); +pub(crate) static SPARED_COMPUTE: AtomicU64 = AtomicU64::new(0); +pub(crate) static GUILTY_MARKED: AtomicU64 = AtomicU64::new(0); +pub(crate) static GRACED: AtomicU64 = AtomicU64::new(0); +pub(crate) static POISON_SKIPPED: AtomicU64 = AtomicU64::new(0); +pub(crate) static FANOUT_SKIPPED: AtomicU64 = AtomicU64::new(0); +pub(crate) static SETTLED_CANCELLED: AtomicU64 = AtomicU64::new(0); +pub(crate) static QUEUES_RETURNED: AtomicU64 = AtomicU64::new(0); +pub(crate) static QUEUE_RETURN_FAILED: AtomicU64 = AtomicU64::new(0); +pub(crate) static FIRST_KICKS_OK: AtomicU64 = AtomicU64::new(0); +pub(crate) static FIRST_KICKS_FAILED: AtomicU64 = AtomicU64::new(0); +pub(crate) static ADMISSION_WAITS: AtomicU64 = AtomicU64::new(0); +pub(crate) static ADMISSION_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +pub(crate) static SUBMIT_WAITS: AtomicU64 = AtomicU64::new(0); +pub(crate) static SUBMIT_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +pub(crate) static RESUME_DOORBELLS: AtomicU64 = AtomicU64::new(0); +pub(crate) static COLLATERAL: AtomicU64 = AtomicU64::new(0); +pub(crate) static REATTRIBUTED: AtomicU64 = AtomicU64::new(0); +pub(crate) static NO_CULPRIT: AtomicU64 = AtomicU64::new(0); +/// iso3.1: unclassified renders queued behind a spared head, failed with it. +pub(crate) static SPARED_SUCCESSORS: AtomicU64 = AtomicU64::new(0); +/// iso4 REPLAY: spared never-started compute kicks published again, and +/// spared kicks that had to fail instead (ran, unprovable, or republish failed). +pub(crate) static REPLAYED_COMPUTE: AtomicU64 = AtomicU64::new(0); +pub(crate) static REPLAY_REFUSED_COMPUTE: AtomicU64 = AtomicU64::new(0); +/// iso4: spared renders failed after firmware retirement, split by whether +/// the pass ever started (pass-start stamp) -- the render replay candidates. +pub(crate) static SPARED_RENDER_UNSTARTED: AtomicU64 = AtomicU64::new(0); +pub(crate) static SPARED_RENDER_STARTED: AtomicU64 = AtomicU64::new(0); +/// iso4: replayed compute jobs given a fresh scheduler timeout, and compute +/// publications that waited for their own queue's witness/replay first. +pub(crate) static REPLAY_TIMEOUT_RENEWED: AtomicU64 = AtomicU64::new(0); +pub(crate) static REPLAY_ORDER_WAITS: AtomicU64 = AtomicU64::new(0); +pub(crate) static REPLAY_ORDER_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +/// iso4: replays refused because a render had already been released +/// against a cancelled kick; consumer failures not charged to the VM. +pub(crate) static REPLAY_REFUSED_CONSUMER: AtomicU64 = AtomicU64::new(0); +pub(crate) static INHERITED_NOT_POISONED: AtomicU64 = AtomicU64::new(0); +/// bit7: compute timeouts renewed because another VM's older kick was still +/// unfinished, and queues quarantined spared after the renewal budget. +pub(crate) static VICTIM_DEFERRED: AtomicU64 = AtomicU64::new(0); +pub(crate) static VICTIM_SPARED: AtomicU64 = AtomicU64::new(0); + +#[inline(always)] +pub(crate) fn count(counter: &AtomicU64) { counter.fetch_add(1, Ordering::Relaxed); } + +static COUNTERS: [(&str, &AtomicU64); 33] = [ + ("recoveries", &RECOVERIES), ("unresolved", &UNRESOLVED), + ("spared-renders", &SPARED_RENDERS), ("spared-compute", &SPARED_COMPUTE), + ("guilty-marked", &GUILTY_MARKED), ("graced", &GRACED), + ("poison-skipped", &POISON_SKIPPED), ("fanout-skipped", &FANOUT_SKIPPED), + ("settled-cancelled", &SETTLED_CANCELLED), ("queues-returned", &QUEUES_RETURNED), + ("queue-return-failed", &QUEUE_RETURN_FAILED), ("first-kicks-ok", &FIRST_KICKS_OK), + ("first-kicks-failed", &FIRST_KICKS_FAILED), ("admission-waits", &ADMISSION_WAITS), + ("admission-timeouts", &ADMISSION_TIMEOUTS), ("submit-waits", &SUBMIT_WAITS), + ("submit-timeouts", &SUBMIT_TIMEOUTS), ("resume-doorbells", &RESUME_DOORBELLS), + ("collateral", &COLLATERAL), ("reattributed", &REATTRIBUTED), ("no-culprit", &NO_CULPRIT), + ("spared-successors", &SPARED_SUCCESSORS), + ("replayed-compute", &REPLAYED_COMPUTE), ("replay-refused-compute", &REPLAY_REFUSED_COMPUTE), + ("spared-render-unstarted", &SPARED_RENDER_UNSTARTED), ("spared-render-started", &SPARED_RENDER_STARTED), + ("replay-timeout-renewed", &REPLAY_TIMEOUT_RENEWED), ("replay-order-waits", &REPLAY_ORDER_WAITS), + ("replay-order-timeouts", &REPLAY_ORDER_TIMEOUTS), + ("replay-refused-consumer", &REPLAY_REFUSED_CONSUMER), + ("inherited-not-poisoned", &INHERITED_NOT_POISONED), + ("victim-deferred", &VICTIM_DEFERRED), ("victim-spared", &VICTIM_SPARED), +]; + +// ---- parameter ----------------------------------------------------------------- + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} + +pub(crate) fn parse_mask(text: &str) -> Option { + let parsed = match text.strip_prefix("0x") { + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + }?; + (parsed & !ALL_BITS == 0).then_some(parsed) +} + +#[cfg(not(test))] +unsafe extern "C" fn mask_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + // SAFETY: the parameter core supplies a NUL-terminated string. + let Ok(text) = unsafe { core::ffi::CStr::from_ptr(value) }.to_str() else { return -22; }; + let text = text.trim(); + if text == "reset" { + for (_, counter) in COUNTERS.iter() { counter.store(0, Ordering::Relaxed); } + return 0; + } + let Some(mask) = parse_mask(text) else { return -22; }; + MASK.store(mask, Ordering::Release); + 0 +} + +#[cfg(not(test))] +unsafe extern "C" fn mask_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes, used: 0 }; + let _ = write!(out, "{:#x}", MASK.load(Ordering::Relaxed)); + for (name, counter) in COUNTERS.iter() { + let _ = write!(out, " {}={}", name, counter.load(Ordering::Relaxed)); + } + let _ = writeln!(out); + out.used as core::ffi::c_int +} + +#[cfg(not(test))] +static MASK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(mask_set), get: Some(mask_get), free: None, +}; +#[cfg(not(test))] +#[link_section = "__param"] +#[used] +static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_recovery_isolation" + } else { c"aurora.g17p_recovery_isolation" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&MASK_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/recovery_path.rs b/drivers/gpu/drm/asahi/g18/compat/recovery_path.rs new file mode 100644 index 0000000000000..f35cf67d91715 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/recovery_path.rs @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! v93r post-recovery controls and control-ring diagnostics (`g17p_recovery_path`). +//! +//! Root-only sysfs, runtime-writable bitmask, also settable at load. Default +//! is RECLAIM_DRAINED; 0 is exactly v93f. Without bit0 a client whose render +//! was failed by a stamp-blamed recovery keeps its whole VM -- every BO it +//! ever mapped -- pinned until both firmware processors stop, which in +//! practice is the next reboot. +//! bit0 RECLAIM_DRAINED: after a firmware stamp-blamed recovery the render +//! whose group could not be proven retired, and every render queued +//! behind it on the same physical pair, are failed with their guards +//! retained (`fail_while_owned`) and the pair is quarantined until both +//! firmware processors stop. With this bit, once the recovery is +//! closed (host and firmware handshake state 0) and the firmware has +//! demonstrably drained that pair -- both engines' post-maintenance +//! completion stamps cover the pair's last published ordinal and the +//! pair FList's completed-kick counter equals its submitted counter -- +//! the event worker retires the pair's trackers through the normal +//! paired-completion transaction (or, for a group the recovery +//! skipped, the existing firmware-retired classifier), releases the +//! retained VM-job and lease guards, and returns the pair. The client's +//! VM stays poisoned; only the physical pair, its FList/HWPB state and +//! the closed queue's teardown are recovered. +//! bit1 CONTROL_TRACE: remember opcode/argument/publish time of every +//! primary device-control record and the time of the last EP 0x21 +//! announcement; a control re-announce then names the oldest pending +//! record and whether it had been announced after its publication. +//! bit2 GROW_REKICK (v96): firmware event 6 (TVB grow) pauses the pool's +//! allocator until the opcode-8 reply. A request whose owning pair is +//! borrowed (`preparing`: warm-up, restage, grow, completion +//! observation) is parked, and the grow worker exits. Without this bit +//! only the next firmware event re-queues the worker; if the stalled +//! render was the only producer of events, nothing does. With this +//! bit, every end of a borrow (and every clearing of control-ring +//! backpressure) re-queues the worker while a request is parked. +//! bit3 reserved (v97 INNOCENT_VICTIM, not in this build; rejected). +//! bit4 STALL_WATCH (v97d, DIAGNOSTIC): while any render pair has +//! outstanding work, a 4 ms delayed work reads the firmware per-QID +//! dependency records (firmware DATA, read-only) of every busy pair. +//! A 3D queue is a stall candidate when its TA's completed stamp has +//! passed the 3D's next stamp (its TA parent is complete) while that +//! queue has not advanced for >= STALL_MS. Other clients' progress +//! must not hide starvation of this queue. The first sample of +//! a stall logs the dependency records, the B1 scheduler masks +//! (requested/sampled/deferred/next), the runnable mask and the park +//! record. Log and count only: no firmware write, no message. +//! bit5 STALL_REDOORBELL (v97d, DIAGNOSTIC, needs bit4): on a detected +//! stall re-send the exact paired render doorbell a submission ends +//! with (the only host action that activates a published render), +//! once per stall. Nothing is sent while a firmware recovery is +//! pending or the session is not accepting work. A doorbell names no +//! queue: firmware rescans its own rings, as after any submission. +//! bit6 STALL_FRAGMENT (v97d, DIAGNOSTIC, needs bits 4+5): if a stall is +//! still present STALL_MS after the paired doorbell, also send the +//! Fragment (3D) doorbell once. Native sends only the paired Vertex +//! doorbell per render, so this arm is a discriminator, never a fix. +//! Grow ingress, deferral, re-kick and recovery-time grow state are always +//! logged (rate-limited) and counted, independent of the mask. +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const RECLAIM_DRAINED: u32 = 1 << 0; +pub(crate) const CONTROL_TRACE: u32 = 1 << 1; +pub(crate) const GROW_REKICK: u32 = 1 << 2; +pub(crate) const STALL_WATCH: u32 = 1 << 4; +pub(crate) const STALL_REDOORBELL: u32 = 1 << 5; +pub(crate) const STALL_FRAGMENT: u32 = 1 << 6; +const ALL_BITS: u32 = RECLAIM_DRAINED | CONTROL_TRACE | GROW_REKICK | STALL_WATCH | STALL_REDOORBELL + | STALL_FRAGMENT; + +const DEFAULT_MASK: u32 = RECLAIM_DRAINED; +static MASK: AtomicU32 = AtomicU32::new(DEFAULT_MASK); + +#[inline(always)] +pub(crate) fn enabled(bit: u32) -> bool { MASK.load(Ordering::Relaxed) & bit != 0 } + +/// Why a reclaim attempt for one quarantined pair stopped (counters and log). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ReclaimBlock { + /// A firmware recovery is open or the handshake state is not 0. + Recovery, + /// The pair is being prepared by a publisher. + Preparing, + /// A completion stamp does not yet cover the last published ordinal. + Stamps, + /// The pair FList still has submitted kicks not completed. + Flist, + /// A tracked group is not yet provably retired. + Tracker, + /// A step returned an error; the pair stays quarantined. + Error, +} + +impl ReclaimBlock { + const fn index(self) -> usize { + match self { + Self::Recovery => 0, Self::Preparing => 1, Self::Stamps => 2, + Self::Flist => 3, Self::Tracker => 4, Self::Error => 5, + } + } + const NAMES: [&'static str; 6] = ["recovery", "preparing", "stamps", "flist", "tracker", "error"]; +} + +/// Attempts, reclaimed pairs, retired trackers, released completions. +pub(crate) static RECLAIM_ATTEMPTS: AtomicU64 = AtomicU64::new(0); +pub(crate) static RECLAIM_PAIRS: AtomicU64 = AtomicU64::new(0); +pub(crate) static RECLAIM_TRACKERS: AtomicU64 = AtomicU64::new(0); +pub(crate) static RECLAIM_RELEASED: AtomicU64 = AtomicU64::new(0); +static RECLAIM_BLOCKED: [AtomicU64; 6] = [const { AtomicU64::new(0) }; 6]; + +/// v96 TVB grow accounting: firmware requests received, requests left parked +/// because their pair was borrowed when the worker looked, borrow ends that +/// found a parked request, and worker re-queues issued for them (bit2). +pub(crate) static GROW_INGRESS: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_DEFERRED: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_PARKED_AT_RETURN: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_REKICKS: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_REPLIED: AtomicU64 = AtomicU64::new(0); + +/// Handshake steps 5-6: the last opcode-8 grow reply published to the +/// primary control ring (bit63 valid | producer_after << 32 | buffer_slot) +/// and its CNTPCT, until the firmware consumer is observed past it. +pub(crate) static GROW_REPLY_WATCH: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_REPLY_AT: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_REPLY_CONSUMED: AtomicU64 = AtomicU64::new(0); + +/// v97d 3D stall watch (bits 4/5). Polls, detections, detections whose 3D +/// QID was in the B1 deferred mask, doorbells sent (paired / Fragment), +/// stalls that cleared after a doorbell, stalls that cleared unaided, and +/// stalls still present when a firmware recovery arrived. +pub(crate) static STALL_POLLS: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_DETECTED: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_DEFERRED: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_DOORBELLS: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_FRAGMENT_DOORBELLS: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_CLEARED_AFTER_DOORBELL: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_CLEARED_UNAIDED: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_AT_RECOVERY: AtomicU64 = AtomicU64::new(0); +/// Longest detected stall (ns from candidate start to clearance). +pub(crate) static STALL_MAX_NS: AtomicU64 = AtomicU64::new(0); + +/// Poll period and detection threshold. The native DM monitor escalates a +/// no-progress DM1 slot after two samples ~36 ms apart (T2 hang: +34/+70 ms +/// after the TA completed); detection must precede it. +pub(crate) const STALL_POLL_MS: u64 = 4; +pub(crate) const STALL_MS: u64 = 20; + +/// Per-pair watch state, owned by the Runtime lock (atomics only for +/// static storage). `since` = ns when the pair became a candidate with this +/// 3D completed stamp (0 = not a candidate); `stage` 0 none, 1 detected, +/// 2 paired doorbell sent, 3 Fragment doorbell sent, 4 send failed. +pub(crate) struct StallSlot { + pub(crate) fragment_completed: AtomicU64, + pub(crate) since: AtomicU64, + pub(crate) stage: AtomicU32, + pub(crate) acted_at: AtomicU64, +} +pub(crate) const STALL_SLOTS: usize = 16; +pub(crate) static STALL_SLOT: [StallSlot; STALL_SLOTS] = [const { + StallSlot { fragment_completed: AtomicU64::new(u64::MAX), since: AtomicU64::new(0), + stage: AtomicU32::new(0), acted_at: AtomicU64::new(0) } +}; STALL_SLOTS]; +/// A 3D queue is a stall candidate when it has published work beyond its +/// completed stamp and the paired TA queue (same stamp numbering) has +/// completed at least the 3D's next stamp: its TA parent is complete. +/// Stamps compare as signed 40-bit values, like the firmware's barriers. +pub(crate) fn stall_candidate(ta_completed: u64, fragment_completed: u64, latest: u64, + next: Option) -> bool { + fragment_completed != latest && next.is_some_and(|next| { + let delta = ta_completed.wrapping_sub(next) & 0xff_ffff_ffff; + delta < (1 << 39) + }) +} + +/// Forget an interrupted observation without claiming the work completed. +/// A borrowed/absent graph or an unreadable dependency record is not a +/// retirement witness. A later sample must start a new contiguous interval. +pub(crate) fn discard_stall_observation(slot: &StallSlot) { + slot.since.store(0, Ordering::Relaxed); + slot.stage.store(0, Ordering::Relaxed); + slot.acted_at.store(0, Ordering::Relaxed); + slot.fragment_completed.store(u64::MAX, Ordering::Relaxed); +} + +/// Observe one queue independently of every other queue's progress. Only a +/// changed completion stamp counts as clearance. Losing the candidate +/// predicate or its graph cannot establish that a doorbell rescued it. +pub(crate) fn sample_stall_slot(slot: &StallSlot, now: u64, + fragment_completed: u64, candidate: bool) -> bool { + if slot.fragment_completed.swap(fragment_completed, Ordering::Relaxed) != fragment_completed { + note_stall_cleared(slot, now); + } + if !candidate { + discard_stall_observation(slot); + return false; + } + let since = slot.since.load(Ordering::Relaxed); + if since == 0 { + slot.since.store(now, Ordering::Relaxed); + return false; + } + now.saturating_sub(since) >= STALL_MS * 1_000_000 +} + +pub(crate) fn note_stall_cleared(slot: &StallSlot, now: u64) { + let since = slot.since.swap(0, Ordering::Relaxed); + let stage = slot.stage.swap(0, Ordering::Relaxed); + if stage == 0 || since == 0 { return; } + let lasted = now.saturating_sub(since); + STALL_MAX_NS.fetch_max(lasted, Ordering::Relaxed); + // "After" is temporal association, never proof of a causal rescue. + // Failed sends are deliberately not credited as sent doorbells. + if matches!(stage, 2 | 3) { + STALL_CLEARED_AFTER_DOORBELL.fetch_add(1, Ordering::Relaxed); + } else { + STALL_CLEARED_UNAIDED.fetch_add(1, Ordering::Relaxed); + } +} + +/// A firmware recovery arrived: count and describe every stall still open. +pub(crate) fn note_recovery_for_stalls() -> OpenStalls { + let mut open = OpenStalls::default(); + for (index, slot) in STALL_SLOT.iter().enumerate() { + let stage = slot.stage.load(Ordering::Relaxed); + if stage != 0 { + STALL_AT_RECOVERY.fetch_add(1, Ordering::Relaxed); + if open.count < open.pairs.len() { + open.pairs[open.count] = (index as u8, stage as u8, + slot.fragment_completed.load(Ordering::Relaxed)); + open.count += 1; + } + } + } + open +} + +/// Fixed-capacity summary of open stalls (no allocation on the event path). +#[derive(Default)] +pub(crate) struct OpenStalls { + pub(crate) pairs: [(u8, u8, u64); 4], + pub(crate) count: usize, +} + +pub(crate) fn watch_grow_reply(producer_after: u32, buffer_slot: u32) { + GROW_REPLY_AT.store(crate::g18::compat::timing_buffer::arch_counter(), Ordering::Relaxed); + GROW_REPLY_WATCH.store(1 << 63 | u64::from(producer_after) << 32 | u64::from(buffer_slot), + Ordering::Release); +} + +/// A bounded log budget: the first `first` events, then one in `every`. +pub(crate) fn log_budget(counter: u64, first: u64, every: u64) -> bool { + counter <= first || counter % every == 0 +} + +pub(crate) fn note_blocked(reason: ReclaimBlock) { + RECLAIM_BLOCKED[reason.index()].fetch_add(1, Ordering::Relaxed); +} + +// ---- control-ring shadow (bit1) ----------------------------------------- + +const CONTROL_SLOTS: usize = crate::g18::fw::initdata::CONTROL_RING_ENTRIES; +/// opcode | arg << 32, and CNTPCT at publication, per ring slot. +static CONTROL_RECORD: [AtomicU64; CONTROL_SLOTS] = [const { AtomicU64::new(0) }; CONTROL_SLOTS]; +static CONTROL_PUBLISHED_AT: [AtomicU64; CONTROL_SLOTS] = [const { AtomicU64::new(0) }; CONTROL_SLOTS]; +static LAST_ANNOUNCE: AtomicU64 = AtomicU64::new(0); +/// Re-announces whose oldest pending record was (not) announced after it +/// was published. +static REANNOUNCE_ANNOUNCED: AtomicU64 = AtomicU64::new(0); +static REANNOUNCE_UNANNOUNCED: AtomicU64 = AtomicU64::new(0); +/// Re-announces by the pending record's opcode (0..0x3f, larger folded to 0x3f). +static REANNOUNCE_BY_OPCODE: [AtomicU32; 64] = [const { AtomicU32::new(0) }; 64]; + +/// Called for every primary device-control publication (ring slot index). +#[inline(always)] +pub(crate) fn note_control_publish(slot: u32, opcode: u32, arg: u32) { + if !enabled(CONTROL_TRACE) { return; } + let index = slot as usize % CONTROL_SLOTS; + CONTROL_RECORD[index].store(u64::from(opcode) | u64::from(arg) << 32, Ordering::Relaxed); + CONTROL_PUBLISHED_AT[index].store(crate::g18::compat::timing_buffer::arch_counter(), Ordering::Relaxed); +} + +/// Called for every EP 0x21 device-control announcement actually sent. +#[inline(always)] +pub(crate) fn note_control_announce() { + if !enabled(CONTROL_TRACE) { return; } + LAST_ANNOUNCE.store(crate::g18::compat::timing_buffer::arch_counter(), Ordering::Relaxed); +} + +/// The oldest pending record at `consumer`, if traced: (opcode, arg, age +/// in counter ticks, announced after publication). +pub(crate) fn describe_pending(consumer: u32) -> Option<(u32, u32, u64, bool)> { + if !enabled(CONTROL_TRACE) { return None; } + let index = consumer as usize % CONTROL_SLOTS; + let published = CONTROL_PUBLISHED_AT[index].load(Ordering::Relaxed); + if published == 0 { return None; } + let record = CONTROL_RECORD[index].load(Ordering::Relaxed); + let now = crate::g18::compat::timing_buffer::arch_counter(); + let announced = LAST_ANNOUNCE.load(Ordering::Relaxed) >= published; + let opcode = record as u32; + let counter = if announced { &REANNOUNCE_ANNOUNCED } else { &REANNOUNCE_UNANNOUNCED }; + counter.fetch_add(1, Ordering::Relaxed); + REANNOUNCE_BY_OPCODE[(opcode as usize).min(63)].fetch_add(1, Ordering::Relaxed); + Some((opcode, (record >> 32) as u32, now.saturating_sub(published), announced)) +} + +// ---- parameter ------------------------------------------------------------ + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} + +/// Accepts "reset" or a decimal/0x-hex mask of known bits. +fn parse_mask(text: &str) -> Option { + let parsed = match text.strip_prefix("0x") { + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + }?; + (parsed & !ALL_BITS == 0).then_some(parsed) +} + +unsafe extern "C" fn mask_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + // SAFETY: the parameter core supplies a NUL-terminated string. + let Ok(text) = unsafe { core::ffi::CStr::from_ptr(value) }.to_str() else { return -22; }; + let text = text.trim(); + if text == "reset" { + for counter in [&RECLAIM_ATTEMPTS, &RECLAIM_PAIRS, &RECLAIM_TRACKERS, &RECLAIM_RELEASED, + &REANNOUNCE_ANNOUNCED, &REANNOUNCE_UNANNOUNCED, &GROW_INGRESS, + &GROW_DEFERRED, &GROW_PARKED_AT_RETURN, &GROW_REKICKS, &GROW_REPLIED, + &GROW_REPLY_CONSUMED, &STALL_POLLS, &STALL_DETECTED, &STALL_DEFERRED, + &STALL_DOORBELLS, &STALL_FRAGMENT_DOORBELLS, &STALL_CLEARED_AFTER_DOORBELL, + &STALL_CLEARED_UNAIDED, &STALL_AT_RECOVERY, &STALL_MAX_NS] { + counter.store(0, Ordering::Relaxed); + } + for counter in RECLAIM_BLOCKED.iter() { counter.store(0, Ordering::Relaxed); } + for counter in REANNOUNCE_BY_OPCODE.iter() { counter.store(0, Ordering::Relaxed); } + return 0; + } + let Some(mask) = parse_mask(text) else { return -22; }; + MASK.store(mask, Ordering::Release); + 0 +} + +unsafe extern "C" fn mask_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes, used: 0 }; + let _ = write!(out, "{:#x} reclaim attempts={} pairs={} trackers={} released={} blocked:", + MASK.load(Ordering::Relaxed), RECLAIM_ATTEMPTS.load(Ordering::Relaxed), + RECLAIM_PAIRS.load(Ordering::Relaxed), RECLAIM_TRACKERS.load(Ordering::Relaxed), + RECLAIM_RELEASED.load(Ordering::Relaxed)); + for (name, counter) in ReclaimBlock::NAMES.iter().zip(RECLAIM_BLOCKED.iter()) { + let _ = write!(out, " {}={}", name, counter.load(Ordering::Relaxed)); + } + let _ = write!(out, " tvb-grow ingress={} replied={} reply-consumed={} deferred={} parked-at-return={} rekicks={}", + GROW_INGRESS.load(Ordering::Relaxed), GROW_REPLIED.load(Ordering::Relaxed), + GROW_REPLY_CONSUMED.load(Ordering::Relaxed), + GROW_DEFERRED.load(Ordering::Relaxed), GROW_PARKED_AT_RETURN.load(Ordering::Relaxed), + GROW_REKICKS.load(Ordering::Relaxed)); + let _ = write!(out, " stall polls={} detected={} deferred={} doorbells={} fragment-doorbells={} cleared-after-doorbell={} cleared-unaided={} at-recovery={} max-ms={}", + STALL_POLLS.load(Ordering::Relaxed), STALL_DETECTED.load(Ordering::Relaxed), + STALL_DEFERRED.load(Ordering::Relaxed), STALL_DOORBELLS.load(Ordering::Relaxed), + STALL_FRAGMENT_DOORBELLS.load(Ordering::Relaxed), + STALL_CLEARED_AFTER_DOORBELL.load(Ordering::Relaxed), + STALL_CLEARED_UNAIDED.load(Ordering::Relaxed), STALL_AT_RECOVERY.load(Ordering::Relaxed), + STALL_MAX_NS.load(Ordering::Relaxed) / 1_000_000); + let _ = write!(out, " reannounce announced={} unannounced={} by-opcode:", + REANNOUNCE_ANNOUNCED.load(Ordering::Relaxed), REANNOUNCE_UNANNOUNCED.load(Ordering::Relaxed)); + for (opcode, counter) in REANNOUNCE_BY_OPCODE.iter().enumerate() { + let count = counter.load(Ordering::Relaxed); + if count != 0 { let _ = write!(out, " {:#x}={}", opcode, count); } + } + let _ = writeln!(out); + out.used as core::ffi::c_int +} + +static MASK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(mask_set), get: Some(mask_get), free: None, +}; +#[link_section = "__param"] +#[used] +static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_recovery_path" + } else { c"aurora.g17p_recovery_path" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&MASK_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/stackwatch.rs b/drivers/gpu/drm/asahi/g18/compat/stackwatch.rs new file mode 100644 index 0000000000000..2c74f17ef7dda --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/stackwatch.rs @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! DIAGNOSTIC (default off, read through sysfs only). +//! +//! `g17p_stack_watch` (root-only, runtime or boot "1"/"0"/"reset"): kernel +//! stack high-water mark of render publication. With it set, publish_with +//! paints the unused part of the task stack below the current SP and, after +//! publication returns, scans for the deepest overwritten word. The result +//! includes every callee (kernel printk, scheduler, exception entry). Kept +//! separately for the submit-ioctl early path and the scheduler worker path, +//! and for publications during which a first-kick KSMFE configure or a +//! registration power bracket ran. THREAD_SIZE is 32 KiB (MIN_THREAD_SHIFT 15, +//! 16K pages, VMAP_STACK). Never a default. +use core::fmt::Write; +use core::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +const THREAD_SIZE: usize = 1 << 15; +const GUARD: usize = 256; +const PATTERN: u64 = 0xa5a5_5a5a_c3c3_3c3c; + +// ---- stack watch ---- +static SW_ENABLED: AtomicU32 = AtomicU32::new(0); +/// [early, scheduler, first-install, bracket]: samples, max used, max SP depth at entry. +static SW_SAMPLES: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4]; +static SW_MAX_USED: [AtomicUsize; 4] = [const { AtomicUsize::new(0) }; 4]; +static SW_MAX_ENTRY: [AtomicUsize; 4] = [const { AtomicUsize::new(0) }; 4]; +static SW_SKIPPED: AtomicU64 = AtomicU64::new(0); +pub(crate) static FIRST_INSTALL_SEQ: AtomicU64 = AtomicU64::new(0); +pub(crate) static BRACKET_SEQ: AtomicU64 = AtomicU64::new(0); + +pub(crate) struct StackWatch { base: usize, top: usize, entry_depth: usize, seq: [u64; 2] } + +#[inline(always)] +fn current_sp() -> usize { + let sp: usize; + // SAFETY: reads the stack pointer; no memory access. + unsafe { core::arch::asm!("mov {}, sp", out(reg) sp, options(nomem, nostack, preserves_flags)) }; + sp +} + +/// Paint the unused stack below this frame (only while enabled). +#[inline(never)] +pub(crate) fn begin() -> Option { + if SW_ENABLED.load(Ordering::Relaxed) == 0 { return None; } + // SAFETY: `current` is always valid in process/kthread context. + let base = unsafe { (*bindings::get_current()).stack } as usize; + let sp = current_sp(); + let top = base.checked_add(THREAD_SIZE)?; + if base == 0 || sp <= base + GUARD + 64 || sp >= top { + SW_SKIPPED.fetch_add(1, Ordering::Relaxed); + return None; + } + let mut at = (base + GUARD + 7) & !7; + let end = (sp - 64) & !7; + while at < end { + // SAFETY: [base+GUARD, sp-64) is this task's own unused stack; the + // arm64 kernel ABI has no red zone and nothing live sits below SP. + unsafe { core::ptr::write_volatile(at as *mut u64, PATTERN) }; + at += 8; + } + Some(StackWatch { base, top, entry_depth: top - sp, + seq: [FIRST_INSTALL_SEQ.load(Ordering::Relaxed), BRACKET_SEQ.load(Ordering::Relaxed)] }) +} + +#[inline(never)] +pub(crate) fn end(watch: Option, early: bool) { + let Some(w) = watch else { return }; + let mut at = (w.base + GUARD + 7) & !7; + // SAFETY: scanning this task's own stack between the guard and the top. + while at < w.top && unsafe { core::ptr::read_volatile(at as *const u64) } == PATTERN { at += 8; } + let used = w.top - at; + let mut classes = [false; 4]; + classes[if early { 0 } else { 1 }] = true; + classes[2] = FIRST_INSTALL_SEQ.load(Ordering::Relaxed) != w.seq[0]; + classes[3] = BRACKET_SEQ.load(Ordering::Relaxed) != w.seq[1]; + for (index, set) in classes.into_iter().enumerate() { + if !set { continue; } + SW_SAMPLES[index].fetch_add(1, Ordering::Relaxed); + SW_MAX_USED[index].fetch_max(used, Ordering::Relaxed); + SW_MAX_ENTRY[index].fetch_max(w.entry_depth, Ordering::Relaxed); + } +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +fn param_text<'a>(value: *const core::ffi::c_char) -> Option<&'a str> { + if value.is_null() { return None; } + // SAFETY: the parameter core supplies a NUL-terminated string. + unsafe { core::ffi::CStr::from_ptr(value) }.to_str().ok().map(str::trim) +} + +unsafe extern "C" fn sw_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + match param_text(value) { + Some("reset") => { + for i in 0..4 { + SW_SAMPLES[i].store(0, Ordering::Relaxed); + SW_MAX_USED[i].store(0, Ordering::Relaxed); + SW_MAX_ENTRY[i].store(0, Ordering::Relaxed); + } + 0 + } + Some("1") => { SW_ENABLED.store(1, Ordering::Relaxed); 0 } + Some("0") => { SW_ENABLED.store(0, Ordering::Relaxed); 0 } + _ => -22, + } +} + +unsafe extern "C" fn sw_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let _ = write!(out, "enabled={} thread_size={} skipped={}", SW_ENABLED.load(Ordering::Relaxed), + THREAD_SIZE, SW_SKIPPED.load(Ordering::Relaxed)); + for (index, name) in ["early", "sched", "first_install", "bracket"].iter().enumerate() { + let used = SW_MAX_USED[index].load(Ordering::Relaxed); + let _ = write!(out, " {}:samples={}/max_used={}/margin={}/max_entry_depth={}", name, + SW_SAMPLES[index].load(Ordering::Relaxed), used, + if used == 0 { 0 } else { THREAD_SIZE as i64 - used as i64 }, + SW_MAX_ENTRY[index].load(Ordering::Relaxed)); + } + let _ = writeln!(out, " first_install_seq={} bracket_seq={}", + FIRST_INSTALL_SEQ.load(Ordering::Relaxed), BRACKET_SEQ.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} + +static SW_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(sw_set), get: Some(sw_get), free: None, +}; +#[link_section = "__param"] +#[used] +static SW_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_stack_watch" + } else { c"aurora.g17p_stack_watch" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&SW_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/timing_buffer.rs b/drivers/gpu/drm/asahi/g18/compat/timing_buffer.rs new file mode 100644 index 0000000000000..611465fa139b3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/timing_buffer.rs @@ -0,0 +1,929 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! One-shot diagnostic buffer. No allocation, printk, waiting or overwrite. +//! Atomic words avoid unsafe shared payload access; ready publishes each slot. +use core::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use core::fmt::{self, Write}; +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const WORDS: usize = 24; +const CAPACITY: usize = 16384; +struct Slot { ready: AtomicU32, words: [AtomicU64; WORDS] } +static SLOTS: [Slot; CAPACITY] = [const { Slot { + ready: AtomicU32::new(0), words: [const { AtomicU64::new(0) }; WORDS], +} }; CAPACITY]; +// 0 unarmed, 1 capturing, 2 permanently frozen until module unload. +static STATE: AtomicU32 = AtomicU32::new(0); +// Freeze and reservation share one atomic: stop cannot miss a late reservation. +const FROZEN: usize = 1usize << (usize::BITS - 1); +static NEXT: AtomicUsize = AtomicUsize::new(FROZEN); +static FROZEN_BOUND: AtomicUsize = AtomicUsize::new(0); +static CURSOR: AtomicUsize = AtomicUsize::new(0); +static DROPPED: AtomicU64 = AtomicU64::new(0); + +pub(crate) fn enabled() -> bool { STATE.load(Ordering::Relaxed) == 1 } +pub(crate) fn now() -> u64 { + ::ktime_get() as u64 +} + +/// Record kinds. 1/2: render admission/completion. 3/4: compute publication +/// return and exact retirement. 5: render publication sub-stages. Records +/// carry the architectural counter. Host and GPU rates must be converted +/// separately before testing whether their epochs align. +pub(crate) const KIND_RENDER_ADMISSION: u64 = 1; +pub(crate) const KIND_RENDER_COMPLETION: u64 = 2; +pub(crate) const KIND_COMPUTE_PUBLISH: u64 = 3; +pub(crate) const KIND_COMPUTE_RETIRE: u64 = 4; +pub(crate) const KIND_RENDER_PUBLISH_STAGES: u64 = 5; +/// Kind6: exact post-maintenance descriptor observations, only while armed. +/// Words1..7: pair slot, ordinal, TA/FRG descriptor VAs, physical TA/FRG kick +/// timestamps, packed TA/FRG QIDs. Words8..9: external pass envelope;10..13: +/// latest qualifying adjusted TA/FRG pair-B segments (24MHz).14: packed observed +/// retirement stamps;15: host counter;16: read-success;17: host frequency. +/// A zero/reversed pair is unavailable. Qualifying segments may be overwritten +/// on rearm; these are not aggregate execution or shader-only measurements. +pub(crate) const KIND_RENDER_DEVICE_SEGMENTS: u64 = 6; +/// Layout of the render completion record's words 10..19: TA start, TA end, +/// 3D start, 3D end (absolute GPU ticks), the architectural counter when the +/// fence is signalled (14), this version (15), then the service pass that +/// proved retirement on the same counter: last notification ingress on the +/// primary and secondary RTKit roles (16, 17), service-pass entry (18) and +/// the observation under the runtime mutex (19). Version 2 had words 10..14 +/// only, with word 14 sampled at observation. +pub(crate) const RENDER_COMPLETION_LAYOUT: u64 = 3; + +/// Host CNTPCT_EL0 ticks; `arch_frequency` gives their rate. Only read while armed. +/// Neo Linux exposes a 1 GHz physical timer while GPU command timestamps use +/// 24 MHz. Consumers must convert each clock separately before comparing them. +pub(crate) fn arch_counter() -> u64 { + let counter: u64; + // SAFETY: CNTPCT_EL0 is readable at EL1 on this platform (the driver's + // QoS publication already reads it) and has no memory side effects. + unsafe { + core::arch::asm!("mrs {c}, CNTPCT_EL0", c = out(reg) counter, + options(nomem, nostack, preserves_flags)); + } + counter +} + +pub(crate) fn arch_frequency() -> u64 { + let frequency: u64; + // SAFETY: CNTFRQ_EL0 is a read-only system register. + unsafe { + core::arch::asm!("mrs {f}, CNTFRQ_EL0", f = out(reg) frequency, + options(nomem, nostack, preserves_flags)); + } + frequency +} +pub(crate) fn record(words: [u64; WORDS]) { + if !enabled() { return; } + let next = NEXT.load(Ordering::Relaxed); + if next & FROZEN != 0 { return; } + if next >= CAPACITY { DROPPED.fetch_add(1, Ordering::Relaxed); return; } + // A freeze racing this reservation either includes this ticket in its + // finite drain bound, or this ticket observes FROZEN and writes nothing. + let index = NEXT.fetch_add(1, Ordering::Relaxed); + if index & FROZEN != 0 { return; } + if index >= CAPACITY { DROPPED.fetch_add(1, Ordering::Relaxed); return; } + let slot = &SLOTS[index]; + for (dst, src) in slot.words.iter().zip(words) { dst.store(src, Ordering::Relaxed); } + slot.ready.store(1, Ordering::Release); +} + +fn reserved() -> usize { + if STATE.load(Ordering::Acquire) == 2 { FROZEN_BOUND.load(Ordering::Relaxed) } + else { (NEXT.load(Ordering::Relaxed) & !FROZEN).min(CAPACITY) } +} + +// Limit each sysfs response to 4095 bytes, below PAGE_SIZE even on 4K kernels. +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} +unsafe extern "C" fn get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + // SAFETY: sysfs module parameter get supplies a PAGE_SIZE output buffer. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4095) }; + let mut out = Output { bytes, used: 0 }; + let mut cursor = CURSOR.load(Ordering::Relaxed); + // The kernel parameter mutex serializes readers/setters, never writers. + // Four records fit even when every u64 takes 20 decimal digits. + for _ in 0..4 { + if cursor >= reserved() { break; } + let slot = &SLOTS[cursor]; + if slot.ready.load(Ordering::Acquire) == 0 { break; } + let _ = write!(out, "record {}", cursor); + for word in &slot.words { let _ = write!(out, " {}", word.load(Ordering::Relaxed)); } + let _ = writeln!(out); + cursor += 1; + } + CURSOR.store(cursor, Ordering::Relaxed); + let reserved = reserved(); + let _ = writeln!(out, "status state={} cursor={} reserved={} dropped={} capacity={}", + STATE.load(Ordering::Relaxed), cursor, reserved, DROPPED.load(Ordering::Relaxed), CAPACITY); + out.used as core::ffi::c_int +} +unsafe extern "C" fn set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + // Bounded parser avoids strlen and rejects commands longer than 7 bytes. + let mut command = [0u8; 8]; + let mut len = 0; + while len < command.len() { + // SAFETY: parameter core supplies a NUL-terminated value; never read + // beyond the first NUL, and at most eight bytes are examined. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let arg = &command[..len]; + match arg { + b"1" | b"1\n" => { + if crate::g18::params::g17p_completion_trace.value() != 0 + || crate::g18::params::g17p_trace_depth.value() != 0 { return -22; } + if STATE.load(Ordering::Relaxed) != 0 { return -16; } + NEXT.store(0, Ordering::Relaxed); + STATE.store(1, Ordering::Relaxed); + } + b"0" | b"0\n" => { + if STATE.load(Ordering::Relaxed) != 2 { + FROZEN_BOUND.store((NEXT.fetch_or(FROZEN, Ordering::Relaxed) & !FROZEN).min(CAPACITY), Ordering::Relaxed); + STATE.store(2, Ordering::Release); + } + } + // A nonempty frozen capture proves that this module has published G17 + // work. Only the explicit root-only command reads fixed firmware DATA. + b"clock" | b"clock\n" => { + if STATE.load(Ordering::Acquire) != 2 || reserved() == 0 { return -22; } + let before = arch_counter(); + let snapshot = crate::g18::compat::g17p::diagnostic_accounting_clocks(); + let after = arch_counter(); + kernel::pr_info!("G17P accounting clocks: host-before={} host-after={} host-hz={} fields(TA,FRG,CL,all,freq-word,offset,sign-word,abs-offset)={:x?}\n", + before, after, arch_frequency(), snapshot); + if let Err(error) = snapshot { return error.to_errno(); } + } + // Rewind readout only. Slots remain immutable, including while capturing. + b"rewind" | b"rewind\n" => { CURSOR.store(0, Ordering::Relaxed); } + _ => return -22, + } + 0 +} +static OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(set), get: Some(get), free: None, +}; +#[link_section = "__param"] +#[used] +static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_timing_buffer" + } else { c"aurora.g17p_timing_buffer" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); + +// One explicit post-window snapshot. No live ring reads occur in record(). +// 0 unused, 1 requested, 2 copying, 3 complete, 4 failed; no reset until unload. +const RING_SLOTS: usize = 2048; +struct CompletionSlot { words: [AtomicU64; 8], changed: AtomicU32 } +static COMPLETION_SLOTS: [CompletionSlot; 2 * RING_SLOTS] = [const { CompletionSlot { + words: [const { AtomicU64::new(0) }; 8], changed: AtomicU32::new(0), +} }; 2 * RING_SLOTS]; +static COMPLETION_STATE: AtomicU32 = AtomicU32::new(0); +static COMPLETION_CURSOR: AtomicUsize = AtomicUsize::new(0); +static COMPLETION_COUNT: AtomicUsize = AtomicUsize::new(0); +static COMPLETION_ERROR: AtomicU32 = AtomicU32::new(0); +// request, copy-start/end CNTPCT, CNTFRQ, bias-before/after, ring0/2 DVA, +// accepted firmware recovery generation; then sixteen u32 table fields before +// and sixteen after the copy (eight per ordinal0/2 row). Non-atomic snapshots. +static COMPLETION_META: [AtomicU64; 49] = [const { AtomicU64::new(0) }; 49]; + +pub(crate) fn claim_completion_snapshot() -> bool { + COMPLETION_STATE.compare_exchange(1, 2, Ordering::AcqRel, Ordering::Acquire).is_ok() +} + +pub(crate) fn completion_snapshot_meta(index: usize, value: u64) { + if let Some(word) = COMPLETION_META.get(index) { word.store(value, Ordering::Relaxed); } +} + +pub(crate) fn store_completion_snapshot(index: usize, words: [u64; 8], changed: bool) { + let Some(slot) = COMPLETION_SLOTS.get(index) else { return; }; + for (dst, value) in slot.words.iter().zip(words) { dst.store(value, Ordering::Relaxed); } + slot.changed.store(u32::from(changed), Ordering::Relaxed); + COMPLETION_COUNT.store(index + 1, Ordering::Relaxed); +} + +pub(crate) fn finish_completion_snapshot(result: kernel::error::Result) { + completion_snapshot_meta(2, arch_counter()); + let error = result.err().map_or(0, |error| error.to_errno()); + COMPLETION_ERROR.store(error as u32, Ordering::Relaxed); + COMPLETION_STATE.store(if error == 0 { 3 } else { 4 }, Ordering::Release); +} + +unsafe extern "C" fn completion_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 8]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + match &command[..len] { + b"1" | b"1\n" => { + if !matches!(STATE.load(Ordering::Acquire), 1 | 2) || reserved() == 0 { return -22; } + // A claimed request is never overwritten. Reads remain unavailable + // until COMPLETE/FAILED acquire, including concurrent sysfs reads. + if COMPLETION_STATE.compare_exchange(0, 5, Ordering::AcqRel, Ordering::Acquire).is_err() { + return -16; + } + completion_snapshot_meta(0, arch_counter()); + COMPLETION_STATE.store(1, Ordering::Release); + } + b"rewind" | b"rewind\n" => { + if !matches!(COMPLETION_STATE.load(Ordering::Acquire), 3 | 4) { return -16; } + COMPLETION_CURSOR.store(0, Ordering::Relaxed); + } + _ => return -22, + } + 0 +} + +unsafe extern "C" fn completion_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4095) }; + let mut out = Output { bytes, used: 0 }; + let state = COMPLETION_STATE.load(Ordering::Acquire); + let count = if state == 3 || state == 4 { COMPLETION_COUNT.load(Ordering::Relaxed) } else { 0 }; + let cursor = COMPLETION_CURSOR.load(Ordering::Relaxed).min(count); + let end = (cursor + 12).min(count); + for index in cursor..end { + let slot = &COMPLETION_SLOTS[index]; + let ordinal = if index < RING_SLOTS { 0 } else { 2 }; + let _ = write!(out, "completion {} {} {}", ordinal, index % RING_SLOTS, + slot.changed.load(Ordering::Relaxed)); + for word in &slot.words { let _ = write!(out, " {:016x}", word.load(Ordering::Relaxed)); } + let _ = writeln!(out); + } + if end != cursor { COMPLETION_CURSOR.store(end, Ordering::Relaxed); } + let _ = write!(out, "completion-status state={} cursor={} count={} error={} meta=", + state, end, count, COMPLETION_ERROR.load(Ordering::Relaxed) as i32); + for word in &COMPLETION_META { let _ = write!(out, " {}", word.load(Ordering::Relaxed)); } + let _ = writeln!(out); + out.used as core::ffi::c_int +} +static COMPLETION_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(completion_set), get: Some(completion_get), free: None, +}; +#[link_section = "__param"] +#[used] +static COMPLETION_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_completion_buffer" + } else { c"aurora.g17p_completion_buffer" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&COMPLETION_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); + +/// Armed-only host observation gate; no additional device reads. Kind7 words: +/// slot, front ordinal (MAX if unavailable), reason (1borrowed,2restaged,3UMA, +/// 4stamp pending,5stamp ready,6notification mask pending), expected stamp, +/// observed TA/FRG stamps, host counter, host frequency, then four optional masks. +pub(crate) fn record_render_gate(slot: u8, ordinal: u64, reason: u64, + expected: u32, stamps: [u32; 2], masks: [u64; 4]) { + if !enabled() { return; } + let mut words = [0; WORDS]; + words[0] = 7; + words[1] = u64::from(slot); + words[2] = ordinal; + words[3] = reason; + words[4] = u64::from(expected); + words[5] = u64::from(stamps[0]); + words[6] = u64::from(stamps[1]); + words[7] = arch_counter(); + words[8] = arch_frequency(); + words[9..13].copy_from_slice(&masks); + record(words); +} + +static TSQ_GATE_SAMPLE: AtomicU32 = AtomicU32::new(0); +/// One in eight armed stamp-gate observations. Ordinary work never maps TSQ +/// diagnostic data. Kind8 is a non-atomic observation, never a completion gate. +pub(crate) fn sample_tsq_gate() -> bool { + enabled() && TSQ_GATE_SAMPLE.fetch_add(1, Ordering::Relaxed) & 7 == 0 +} + +/// Root-only runtime diagnostic knobs, `g17p_diag_knobs` (u32 bitmask, +/// decimal or 0x hex). Read when each render descriptor/kick is built, so a +/// change applies to the next submission and permits same-boot A/B. Cached +/// family bits affect fresh resource owners only. bit0: FRG +0x222d = 0 (native light +/// captures); bit1: omit the per-pass KSM MCache aperture (native final +/// commands carry pointer/count zero). +/// v84 production defaults (native-matching, hardware-qualified since v64/v67 +/// under 0x9000 in every throughput and stress run): bit12 native TA +/// force-drain value (+0x8c2 = 0) and bit15 aligned word record snapshots. +/// Bits27..30 cache native command-object families with reserved return storage +/// and final-stop/UAT lifetime checks. Writing the knob still permits A/B. +pub(crate) const DIAG_PRODUCTION_DEFAULT: u32 = (1 << 12) | (1 << 15) | (0xf << 27); +static DIAG_KNOBS: AtomicU32 = AtomicU32::new(DIAG_PRODUCTION_DEFAULT); +pub(crate) const DIAG_FRG_222D_CLEAR: u32 = 1 << 0; +pub(crate) const DIAG_NO_KSM_MCACHE: u32 = 1 << 1; +/// bit2: TA 0x0a5a1 PTC high word 0xfe (captured Metal value) instead of 0x60. +pub(crate) const DIAG_TA_PTC_NATIVE: u32 = 1 << 2; +/// bit3: FRG 0x0a5a9 PTC high word 0xd5 (captured Metal value) instead of 0x60. +pub(crate) const DIAG_FRG_PTC_NATIVE: u32 = 1 << 3; +/// bit4: hold firmware idle descent off (IdlePowerOff(0)) while set; the next +/// submission after clearing it restores the ordinary policy (arg 1). +pub(crate) const DIAG_IDLE_HOLD: u32 = 1 << 4; +/// bit5: copy the first 64 KiB of the host-owned primary state grid (Status-A +/// and Status-B) into `GRID` at the next submission, then clear this bit. +pub(crate) const DIAG_GRID_SNAPSHOT: u32 = 1 << 5; +/// bit6: native `updateSubmitInProgress` accounting. Maintain RegionC+0x7c +/// (submissions in progress) and +0x80 (monotonic submit count) around each +/// render/compute submission instead of leaving the static init values. +pub(crate) const DIAG_SUBMIT_ACCOUNTING: u32 = 1 << 6; +/// bit7: native kickFirmware note (EP 0x21 `0x0087_0000_0000_0010`) right after +/// the submit-in-progress bracket opens, as native `notifyFirmware(4, 1, 0)`. +pub(crate) const DIAG_SUBMIT_KICK: u32 = 1 << 7; +/// bit8: copy the live hardware-data block (shared cluster, 0x4000 bytes) +/// into `GRID` at the next submission, then clear this bit. +pub(crate) const DIAG_HWDATA_SNAPSHOT: u32 = 1 << 8; +/// bit9: CLPC deadline control effort. RegionC+0xe4 = 0x10000 (1.0 in 16.16) +/// with +0xe8 = 0 while set, as native CLPC writes during active streams; +/// clearing restores 0. GFX1 reads it live as a pstate floor (unit 9). +pub(crate) const DIAG_CLPC_EFFORT: u32 = 1 << 9; +pub(crate) fn clear_diag_knob(bit: u32) { DIAG_KNOBS.fetch_and(!bit, Ordering::Relaxed); } +pub(crate) const GRID_QWORDS: usize = 0x10000 / 8; +pub(crate) static GRID: [AtomicU64; GRID_QWORDS] = [const { AtomicU64::new(0) }; GRID_QWORDS]; +pub(crate) fn diag_knob(bit: u32) -> bool { DIAG_KNOBS.load(Ordering::Relaxed) & bit != 0 } +unsafe extern "C" fn diag_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 16]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + let parsed = match text.strip_prefix("0x") { + Some(hex) => u32::from_str_radix(hex, 16), + None => text.parse::(), + }; + let Ok(value) = parsed else { return -22; }; + DIAG_KNOBS.store(value, Ordering::Relaxed); + kernel::pr_info!("G17P diagnostic knobs = {:#x}\n", value); + 0 +} +unsafe extern "C" fn diag_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 512) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let _ = writeln!(out, "{:#x}", DIAG_KNOBS.load(Ordering::Relaxed)); + // Firmware DATA power bookkeeping (RAM, not MMIO; bounded single loads): + // 124ba0 power-up count, 1771f8 power state (2 on, 4 stage-1 idle), + // 177838 idle-off policy (RegionC+0x30 copy), 128a68 keep-on bits. + for (name, link) in [("powerups", 0xffff_fc00_0012_4ba0u64), ("state", 0xffff_fc00_0017_71f8), + ("idleoff", 0xffff_fc00_0017_7838), ("keepon", 0xffff_fc00_0012_8a68)] { + let value = crate::g18::compat::g17p::LiveFirmwareU64Probe::new( + crate::g18::compat::g17p::g17p_gfx_link_to_physical(link)) + .and_then(|probe| probe.observe_mask(u64::MAX, 1)) + .map(|sample| sample.first); + let _ = writeln!(out, "{} {:x?}", name, value); + } + out.used as core::ffi::c_int +} +/// Root-only firmware DATA peek, `g17p_fw_peek`: write a GFX LINK address in +/// the DATA segment (hex, 8-aligned); reading returns up to 64 qwords from it, +/// stopping at the page end. Bounded host reads of firmware RAM, no writes. +static PEEK_ADDR: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +const PEEK_DATA_START: u64 = 0xffff_fc00_0004_c000; +const PEEK_DATA_END: u64 = 0xffff_fc00_0018_0000; +unsafe extern "C" fn peek_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 24]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + let Ok(address) = u64::from_str_radix(text.strip_prefix("0x").unwrap_or(text), 16) else { return -22; }; + if address & 7 != 0 || !(PEEK_DATA_START..PEEK_DATA_END).contains(&address) { return -22; } + PEEK_ADDR.store(address, Ordering::Relaxed); + 0 +} +unsafe extern "C" fn peek_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let address = PEEK_ADDR.load(Ordering::Relaxed); + if address < PEEK_DATA_START { return 0; } + let page = crate::pgtable::UAT_PGSZ as u64; + let physical = crate::g18::compat::g17p::g17p_gfx_link_to_physical(address); + let room = ((page - (physical & (page - 1))) / 8).min(64) as usize; + let room = room.min(((PEEK_DATA_END - address) / 8) as usize); + let mut values = [0u64; 64]; + match crate::g18::compat::g17p::LiveFirmwareU64Probe::new(physical) + .and_then(|probe| probe.read_run(&mut values[..room])) { + Ok(()) => for (row, chunk) in values[..room].chunks(4).enumerate() { + let _ = write!(out, "{:x}:", address + row as u64 * 32); + for value in chunk { let _ = write!(out, " {:016x}", value); } + let _ = writeln!(out); + }, + Err(error) => { let _ = writeln!(out, "error {:?}", error); } + } + out.used as core::ffi::c_int +} +static PEEK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(peek_set), get: Some(peek_get), free: None, +}; +#[link_section = "__param"] +#[used] +static PEEK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_fw_peek" + } else { c"aurora.g17p_fw_peek" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PEEK_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +/// Root-only `g17p_grid_peek`: write a byte offset (hex, 8-aligned) into the +/// last `DIAG_GRID_SNAPSHOT` copy; reading returns up to 64 qwords from it. +static GRID_OFFSET: AtomicUsize = AtomicUsize::new(0); +unsafe extern "C" fn grid_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 24]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + let Ok(offset) = usize::from_str_radix(text.strip_prefix("0x").unwrap_or(text), 16) else { return -22; }; + if offset & 7 != 0 || offset >= GRID_QWORDS * 8 { return -22; } + GRID_OFFSET.store(offset, Ordering::Relaxed); + 0 +} +unsafe extern "C" fn grid_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let first = GRID_OFFSET.load(Ordering::Relaxed) / 8; + let last = (first + 64).min(GRID_QWORDS); + for row in (first..last).step_by(4) { + let _ = write!(out, "{:x}:", row * 8); + for slot in &GRID[row..(row + 4).min(last)] { + let _ = write!(out, " {:016x}", slot.load(Ordering::Relaxed)); + } + let _ = writeln!(out); + } + out.used as core::ffi::c_int +} +static GRID_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(grid_set), get: Some(grid_get), free: None, +}; +#[link_section = "__param"] +#[used] +static GRID_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_grid_peek" + } else { c"aurora.g17p_grid_peek" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&GRID_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +/// Root-only `g17p_gpc_sample`: writing N (1..=400000) samples the always-on +/// GPC performance state (sgx+0xe01000 & 0xf, J700 physical 0x4_80e0_1000) +/// N times, sleeping ~20 us between reads; reading prints the histogram of +/// states 0..15. Read-only MMIO of an always-on power register. +static GPC_HIST: [AtomicU64; 16] = [const { AtomicU64::new(0) }; 16]; +static GPC_SAMPLED_NS: AtomicU64 = AtomicU64::new(0); +const GPC_STATE_PHYS: u64 = 0x4_80e0_1000; +unsafe extern "C" fn gpc_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 16]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + let Ok(samples) = text.parse::() else { return -22; }; + if samples == 0 || samples > 400_000 { return -22; } + // SAFETY: maps one always-on 4-byte device register; unmapped below. + let base = unsafe { bindings::ioremap(GPC_STATE_PHYS as bindings::phys_addr_t, 4) }; + if base.is_null() { return -12; } + for slot in &GPC_HIST { slot.store(0, Ordering::Relaxed); } + let start = now(); + for _ in 0..samples { + // SAFETY: `base` is a live 4-byte MMIO mapping. + let state = unsafe { bindings::readl(base) } & 0xf; + GPC_HIST[state as usize].fetch_add(1, Ordering::Relaxed); + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(20)); + } + GPC_SAMPLED_NS.store(now().wrapping_sub(start), Ordering::Relaxed); + // SAFETY: `base` came from ioremap above and is not used afterwards. + unsafe { bindings::iounmap(base) }; + 0 +} +unsafe extern "C" fn gpc_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1000) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let _ = write!(out, "ns={}", GPC_SAMPLED_NS.load(Ordering::Relaxed)); + for (state, slot) in GPC_HIST.iter().enumerate() { + let count = slot.load(Ordering::Relaxed); + if count != 0 { let _ = write!(out, " s{}={}", state, count); } + } + let _ = writeln!(out); + out.used as core::ffi::c_int +} +static GPC_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(gpc_set), get: Some(gpc_get), free: None, +}; +#[link_section = "__param"] +#[used] +static GPC_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_gpc_sample" + } else { c"aurora.g17p_gpc_sample" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&GPC_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +static DIAG_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(diag_set), get: Some(diag_get), free: None, +}; +#[link_section = "__param"] +#[used] +static DIAG_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_diag_knobs" + } else { c"aurora.g17p_diag_knobs" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&DIAG_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +/// Opt-in `g17p_diag_knobs` bit10: one-shot log of host<->firmware traffic. +/// Each entry is [counter | kind << 56, a, b, c]; the log stops at capacity +/// until `g17p_msglog` is written "reset". Logging only; changes no behaviour. +pub(crate) const DIAG_MSGLOG: u32 = 1 << 10; +/// a=endpoint, b=message, c=RTKit instance address (role tag). +pub(crate) const MSG_SEND: u64 = 1; +/// a=endpoint | processor << 8, b=message. Hard-IRQ callback. +pub(crate) const MSG_RECV_EARLY: u64 = 2; +/// a=endpoint | processor << 8, b=message. Workqueue callback. +pub(crate) const MSG_RECV: u64 = 3; +/// a=opcode, b=arg, c=producer before | consumer before << 32. +pub(crate) const MSG_DEVICE_CONTROL: u64 = 4; +/// a=role | consumer << 8 | producer << 32, b/c=record qwords 0/1. +pub(crate) const MSG_EVENT: u64 = 5; +/// a/b/c=record qwords 2/3/4 of the preceding MSG_EVENT. +pub(crate) const MSG_EVENT_TAIL: u64 = 6; +/// a=producer before, b=command VA, c=producer after (compute channel). +pub(crate) const MSG_CHANNEL: u64 = 7; +/// a=render slot | 1 << 32 for compute, b=phase (0 begin, 1 end). +pub(crate) const MSG_SUBMIT: u64 = 8; +/// a=role | consumer << 8 | producer << 32, b=consumed (stats ring). +pub(crate) const MSG_STATS: u64 = 9; +/// a/b/c=first three qwords of the statistics record at the consumer. +pub(crate) const MSG_STATS_RECORD: u64 = 10; +/// Opt-in bit11: consume both roles' statistics rings on every event-worker +/// pass, as native does on every interrupt. Diagnostic; default off. +pub(crate) const DIAG_STATS_DRAIN: u32 = 1 << 11; +/// Opt-in bit12: use the existing native-TA-drain probe for ordinary shared +/// TSQ render jobs. Keep the bootstrap and split-launch paths unchanged. +/// This permits same-boot A/B of combined TA/FRG visibility publication; +/// mandatory cache maintenance and both final-stamp retirement checks stay. +pub(crate) const DIAG_NATIVE_TA_DRAIN: u32 = 1 << 12; +/// Opt-in bit13: announce an eligible render submission before descriptor +/// preparation, to test overlap of native wake protocol with CPU preparation. +/// No clock, idle-policy, dependency or queue-capacity changes. +pub(crate) const DIAG_EARLY_RENDER_KICK: u32 = 1 << 13; +/// a=firmware timestamp (24 MHz), b=key | thread << 16 | secondary << 24 +/// | flag << 32, c=arg0. Logged instead of printed while bit10 is set. +pub(crate) const MSG_KTRACE: u64 = 11; +/// a/b/c=args 1..3 of the preceding MSG_KTRACE. +pub(crate) const MSG_KTRACE_ARGS: u64 = 12; +/// v93: a=record qword 3 (+0x18; GPU pstate index in type-0x10 records), +/// b=role, c=consumer, following the MSG_STATS_RECORD of the same record. +pub(crate) const MSG_STATS_RECORD_Q3: u64 = 13; +const MSGLOG_ENTRIES: usize = 8192; +static MSGLOG: [[AtomicU64; 4]; MSGLOG_ENTRIES] = + [const { [const { AtomicU64::new(0) }; 4] }; MSGLOG_ENTRIES]; +static MSGLOG_NEXT: AtomicUsize = AtomicUsize::new(0); +static MSGLOG_READ: AtomicUsize = AtomicUsize::new(0); +pub(crate) fn msglog(kind: u64, a: u64, b: u64, c: u64) { + if !diag_knob(DIAG_MSGLOG) { return; } + let index = MSGLOG_NEXT.fetch_add(1, Ordering::Relaxed); + let Some(slot) = MSGLOG.get(index) else { return; }; + slot[1].store(a, Ordering::Relaxed); + slot[2].store(b, Ordering::Relaxed); + slot[3].store(c, Ordering::Relaxed); + slot[0].store((arch_counter() & 0x00ff_ffff_ffff_ffff) | (kind << 56), Ordering::Release); +} +/// Root-only `g17p_msglog`: write "reset" to clear, or a hex entry index to +/// choose where reading starts; reading prints up to 40 entries from there. +unsafe extern "C" fn msglog_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 24]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + if text == "reset" { + for slot in &MSGLOG { slot[0].store(0, Ordering::Relaxed); } + MSGLOG_READ.store(0, Ordering::Relaxed); + MSGLOG_NEXT.store(0, Ordering::Release); + return 0; + } + let Ok(index) = usize::from_str_radix(text.strip_prefix("0x").unwrap_or(text), 16) else { return -22; }; + if index >= MSGLOG_ENTRIES { return -22; } + MSGLOG_READ.store(index, Ordering::Relaxed); + 0 +} +unsafe extern "C" fn msglog_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let next = MSGLOG_NEXT.load(Ordering::Acquire).min(MSGLOG_ENTRIES); + let first = MSGLOG_READ.load(Ordering::Relaxed); + let _ = writeln!(out, "n={:x}", next); + for index in first..(first + 40).min(next) { + let slot = &MSGLOG[index]; + let head = slot[0].load(Ordering::Acquire); + let _ = writeln!(out, "{:x} {:x} {} {:x} {:x} {:x}", index, + head & 0x00ff_ffff_ffff_ffff, head >> 56, slot[1].load(Ordering::Relaxed), + slot[2].load(Ordering::Relaxed), slot[3].load(Ordering::Relaxed)); + } + out.used as core::ffi::c_int +} +static MSGLOG_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(msglog_set), get: Some(msglog_get), free: None, +}; +#[link_section = "__param"] +#[used] +static MSGLOG_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_msglog" + } else { c"aurora.g17p_msglog" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&MSGLOG_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); + +/// Opt-in bit14: request a work-ring scan before eligible render preparation. +pub(crate) const DIAG_EARLY_RENDER_DOORBELL: u32 = 1 << 14; + +/// Retired bit15: published firmware records are always snapshotted with +/// aligned word reads (g17p_snapshot_fw_record); the bit is ignored. + +/// Opt-in bit16: early work scan only when host tracks no submitted GPU work. +pub(crate) const DIAG_IDLE_RENDER_DOORBELL: u32 = 1 << 16; + +/// v92c bits20..22: DISABLE switches for the default-on fault guards (all +/// clear = guarded; the lab's 0x9000/0x4009000 writes keep every guard on). +/// bit20: the 1 ms teardown poll no longer settles consumed FList releases. +pub(crate) const DIAG_TEARDOWN_SETTLE_OFF: u32 = 1 << 20; +/// bit21: a new owner context may take over a render pair whose FList still +/// carries an unobserved 0x2e release witness (the pre-v92c behaviour). +pub(crate) const DIAG_FLIST_TAKEOVER_GUARD_OFF: u32 = 1 << 21; +/// bit22: no device-control re-announcement while control records stay +/// unconsumed (teardown poll and takeover wait). +pub(crate) const DIAG_CONTROL_REANNOUNCE_OFF: u32 = 1 << 22; + +/// v92d DIAGNOSTIC bit23 (default off): write MCache selector 1 (native's +/// captured value) into every fragment MCache aperture entry instead of the +/// render's own application-GART context ID. Native's value is its process's +/// GART context ID (AGXContextIDManager::alloc -> TA descriptor +0x1274 -> +/// halMCacheApertureSetup bits 41..46); on Linux context 1 is a different +/// address space, so this arm tests whether the aperture match matters at +/// all. It is not native parity. Each patched table is read back and must +/// differ from the default only in those six bits. +pub(crate) const DIAG_MCACHE_SELECTOR_NATIVE: u32 = 1 << 23; + +/// v89 A/B-only bit26: suspend the default native both-role trace/statistics +/// ring consumers at runtime (the pre-v89 diagnostic-gated behaviour). +pub(crate) const DIAG_NATIVE_RING_DRAIN_OFF: u32 = 1 << 26; +/// bit25: A/B only. Publish the compute tag-16 EntrySignal on every repeat CL +/// kick, as before the native per-command policy. Default clear: only a kick +/// whose barriers name another hardware queue publishes one. +pub(crate) const DIAG_CL_ENTRY_SIGNAL_EVERY_KICK: u32 = 1 << 25; + +/// v90 bits27..30: allocate render descriptors / TSQ node page / ChannelState +/// prefix / ConfigUpdate as pooled firmware-cached objects (native bit41). +/// Sampled for fresh allocations only; existing objects never change. +pub(crate) fn fw_cached_family_mask() -> u32 { + (DIAG_KNOBS.load(Ordering::Relaxed) >> 27) & 0xf +} + +// --------------------------------------------------------------------------- +/// v93 statistics pstate sampler: root-only `g17p_stats_pstate`, default 0. +/// +/// Writing 1 makes the (default, native-parity) statistics-ring drain also +/// read four u32 words of every drained record -- +0x04/+0x08 (timestamp, as +/// one u64) and +0x18/+0x1c (qword 3; the GPU pstate index of type-0x10 +/// records) -- before it publishes the consumer, and record them here. The +/// drain's legality check, consumer publication and order are unchanged; a +/// failed sample read is counted and skipped, never propagated. Written 0 +/// (default) the drain pays one relaxed load per pass. "reset" clears. +/// +/// Reading prints cumulative counters (difference two reads for a window; +/// the lab samples at 4 Hz), the latest type-0x10 sample, per-pstate record +/// counts and timestamp ticks (each type-0x10 delta is attributed to the +/// previous record's pstate, per role), then the newest ring entries. +/// Buckets are the low byte of qword 3, 16 meaning >= 16. The pstate +/// meaning of qword 3 is taken from native RE, not verified on Linux. +pub(crate) const STATS_PSTATE_TYPE: u32 = 0x10; +const PSTATE_BUCKETS: usize = 17; +const PSTATE_RING: usize = 64; +static PSTATE_ON: AtomicU32 = AtomicU32::new(0); +static PSTATE_RECORDS: AtomicU64 = AtomicU64::new(0); +static PSTATE_TYPE10: AtomicU64 = AtomicU64::new(0); +static PSTATE_READ_FAILS: AtomicU64 = AtomicU64::new(0); +/// Latest type-0x10 sample: timestamp, qword 3, role. +static PSTATE_LAST: [AtomicU64; 3] = [const { AtomicU64::new(0) }; 3]; +static PSTATE_COUNT: [AtomicU64; PSTATE_BUCKETS] = [const { AtomicU64::new(0) }; PSTATE_BUCKETS]; +static PSTATE_TICKS: [AtomicU64; PSTATE_BUCKETS] = [const { AtomicU64::new(0) }; PSTATE_BUCKETS]; +/// Per role: previous type-0x10 timestamp and bucket + 1 (0 = none yet). +static PSTATE_PREV: [[AtomicU64; 2]; 2] = [const { [const { AtomicU64::new(0) }; 2] }; 2]; +/// (timestamp, qword 3 with role in bit 63). +static PSTATE_SAMPLES: [[AtomicU64; 2]; PSTATE_RING] = [const { [const { AtomicU64::new(0) }; 2] }; PSTATE_RING]; +static PSTATE_NEXT: AtomicUsize = AtomicUsize::new(0); + +#[inline(always)] +pub(crate) fn stats_pstate_enabled() -> bool { PSTATE_ON.load(Ordering::Relaxed) != 0 } + +pub(crate) fn stats_pstate_read_failed() { PSTATE_READ_FAILS.fetch_add(1, Ordering::Relaxed); } + +/// Pure bucket choice (low byte of qword 3; 16 = out of range). +pub(crate) const fn stats_pstate_bucket(qword3: u64) -> usize { + let index = (qword3 & 0xff) as usize; + if index < PSTATE_BUCKETS - 1 { index } else { PSTATE_BUCKETS - 1 } +} + +/// Record one drained statistics record (single writer: the drain runs +/// under the runtime mutex; readers only load). +pub(crate) fn stats_pstate_note(role: usize, entry_type: u32, timestamp: u64, qword3: u64) { + PSTATE_RECORDS.fetch_add(1, Ordering::Relaxed); + if entry_type != STATS_PSTATE_TYPE { return; } + let role = role.min(1); + PSTATE_TYPE10.fetch_add(1, Ordering::Relaxed); + let bucket = stats_pstate_bucket(qword3); + PSTATE_COUNT[bucket].fetch_add(1, Ordering::Relaxed); + let previous = PSTATE_PREV[role][1].load(Ordering::Relaxed); + let previous_ts = PSTATE_PREV[role][0].load(Ordering::Relaxed); + if previous != 0 && timestamp > previous_ts { + PSTATE_TICKS[(previous - 1) as usize % PSTATE_BUCKETS] + .fetch_add(timestamp - previous_ts, Ordering::Relaxed); + } + PSTATE_PREV[role][0].store(timestamp, Ordering::Relaxed); + PSTATE_PREV[role][1].store(bucket as u64 + 1, Ordering::Relaxed); + PSTATE_LAST[0].store(timestamp, Ordering::Relaxed); + PSTATE_LAST[1].store(qword3, Ordering::Relaxed); + PSTATE_LAST[2].store(role as u64, Ordering::Relaxed); + let slot = PSTATE_NEXT.fetch_add(1, Ordering::Relaxed) % PSTATE_RING; + PSTATE_SAMPLES[slot][0].store(timestamp, Ordering::Relaxed); + PSTATE_SAMPLES[slot][1].store((qword3 & !(1 << 63)) | (role as u64) << 63, Ordering::Relaxed); +} + +unsafe extern "C" fn pstate_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 16]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + match text { + "0" => PSTATE_ON.store(0, Ordering::Release), + "1" => PSTATE_ON.store(1, Ordering::Release), + "reset" => { + for counter in [&PSTATE_RECORDS, &PSTATE_TYPE10, &PSTATE_READ_FAILS] { + counter.store(0, Ordering::Relaxed); + } + for counter in PSTATE_LAST.iter().chain(PSTATE_COUNT.iter()).chain(PSTATE_TICKS.iter()) { + counter.store(0, Ordering::Relaxed); + } + for role in &PSTATE_PREV { role[0].store(0, Ordering::Relaxed); role[1].store(0, Ordering::Relaxed); } + for slot in &PSTATE_SAMPLES { slot[0].store(0, Ordering::Relaxed); slot[1].store(0, Ordering::Relaxed); } + PSTATE_NEXT.store(0, Ordering::Release); + } + _ => return -22, + } + 0 +} +unsafe extern "C" fn pstate_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let _ = writeln!(out, "on={} records={} type10={} read_fails={} last_ts={:#x} last_q3={:#x} last_role={} now={:#x}", + PSTATE_ON.load(Ordering::Relaxed), PSTATE_RECORDS.load(Ordering::Relaxed), + PSTATE_TYPE10.load(Ordering::Relaxed), PSTATE_READ_FAILS.load(Ordering::Relaxed), + PSTATE_LAST[0].load(Ordering::Relaxed), PSTATE_LAST[1].load(Ordering::Relaxed), + PSTATE_LAST[2].load(Ordering::Relaxed), arch_counter()); + let _ = write!(out, "count"); + for counter in &PSTATE_COUNT { let _ = write!(out, " {}", counter.load(Ordering::Relaxed)); } + let _ = write!(out, "\nticks"); + for counter in &PSTATE_TICKS { let _ = write!(out, " {}", counter.load(Ordering::Relaxed)); } + let _ = writeln!(out); + let next = PSTATE_NEXT.load(Ordering::Acquire); + let first = next.saturating_sub(PSTATE_RING); + for index in first..next { + let slot = &PSTATE_SAMPLES[index % PSTATE_RING]; + let _ = writeln!(out, "{:x} {:x}", slot[0].load(Ordering::Relaxed), slot[1].load(Ordering::Relaxed)); + } + out.used as core::ffi::c_int +} +static PSTATE_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(pstate_set), get: Some(pstate_get), free: None, +}; +#[link_section = "__param"] +#[used] +static PSTATE_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_stats_pstate" + } else { c"aurora.g17p_stats_pstate" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PSTATE_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/debug.rs b/drivers/gpu/drm/asahi/g18/debug.rs new file mode 100644 index 0000000000000..99b28228eced1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/debug.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 kick, frame-pass and resource logging policy (`g18_kick_log`). +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. +use core::sync::atomic::{AtomicBool, Ordering}; +use crate::g18::params; + +/// Set once the G18 application DRM node is live (`G18_DRM_READY`). +static G18_DRM_LIVE: AtomicBool = AtomicBool::new(false); + +/// Mark the G18 application DRM node live. Output before this is unaffected. +pub(crate) fn g18_mark_drm_live() { + G18_DRM_LIVE.store(true, Ordering::Relaxed); +} + +/// Whether a per-kick, frame-pass or per-job resource line is printed: always +/// before `G18_DRM_READY`, afterwards only with `g18_kick_log=1`. Fault, +/// recovery, refusal, wait, stall and admission lines never consult this. +#[inline(always)] +pub(crate) fn g18_kick_log() -> bool { + params::g18_kick_log.value() != 0 || !G18_DRM_LIVE.load(Ordering::Relaxed) +} + +/// dev_info!() for a G18 per-kick, frame-pass or per-job resource line +#[macro_export] +macro_rules! g18_kick_info ( + ($dev:expr, $($arg:tt)*) => ( + if $crate::g18::debug::g18_kick_log() { + ::kernel::dev_info! ( $dev, $($arg)* ); + } + ) +); + +/// pr_info!() for a G18 per-kick, frame-pass or per-job resource line +#[macro_export] +macro_rules! g18_kick_pr_info ( + ($($arg:tt)*) => ( + if $crate::g18::debug::g18_kick_log() { + ::kernel::pr_info! ( $($arg)* ); + } + ) +); + +/// dev_info!() for a firmware event record: only a `routine` record follows +/// `g18_kick_log`; every other record is always printed +#[macro_export] +macro_rules! g18_event_info ( + ($routine:expr, $dev:expr, $($arg:tt)*) => ( + if !($routine) || $crate::g18::debug::g18_kick_log() { + ::kernel::dev_info! ( $dev, $($arg)* ); + } + ) +); diff --git a/drivers/gpu/drm/asahi/g18/drm.rs b/drivers/gpu/drm/asahi/g18/drm.rs new file mode 100644 index 0000000000000..8dfd44c87b5a0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/drm.rs @@ -0,0 +1,583 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G18 (J873, T8152/G18G) DRM render-node adapter. +//! +//! A structural copy of the M5 adapter (`m5_drm.rs`): synchronous +//! submission, one hardware command per submit, out-fences signalled by a +//! publication guard. It differs where G18 differs: +//! - render commands go through `g18_application` to a job sink the G18 +//! runtime implements, not through a captured reference profile; +//! - `map_timestamp_buffer` is implemented (contract K8) with the shared +//! CPU-written timestamp buffers, so Mesa's G18G screen exposes timer +//! queries and with them GL 3.3-4.6; +//! - compute is refused unless the adapter is built with `admit_compute`. +//! +//! Commands use the shared command layout (`include/uapi/drm/asahi_drm.h`); +//! `g18::uapi::command` parses it. The adapter holds no firmware state. +//! Everything it needs from the live G18 session goes through +//! [`G18DrmRuntime`]. + +use core::ops::Range; +use kernel::{bindings, macros::vtable, new_mutex, prelude::*, static_lock_class, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + sync::{Arc, Mutex, SpinLock}, uapi, xarray}; +use crate::{alloc, driver::AsahiRegData, file, g18::uapi::command as g17_uapi, g18::hw, + g18::uapi::application::{self as g18_application, G18ApplicationRefusal, G18RenderJobSink, G18RenderRetirement}, + gem, gpu, g18::fw::vm_layout as m5_vm, mmu, queue, stats, stats_raw, timestamp}; + +const USER_END: u64 = (1 << 42) - 2 * mmu::UAT_PGSZ as u64; +/// Largest compute control stream admitted, in bytes. The kernel does not read +/// the stream (the CDM does); this only bounds what the firmware is pointed at. +/// 64 KiB is a sizing choice, not a hardware limit: no decode names one. It +/// admits a stream of many launches, which cost 2.10 ms per op on the M6 against +/// 10.45 ms when each op is its own stream. Only reachable with `admit_compute`. +pub(crate) const G18_COMPUTE_STREAM_BYTES: u64 = 64 * 1024; +/// `DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE`: the dispatch binds texture or image +/// state, so the command takes the native texture compute mode (d62a, +/// `g18_application_compute_resources.rs`). +const G18_COMPUTE_TEXTURE_STATE: u32 = + uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE as u32; + +// `g18::uapi::command` parses by byte offset (it is also built on the host); +// keep its constants equal to the UAPI header. +const _: () = { + use core::mem::{offset_of, size_of}; + assert!(g17_uapi::UAPI_RENDER_SIZE == size_of::()); + assert!(g17_uapi::UAPI_COMPUTE_SIZE == size_of::()); + assert!(g17_uapi::UAPI_G18_RENDER_SIZE == size_of::()); + assert!(g17_uapi::UAPI_G18_COMPUTE_SIZE == size_of::()); + assert!(offset_of!(uapi::drm_asahi_cmd_render_g18, resources) == g17_uapi::UAPI_RENDER_SIZE); + assert!(offset_of!(uapi::drm_asahi_cmd_compute_g18, resources) == g17_uapi::UAPI_COMPUTE_SIZE); + assert!(size_of::() == 80); + assert!(size_of::() == 64); + assert!(offset_of!(uapi::drm_asahi_cmd_render, ray_query) == 0x150); + assert!(offset_of!(uapi::drm_asahi_cmd_compute, prior_vdm) == 0x88); + assert!(offset_of!(uapi::drm_asahi_cmd_render, ts_frag_end_copies) == 0x100); + assert!(offset_of!(uapi::drm_asahi_cmd_render, prior_vdm) == 0x140); + assert!(offset_of!(uapi::drm_asahi_cmd_render, prior_cdm) == 0x148); + assert!(offset_of!(uapi::drm_asahi_cmd_compute, ts_end_copies) == 0x40); + assert!(offset_of!(uapi::drm_asahi_cmd_compute, scs_layout) == 0x80); + assert!(offset_of!(uapi::drm_asahi_cmd_compute, prior_cdm) == 0x90); + assert!(offset_of!(uapi::drm_asahi_cmd_compute, cdm_auxiliary_allocation) == 0x98); + assert!(g17_uapi::RENDER_G18_RESOURCES + == uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_G18_RESOURCES as u32); + assert!(g17_uapi::COMPUTE_G18_RESOURCES + == uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_G18_RESOURCES as u32); +}; + +// Identity Mesa's G18G bring-up gate accepts (`agx_chip.c:471-477`) and the +// drm-shim `g18g` profile publishes (`asahi_noop.c:129-147`), measured on +// J873: two clusters of six cores (ioreg core_mask_list=(63,63)). +const G18_CHIP_ID: u32 = 0x8152; +const G18_CLUSTERS: u32 = 2; +const G18_CORES_PER_CLUSTER: u32 = 6; +const G18_CORE_MASK: u32 = 0x3f; +const G18_USC_GENERATION: u32 = 3; +/// Revision B1, what the Linux SGX ID probe decodes on J873. Used only when +/// the runtime supplies no observed identity. +const G18_REVISION_B1: u32 = hw::GpuRevision::B1 as u32; +/// `DRM_ASAHI_GPU_HAL_300`: the UAPI has no HAL400 value yet and Mesa's +/// G18G gate requires 300. Apple's kext is PI_400. +const G18_UAPI_HAL_GENERATION: u32 = uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_300 as u32; + +static QUEUE_NAME: &CStr = c"g18_queue"; +static QUEUE_CLASS_KEY: Pin<&'static kernel::sync::LockClassKey> = static_lock_class!(); + +/// What the adapter needs from the live G18 session. The conductor +/// implements it over the G18 runtime once firmware and the graphics +/// queues are up (see the G18 DRM adapter notes). +pub(crate) trait G18DrmRuntime: G18RenderJobSink { + /// Any crash, fault or quarantine that ends submission for this boot. + fn is_crashed(&self) -> bool; + /// VM_CREATE: a user VM in the session UAT. The adapter has already + /// checked `kernel_range` against the M5/Mesa compact contract. + fn new_user_vm(&self, id: u64, kernel_range: Range) -> Result; + /// Compute (K1). Only called when the adapter admits compute. + fn submit_compute(&self, _vm: &mmu::Vm, _command: &g17_uapi::TranslatedComputeCommand) + -> Result { + Err(ENOTSUPP) + } +} + +/// Adapter options fixed at registration. +#[derive(Debug, Copy, Clone, Default)] +pub(crate) struct G18DrmConfig { + /// Admit compute commands (module parameter `g18_admit_compute`, default 0). + pub(crate) admit_compute: bool, + /// Admit exactly the native 4x MSAA render set (module parameter + /// `g18_admit_msaa4`, default 0; msaa163). Off refuses every + /// multisampled render as before. + pub(crate) admit_msaa4: bool, + /// Admit renders carrying `DRM_ASAHI_RENDER_G18_RESOURCES` (module + /// parameter `g18_admit_graphics_scratch`, default 0). Off leaves the + /// feature unadvertised and refuses the flag, so Mesa keeps refusing + /// draws whose programs need a thread-private frame. + pub(crate) admit_graphics_scratch: bool, +} + +pub(crate) struct G18DrmGpu { + runtime: Arc, + /// Serializes job-sink calls across every queue of this device. + submit_lock: Arc>, + config: G18DrmConfig, + revision: u32, + ids: gpu::SequenceIDs, +} + +impl G18DrmGpu { + /// `observed` is the SGX identity read after firmware start, if the + /// runtime has one. It must match the published topology exactly. + pub(crate) fn new(runtime: Arc, observed: Option<&hw::GpuIdConfig>, + config: G18DrmConfig) -> Result { + let revision = match observed { + Some(id) => { + if id.gpu_gen != hw::GpuGen::G18 || id.gpu_variant != hw::GpuVariant::G + || id.num_dies != 1 || id.num_clusters != G18_CLUSTERS + || id.num_cores != G18_CORES_PER_CLUSTER + || id.usc_generation != G18_USC_GENERATION + || id.core_masks.as_slice() != [G18_CORE_MASK; G18_CLUSTERS as usize] { + pr_err!("G18_DRM: observed identity {:?} is not the J873 G18G topology\n", id); + return Err(ENODEV); + } + id.gpu_rev as u32 + } + None => G18_REVISION_B1, + }; + let submit_lock = Arc::pin_init(new_mutex!(()), GFP_KERNEL)?; + Ok(Self { runtime, submit_lock, config, revision, ids: Default::default() }) + } +} + +// Module 1134 reached G18 through its `DrmGpu` trait (drm_gpu.rs) and +// computed GET_PARAMS in file.rs from `DrmGpu::params`; here the same facts +// fill `drm_asahi_params_global` through `gpu::Gpu::get_params`. The shared +// file layer adds VM_STATUS and VM_STATUS_MIRROR (supports_vm_status) and +// SOFT_FAULTS when fault_control is 0xb; there are no queue limits. +impl gpu::Gpu for G18DrmGpu { + fn stats_snapshot(&self) -> Option> { None } + fn raw_stats_snapshot(&self) -> Option>> { None } + fn as_any(&self) -> &dyn core::any::Any { self } + fn arc_as_any(self: Arc) -> Arc { self } + fn manager(&self) -> Option<&dyn gpu::GpuManager> { None } + fn init(&self) -> Result { + if self.is_crashed() { Err(ENODEV) } else { Ok(()) } + } + fn ids(&self) -> &gpu::SequenceIDs { &self.ids } + fn is_crashed(&self) -> bool { self.runtime.is_crashed() } + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + let mut core_masks = [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize]; + for mask in core_masks.get_mut(..G18_CLUSTERS as usize).ok_or(EIO)? { + *mask = G18_CORE_MASK; + } + params.gpu_generation = hw::GpuGen::G18 as u32; + params.gpu_variant = hw::GpuVariant::G as u32; + params.gpu_revision = self.revision; + params.chip_id = G18_CHIP_ID; + params.num_dies = 1; + params.num_clusters_total = G18_CLUSTERS; + params.num_cores_per_cluster = G18_CORES_PER_CLUSTER; + params.core_masks = core_masks.map(|mask| mask.into()); + // No G18 clock policy is implemented by this adapter. + params.max_frequency_khz = 0; + params.usc_generation = G18_USC_GENERATION; + params.gpu_hal_generation = G18_UAPI_HAL_GENERATION; + params.max_commands_per_submission = 1; + if self.config.admit_graphics_scratch { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH as u64; + } + params.features |= uapi::DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES as u64; + Ok(()) + } + fn supports_vm_status(&self) -> bool { true } + /// VM_BIND validates every op before committing any, and SUBMIT copies + /// the command buffer before it resolves the syncs (module 1134 did both). + fn tracked_submission(&self) -> bool { true } + /// `DRM_IOCTL_ASAHI_GET_TIME` scales CNTPCT_EL0 by this rate; report the + /// counter's own frequency, as G17P does. J873 runs it at 24 MHz. + fn base_clock_hz(&self) -> u32 { counter_frequency() as u32 } + /// 42-bit T8152 user space: user range UAT_PGSZ..USER_END with the dummy + /// page at USER_END, as in module 1134. + fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } + fn new_vm(&self, kernel_range: Range) -> Result { + if self.is_crashed() { return Err(ENODEV); } + m5_vm::compact_range(&kernel_range).ok_or(EINVAL)?; + let vm = self.runtime.new_user_vm(self.ids.vm.next(), kernel_range)?; + // Module 1134 VM_CREATE: a simplefb alias is T8140-only; any other + // GPU refuses the VM when the diagnostic parameter is set. + if crate::g18::params::g17p_simplefb_iova.value() != 0 { + return Err(ENOTSUPP); + } + Ok(vm) + } + fn new_queue(&self, vm: mmu::Vm, + _ualloc: Arc>, + _ualloc_priv: Arc>, + priority: u32, usc_exec_base: u64, + _usage: Arc) -> Result> { + if priority > 3 { return Err(EINVAL); } + let window = g17_uapi::QueueUscWindow { + base: usc_exec_base, user_start: m5_vm::DRIVER_BASE, user_end: USER_END, + }; + window.validate().map_err(|_| EINVAL)?; + let queue = G18Queue { + runtime: self.runtime.clone(), submit_lock: self.submit_lock.clone(), + config: self.config, vm, window, + fences: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, + }; + Ok(KBox::new(queue, GFP_KERNEL)?) + } + /// K8. Mesa binds `128 * sizeof(struct agx_timestamps)` of a WRITEBACK BO + /// at screen creation and enables timer queries only if this works. G18 + /// publishes timestamps from the CPU after retirement, so the buffer needs + /// no firmware mapping. + fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) + -> Result { + if self.is_crashed() { return Err(ENODEV); } + if range.is_empty() { return Err(EINVAL); } + timestamp::Buffer::cpu(bo, range) + } +} + +/// CNTFRQ_EL0, or Apple's 24 MHz if boot firmware left it unprogrammed: +/// GET_TIME divides by this rate (`file.rs:1225-1229`) and must never see 0. +fn counter_frequency() -> u64 { + let frequency: u64; + // SAFETY: CNTFRQ_EL0 is a read-only system register. + unsafe { core::arch::asm!("mrs {x}, CNTFRQ_EL0", x = out(reg) frequency) }; + if frequency == 0 { 24_000_000 } else { frequency } +} + +/// The counter `DRM_IOCTL_ASAHI_GET_TIME` reads, ordered after prior code. +fn counter_ticks() -> u64 { + let ticks: u64; + // SAFETY: the ISB only orders the read; CNTPCT_EL0 is read-only. + unsafe { core::arch::asm!("isb", "mrs {x}, CNTPCT_EL0", x = out(reg) ticks) }; + ticks +} + +struct G18SubmissionFence; +#[vtable] +impl FenceOps for G18SubmissionFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c"aurora" } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c"g18-submit" } +} + +/// Keep the original sync vector: creating a second output list would add an +/// allocation failure after userspace has already supplied its wait targets. +struct OutputPublication { + fence: UserFence, + syncs: KVec, + input_count: usize, + result: Result, +} + +impl Drop for OutputPublication { + fn drop(&mut self) { + if let Err(error) = self.result { self.fence.set_error(error); } + let fence = Fence::from_fence(&self.fence); + for mut sync in self.syncs.drain(self.input_count..) { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj.add_point(chain, &fence, sync.timeline_value); + } else { sync.syncobj.replace_fence(Some(&fence)); } + } + self.fence.signal(); + } +} + +struct UserAddressSpace<'a> { vm: &'a mmu::Vm } +impl g17_uapi::GpuAddressSpace for UserAddressSpace<'_> { + fn covers(&self, address: u64, size: u64, access: g17_uapi::GpuAccess) -> bool { + let (read, write) = match access { + g17_uapi::GpuAccess::Read => (true, false), + g17_uapi::GpuAccess::Write => (false, true), + g17_uapi::GpuAccess::ReadWrite => (true, true), + }; + m5_vm::user_range_contains(&self.vm.kernel_range(), address, size, USER_END) + && self.vm.covers_range(address, size, read, write) + } +} + +/// A parsed timestamp pair in the UAPI's own type, for the shared resolver. +fn uapi_timestamps(timestamps: g17_uapi::UapiTimestamps) -> uapi::drm_asahi_timestamps { + uapi::drm_asahi_timestamps { + start: uapi::drm_asahi_timestamp { + handle: timestamps.start.handle, + offset: timestamps.start.offset, + }, + end: uapi::drm_asahi_timestamp { + handle: timestamps.end.handle, + offset: timestamps.end.offset, + }, + } +} + +struct G18Queue { + runtime: Arc, + submit_lock: Arc>, + config: G18DrmConfig, + vm: mmu::Vm, + window: g17_uapi::QueueUscWindow, + fences: FenceContexts, +} + +impl queue::Queue for G18Queue { + fn submit(&mut self, _reg_data: &AsahiRegData<'_>, _submission_id: u64, + syncs: KVec, + in_sync_count: usize, cmdbuf_raw: &[u8], + objects: Pin<&xarray::XArray>>) -> Result { + if in_sync_count > syncs.len() { return Err(EINVAL); } + let fence = self.fences.new_fence(0, G18SubmissionFence)?.into(); + let mut publication = OutputPublication { + fence, syncs, input_count: in_sync_count, result: Err(ECANCELED), + }; + let result = self.submit_one(&publication.syncs[..in_sync_count], cmdbuf_raw, objects); + // The runtime records and quarantines a failed published VM before + // this output fence is installed/signalled by the publication guard. + publication.result = result; + result + } +} + +impl G18Queue { + fn submit_one(&self, inputs: &[file::SyncItem], cmdbuf_raw: &[u8], + objects: Pin<&xarray::XArray>>) -> Result { + if self.vm.status().map_or(0, |s| s.get()) != 0 { return Err(EIO); } + if self.runtime.is_crashed() { return Err(ENODEV); } + // Wait outside the submission lock; another client may own the job + // whose fence satisfies this submission's dependency. + for sync in inputs { + if let Some(fence) = &sync.fence { + // SAFETY: both calls borrow the strongly held input fence. + let waited = unsafe { bindings::dma_fence_wait_timeout( + fence.raw(), true, kernel::task::MAX_SCHEDULE_TIMEOUT) }; + if waited < 0 { return Err(Error::from_errno(waited as i32)); } + if waited == 0 { return Err(ETIMEDOUT); } + let status = unsafe { bindings::dma_fence_get_status(fence.raw()) }; + if status < 0 { return Err(Error::from_errno(status)); } + } + } + // An empty command buffer is a synchronization-only submission. Input + // fences have completed above; the normal publication guard installs + // and signals its output fence, preserving timeline chaining. Never + // manufacture a hardware job or relax hardware retirement checks. + if cmdbuf_raw.is_empty() { + let _serialized = self.submit_lock.lock(); + if self.vm.status().map_or(0, |s| s.get()) != 0 { return Err(EIO); } + if self.runtime.is_crashed() { return Err(ENODEV); } + pr_info!("G18_SYNC_ONLY inputs={}\n", inputs.len()); + return Ok(()); + } + let _vm_job = self.vm.retain_t8140_job()?; + // Parsing and translation frames end here, before the job runs its + // own register builders on this stack. + let admitted = self.admit(cmdbuf_raw, objects)?; + let _serialized = self.submit_lock.lock(); + // Commands run in submission order and each retires before the next + // starts, so every barrier (an earlier command of this submission, or + // any earlier submission) has completed when its waiter begins. + for command in admitted { + if self.runtime.is_crashed() { return Err(ENODEV); } + // Without job stage ticks, the CPU bracket `[t0, t1]` around the + // synchronous job bounds every stage: starts get t0, ends get t1. + let t0 = counter_ticks(); + match command { + Admitted::Render(render, slots) => { + let retired = self.runtime.submit_render(&self.vm, &render)?; + let t1 = counter_ticks(); + slots.publish(retired.stage_ticks.unwrap_or([t0, t1, t0, t1]), + counter_frequency()); + } + Admitted::Compute(command, slots) => { + let retired = self.runtime.submit_compute(&self.vm, &command)?; + let t1 = counter_ticks(); + let [start, _, _, end] = retired.stage_ticks.unwrap_or([t0, t1, t0, t1]); + slots.publish([start, end, 0, 0], counter_frequency()); + } + } + } + Ok(()) + } + + /// Parse every hardware command of a submission, apply the G18 admission + /// rules and resolve their timestamp destinations, all before any runs: + /// a submission is admitted whole or not at all. Nothing here touches + /// the GPU. A compute prepass and the render that consumes it (transform + /// feedback, geometry and tessellation draws) arrive as one submission. + #[inline(never)] + fn admit(&self, cmdbuf_raw: &[u8], objects: Pin<&xarray::XArray>>) + -> Result> { + let malformed = |error: g17_uapi::UapiTranslateError| { + pr_err!("G18 submit admission rejected: {:?}\n", error); + EINVAL + }; + let mut parser = g17_uapi::UapiCommandParser::new(cmdbuf_raw); + let mut admitted = KVec::new(); + while let Some(command) = parser.next_hardware().map_err(malformed)? { + let command = match command { + g17_uapi::ParsedHardwareCommand::Render { + payload, vertex_attachments, fragment_attachments, .. + } => self.admit_render(&payload, vertex_attachments, fragment_attachments, objects)?, + g17_uapi::ParsedHardwareCommand::Compute { payload, attachments, .. } => + self.admit_compute(&payload, attachments, objects)?, + }; + admitted.push(command, GFP_KERNEL)?; + } + parser.finish().map_err(malformed)?; + Ok(admitted) + } + + #[inline(never)] + fn admit_render(&self, payload: &g17_uapi::UapiRenderCommand, + vertex_attachments: g17_uapi::UapiAttachmentList, + fragment_attachments: g17_uapi::UapiAttachmentList, + objects: Pin<&xarray::XArray>>) -> Result { + let address_space = UserAddressSpace { vm: &self.vm }; + let scratch = if payload.flags & g17_uapi::RENDER_G18_RESOURCES != 0 { + Some(self.admit_render_scratch(payload)?) + } else { + None + }; + // The shared translator refuses the G18 flag and never sees the + // suffix. Only this adapter consumes and strips a validated suffix. + let mut untagged = *payload; + untagged.flags &= !g17_uapi::RENDER_G18_RESOURCES; + untagged.g18_resources = [0; 20]; + let mut render = g18_application::translate_g18_render_admitting(&untagged, + vertex_attachments, fragment_attachments, self.window, &address_space, + self.config.admit_msaa4) + .map_err(|refusal| refuse_render(payload, refusal))?; + render.scratch = scratch; + let render = KBox::new(render, GFP_KERNEL)?; + if !g18_application::job_owned_fields_clear(&render.parameters, &render.controls) { + pr_err!("G18 render admission rejected: job-owned fields set\n"); + return Err(EINVAL); + } + let slots = timestamp::Destinations::resolve_unaligned(objects, &[ + uapi_timestamps(payload.vertex_timestamps), + uapi_timestamps(payload.fragment_timestamps), + ])?; + Ok(Admitted::Render(render, slots)) + } + + /// Derive the TA/3D private-memory selection for a flagged render. Only + /// single-sample, single-layer renders with one-page tile storage are in + /// the evidenced envelope (g18_graphics_scratch.rs). The selection never + /// resizes the owned pool: ideal when both kicks fit it, else the + /// native minimum result, else ENOSPC. Nothing is published on refusal. + #[inline(never)] + fn admit_render_scratch(&self, payload: &g17_uapi::UapiRenderCommand) + -> Result { + use crate::{g18::uapi::graphics_scratch::G18RenderMetadata, g18::uapi::render_policy as policy}; + if !self.config.admit_graphics_scratch { + pr_err!("G18 render admission rejected: graphics scratch is disabled (g18_admit_graphics_scratch=0)\n"); + return Err(ENOTSUPP); + } + if payload.samples != 1 || payload.layers != 1 + || (payload.utile_width, payload.utile_height) != (32, 32) { + pr_err!("G18 render admission rejected: graphics scratch outside single-sample 32x32 envelope samples={} layers={} utiles={}x{}\n", + payload.samples, payload.layers, payload.utile_width, payload.utile_height); + return Err(ENOTSUPP); + } + let tile_bytes = 32 * 32 * u32::from(payload.sample_size); + let metadata = G18RenderMetadata::decode(&payload.g18_resources, tile_bytes) + .ok_or_else(|| { + pr_err!("G18 render admission rejected: invalid graphics scratch metadata words={:x?} tile_bytes={}\n", + payload.g18_resources, tile_bytes); + EINVAL + })?; + let plan = metadata.plan(policy::RENDER_POOL_BLOCK_BYTES).ok_or_else(|| { + pr_err!("G18 render admission rejected: graphics scratch exceeds the native pool capacity\n"); + ENOSPC + })?; + let (owned_bytes, owned_chunks) = policy::owned_scratch_capacity(); + let selected = plan.select(owned_bytes, owned_chunks).ok_or_else(|| { + pr_err!("G18 render admission rejected: graphics scratch needs {:?} beyond owned pool {} bytes / {} chunks\n", + plan.minimum, owned_bytes, owned_chunks); + ENOSPC + })?; + pr_info!("G18_RENDER_SCRATCH vertex={:?} fragment={:?} tiles={:?} tile_bytes={} mode={:?} ta={:?} d3={:?}\n", + metadata.vertex, metadata.fragment, metadata.tiles, tile_bytes, selected.mode, + selected.stages[0], selected.stages[1]); + Ok(selected) + } + + #[inline(never)] + fn admit_compute(&self, payload: &g17_uapi::UapiComputeCommand, + attachments: g17_uapi::UapiAttachmentList, + objects: Pin<&xarray::XArray>>) -> Result { + if !self.config.admit_compute { + pr_err!("G18 compute admission rejected: compute is disabled (g18_admit_compute=0)\n"); + return Err(ENOTSUPP); + } + let stream_bytes = payload.control_stream_end.saturating_sub(payload.control_stream_base); + let g18_flags = G18_COMPUTE_TEXTURE_STATE | g17_uapi::COMPUTE_G18_RESOURCES; + // ADD3 and every flag outside the G18 contract remain refused. + if payload.flags & !g18_flags != 0 || payload.sampler_heap != 0 + || payload.sampler_count != 0 || stream_bytes > G18_COMPUTE_STREAM_BYTES { + pr_err!("G18 compute admission rejected flags={:#x} sampler_heap={:#x} samplers={} stream_bytes={} (admits at most {} bytes)\n", + payload.flags, payload.sampler_heap, payload.sampler_count, stream_bytes, + G18_COMPUTE_STREAM_BYTES); + return Err(ENOTSUPP); + } + let resource_descriptor = if payload.flags & g17_uapi::COMPUTE_G18_RESOURCES != 0 { + let metadata = crate::g18::uapi::compute_metadata::G18ComputeMetadata::decode(payload.g18_resources) + .ok_or_else(|| { + pr_err!("G18 compute admission rejected: invalid resource metadata\n"); + EINVAL + })?; + if crate::g18::uapi::compute_plan::G18ComputePlan::from_descriptor(&metadata.descriptor).is_none() { + pr_err!("G18 compute admission rejected: resource plan exceeds supported capacity\n"); + return Err(ENOSPC); + } + // Main static TPS uses the metadata-derived owned pool and SMA + // launch pair. Mesa's G18 compute encoder carries the same aligned + // frame in both SPECTPR scratch fields. Constant-program scratch + // and SCS still need their separate launch contract. + if metadata.demand.constant_scratch_bytes != 0 || payload.scs_layout != 0 + { + pr_err!("G18 compute admission rejected: constant scratch or SCS integration pending\n"); + return Err(ENOTSUPP); + } + crate::g18_kick_pr_info!("G18_COMPUTE_METADATA stream={:#x} registers={} scratch_bytes={} spectpr_bytes={} constant_scratch={} local={:?}\n", + payload.control_stream_base, payload.g18_resources[2], + metadata.demand.scratch_bytes, payload.g18_resources[4], + metadata.demand.constant_scratch_bytes, &payload.g18_resources[12..15]); + Some(metadata.descriptor) + } else { + None + }; + // The shared G17 translator refuses G18 flags and nonzero resource + // words. Only this adapter may consume and strip a validated suffix. + let mut untagged = *payload; + untagged.flags &= !g18_flags; + if resource_descriptor.is_some() { untagged.g18_resources = [0; 16]; } + let address_space = UserAddressSpace { vm: &self.vm }; + let mut translated = g17_uapi::translate_compute_command(untagged, attachments, + self.window, &address_space).map_err(|error| { + pr_err!("G18 compute admission rejected {:?}: helper={:#x}/{:#x}/{:#x} flags={:#x}\n", + error, payload.helper.binary, payload.helper.config, payload.helper.data, payload.flags); + EINVAL + })?; + translated.g18_texture_state = payload.flags & G18_COMPUTE_TEXTURE_STATE != 0; + translated.g18_resource_descriptor = resource_descriptor; + let slots = timestamp::Destinations::resolve_unaligned(objects, + &[uapi_timestamps(payload.timestamps)])?; + Ok(Admitted::Compute(KBox::new(translated, GFP_KERNEL)?, slots)) + } +} + +/// An admitted command, heap-held so the submit frame stays small. +enum Admitted { + Render(KBox, timestamp::Destinations), + Compute(KBox, timestamp::Destinations), +} + +/// Log a render refusal with the fields that decided it and map it to the +/// M5 errno convention: EINVAL for a malformed command, ENOTSUPP otherwise. +fn refuse_render(payload: &g17_uapi::UapiRenderCommand, refusal: G18ApplicationRefusal) -> Error { + pr_err!("G18 render admission rejected {:?}: size={}x{} layers={} utiles={}x{} samples={} msaa={:#x} flags={:#x} zls={:#x} depth={:#x}/{:#x} stencil={:#x} bgobjvals={:#x} samplers={}\n", + refusal, payload.width, payload.height, payload.layers, payload.utile_width, + payload.utile_height, payload.samples, payload.multisample_control, payload.flags, + payload.zls_control, payload.depth.base, payload.depth.compression_base, + payload.stencil.base, payload.stencil_clear, payload.sampler_count); + if refusal.is_invalid() { EINVAL } else { ENOTSUPP } +} diff --git a/drivers/gpu/drm/asahi/g18/fixture/compute_payload.rs b/drivers/gpu/drm/asahi/g18/fixture/compute_payload.rs new file mode 100644 index 0000000000000..0fa3860c7c871 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fixture/compute_payload.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Private 26A428 compute-add capture, with a poisoned Linux output buffer. +//! All records are user virtual pages, never captured physical/kernel mappings. +//! This fixed diagnostic format does not grant permission to submit a GPU job. + +#![allow(dead_code)] + +use crate::g18::fw::compute::{G18ComputeCommandInputs, G18ComputeRegisterInputs}; +use crate::g18::fw::compute_state::{G18ComputeState, G18_COMPUTE_STATE_BYTES}; + +pub(crate) const HEADER_BYTES: usize = 64; +pub(crate) const PAGE_BYTES: usize = 0x4000; +pub(crate) const RECORD_BYTES: usize = PAGE_BYTES + 16; +pub(crate) const PAGE_COUNT: usize = 46; +pub(crate) const INPUT_A: u64 = 0x100_0003_0000; +pub(crate) const INPUT_B: u64 = 0x100_0003_8000; +pub(crate) const OUTPUT: u64 = 0x100_0004_0000; +pub(crate) const POISON: u32 = 0xdead_beef; + +// Exact allocations in submission 1. Submission 2 is the post-compute blit. +const RANGES: [(u64, usize); 15] = [ + (0, 0x10000), + (0x100_0000_0000, 0x10000), (0x100_0001_8000, 0x10000), + (INPUT_A, PAGE_BYTES), (INPUT_B, PAGE_BYTES), (OUTPUT, PAGE_BYTES), + (0x100_0004_8000, 0x20000), (0x100_0007_0000, 0x20000), + (0x100_0009_8000, 0x8000), (0x100_000a_8000, 0x8000), + (0x100_000b_8000, 0x8000), (0x100_000c_8000, 0x8000), + (0x100_000d_8000, 0x8000), (0x100_000e_8000, 0x8000), + (0x100_000f_8000, 0xc000), +]; + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum PayloadError { Header, Length, Checksum, Page, Input, Output, State } + +pub(crate) struct ComputePayload<'a> { bytes: &'a [u8] } +pub(crate) struct PayloadPage<'a> { pub(crate) address: u64, pub(crate) bytes: &'a [u8] } + +fn u32_at(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} +fn u64_at(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) +} + +/// Transfer corruption check only. Capture provenance is in the SHA256 receipt. +pub(crate) fn fnv64(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |h, b| + (h ^ u64::from(*b)).wrapping_mul(0x100_0000_01b3)) +} + +/// Build the captured addition's state and default device control fields. +/// Original 26A428 setup 8bf1280/8bf1414 supplies the spill inputs to 8bf16cc: +/// d518/d520/d524 and d5a8/d5b0/d5d8 are all zero for submission 1. Native +/// configureDevice stores at 8bc73fc, 8c492a0, 8bc93bc and 8bc7774 establish +/// the register defaults below. Optional tuning and debug policies stay off. +/// Context, pool, scratch, scheduler, completion and addresses belong to the +/// retained owners; this function never publishes or submits a command. +pub(crate) fn addition_inputs(state: &[u8]) + -> Result<(G18ComputeCommandInputs, G18ComputeRegisterInputs), PayloadError> { + if state.len() != G18_COMPUTE_STATE_BYTES || fnv64(state) != 0x956a_f8e1_8848_5ce8 { + return Err(PayloadError::State); + } + let mut command = G18ComputeCommandInputs::default(); + let mut registers = G18ComputeRegisterInputs { + a10aac: 1, + a6d0: 1, + clusters: 2, + cluster_width: 6, + ..G18ComputeRegisterInputs::default() + }; + G18ComputeState::parse(state).ok_or(PayloadError::State)?.apply(&mut command, &mut registers); + Ok((command, registers)) +} + +/// Exact IEEE754 encoding of a small signed integer divided by four. Keeps +/// float operations out of kernel code; the qualified values fit in 13 bits. +fn quarter_bits(value: i32) -> u32 { + if value == 0 { return 0; } + let magnitude = value.unsigned_abs(); + let top = 31 - magnitude.leading_zeros(); + (u32::from(value < 0) << 31) | ((top + 125) << 23) + | ((magnitude << (23 - top)) & 0x7f_ffff) +} + +impl<'a> ComputePayload<'a> { + pub(crate) fn parse(bytes: &'a [u8]) -> Result { + let expected = HEADER_BYTES + G18_COMPUTE_STATE_BYTES + PAGE_COUNT * RECORD_BYTES; + if bytes.len() != expected { return Err(PayloadError::Length); } + if &bytes[..8] != b"G18CMP01" || u32_at(bytes, 8) != 1 + || u32_at(bytes, 12) != HEADER_BYTES as u32 + || u32_at(bytes, 16) != PAGE_BYTES as u32 + || u32_at(bytes, 20) != PAGE_COUNT as u32 + || u32_at(bytes, 24) != G18_COMPUTE_STATE_BYTES as u32 + || u32_at(bytes, 28) != 4096 + || u64_at(bytes, 32) != INPUT_A || u64_at(bytes, 40) != INPUT_B + || u64_at(bytes, 48) != OUTPUT + { return Err(PayloadError::Header); } + if fnv64(&bytes[HEADER_BYTES..]) != u64_at(bytes, 56) { + return Err(PayloadError::Checksum); + } + let payload = Self { bytes }; + let mut index = 0; + for (base, size) in RANGES { + for offset in (0..size).step_by(PAGE_BYTES) { + let record = HEADER_BYTES + G18_COMPUTE_STATE_BYTES + index * RECORD_BYTES; + let page = payload.page(index).ok_or(PayloadError::Page)?; + if page.address != base + offset as u64 || u32_at(bytes, record + 8) != 3 + || u32_at(bytes, record + 12) != PAGE_BYTES as u32 + { return Err(PayloadError::Page); } + if page.address == OUTPUT { + if page.bytes.chunks_exact(4).any(|v| u32_at(v, 0) != POISON) { + return Err(PayloadError::Output); + } + } else if page.address == INPUT_A || page.address == INPUT_B { + for (i, word) in page.bytes.chunks_exact(4).enumerate() { + let value = if page.address == INPUT_A { 2 * i as i32 } else { 4000 - i as i32 }; + if u32_at(word, 0) != quarter_bits(value) { return Err(PayloadError::Input); } + } + } + index += 1; + } + } + if index != PAGE_COUNT { return Err(PayloadError::Page); } + let mut c = G18ComputeCommandInputs::default(); + let mut r = G18ComputeRegisterInputs::default(); + payload.state().apply(&mut c, &mut r); + if r.program != [0x100_000f_8000, 0x100_000c_8000, 0x100_000f_9b80, + 0x100_000f_9b88, 0x100_000f_9b90, 0x100_000f_9b98] + || c.d4e0 != 0x100_000c_802c || r.aliases != [0; 6] || r.d548 != 0 + { return Err(PayloadError::State); } + // All six program pointers and the command-stream end must be backed. + for address in r.program.into_iter().chain([c.d4e0]) { + if !RANGES.iter().any(|(base, size)| *base <= address && address < *base + *size as u64) { + return Err(PayloadError::State); + } + } + Ok(payload) + } + + pub(crate) fn page_count(&self) -> usize { PAGE_COUNT } + + pub(crate) fn state(&self) -> G18ComputeState<'a> { + G18ComputeState::parse(&self.bytes[HEADER_BYTES..HEADER_BYTES + G18_COMPUTE_STATE_BYTES]).unwrap() + } + + pub(crate) fn state_bytes(&self) -> &[u8] { + &self.bytes[HEADER_BYTES..HEADER_BYTES + G18_COMPUTE_STATE_BYTES] + } + + pub(crate) fn page(&self, index: usize) -> Option> { + if index >= PAGE_COUNT { return None; } + let offset = HEADER_BYTES + G18_COMPUTE_STATE_BYTES + index * RECORD_BYTES; + Some(PayloadPage { address: u64_at(self.bytes, offset), + bytes: &self.bytes[offset + 16..offset + RECORD_BYTES] }) + } +} + +/// Byte evidence only: callers additionally require a completed, fault-free +/// live submission before interpreting a correct output as GPU execution. +#[derive(Debug, Default, Copy, Clone)] +pub(crate) struct ComputeResult { + pub(crate) checked: usize, + pub(crate) correct: usize, + pub(crate) untouched: usize, +} + +impl ComputeResult { + pub(crate) fn inspect(bytes: &[u8]) -> Option { + if bytes.len() != PAGE_BYTES { return None; } + let mut result = Self::default(); + for (i, word) in bytes.chunks_exact(4).enumerate() { + let value = u32_at(word, 0); + result.checked += 1; + result.correct += usize::from(value == quarter_bits(4000 + i as i32)); + result.untouched += usize::from(value == POISON); + } + Some(result) + } +} diff --git a/drivers/gpu/drm/asahi/g18/fixture/compute_triangle.rs b/drivers/gpu/drm/asahi/g18/fixture/compute_triangle.rs new file mode 100644 index 0000000000000..0a9457a8cbb9a --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fixture/compute_triangle.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Fixed, private compute-triangle patch and post-completion pixel checks. +//! Failure model: the triangle plan, section 2.7. This format only +//! patches four known sites in an unpublished, verified compute-add VM. + +use crate::g18::fixture::compute_payload::{fnv64, PAGE_BYTES, POISON}; + +pub(crate) const PATCH_COUNT: usize = 4; +pub(crate) const OUTPUT_FNV: u64 = 0x5a2c_fd9c_8627_b035; +const BASE_FNV: u64 = 0x4263_044c_89a6_b244; +const PATCH_FNV: u64 = 0xe40f_1f0c_5b96_cb69; +const HEADER_BYTES: usize = 64; +const PATCH_BYTES: usize = 16574; +// Address, length, original bytes' FNV. Patch provenance is SHA256-pinned in +// the host build receipt; these checks reject damaged or different payloads. +const SITES: [(u64, usize, u64); PATCH_COUNT] = [ + (0x100_0000_8000, PAGE_BYTES, 0x9c1b_da7f_8c87_2325), + (0x100_000a_8038, 10, 0x3def_155f_b602_ffc3), + (0x100_000a_8042, 12, 0xa522_f63b_a76a_995c), + (0x100_0001_8020, 8, 0xa8c7_f832_281a_39c5), +]; + +fn u32_at(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} +fn u64_at(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) +} + +pub(crate) struct TrianglePatch<'a> { bytes: &'a [u8] } +pub(crate) struct PatchSite<'a> { + pub(crate) address: u64, + pub(crate) before_fnv: u64, + pub(crate) bytes: &'a [u8], +} + +impl<'a> TrianglePatch<'a> { + pub(crate) fn parse(bytes: &'a [u8], base_fnv: u64) -> Option { + if bytes.len() != PATCH_BYTES || &bytes[..8] != b"G18TRI01" + || u32_at(bytes, 8) != 1 || u32_at(bytes, 12) != HEADER_BYTES as u32 + || u32_at(bytes, 16) != PATCH_COUNT as u32 + || u32_at(bytes, 20) != 64 || u32_at(bytes, 24) != 64 + || u32_at(bytes, 28) != 1568 || u64_at(bytes, 32) != OUTPUT_FNV + || base_fnv != BASE_FNV || u64_at(bytes, 40) != BASE_FNV + || u64_at(bytes, 48) != fnv64(&bytes[HEADER_BYTES..]) + || u64_at(bytes, 56) != 0 || fnv64(bytes) != PATCH_FNV { + return None; + } + let mut cursor = HEADER_BYTES; + for (address, length, before_fnv) in SITES { + let end = cursor.checked_add(24 + length)?; + if end > bytes.len() || u64_at(bytes, cursor) != address + || u32_at(bytes, cursor + 8) as usize != length + || u32_at(bytes, cursor + 12) != 0 + || u64_at(bytes, cursor + 16) != before_fnv { + return None; + } + cursor = end; + } + if cursor != bytes.len() { return None; } + Some(Self { bytes }) + } + + pub(crate) fn site(&self, index: usize) -> Option> { + if index >= PATCH_COUNT { return None; } + let cursor = HEADER_BYTES + SITES[..index].iter().map(|s| 24 + s.1).sum::(); + let (address, length, before_fnv) = SITES[index]; + Some(PatchSite { address, before_fnv, bytes: &self.bytes[cursor + 24..cursor + 24 + length] }) + } +} + +#[derive(Debug)] +pub(crate) struct TriangleResult { + pub(crate) digest: u64, + pub(crate) foreground: usize, + pub(crate) background: usize, + pub(crate) untouched: usize, + pub(crate) zero: usize, +} + +impl TriangleResult { + /// Only interpret after independent completion; this function neither + /// generates pixels nor establishes GPU execution by itself. + pub(crate) fn inspect(bytes: &[u8]) -> Option { + if bytes.len() != PAGE_BYTES { return None; } + let mut result = Self { digest: fnv64(bytes), foreground: 0, background: 0, + untouched: 0, zero: 0 }; + for word in bytes.chunks_exact(4) { + match u32_at(word, 0) { + POISON => result.untouched += 1, + 0 => result.zero += 1, + 0xff20_2020 => result.background += 1, + _ => result.foreground += 1, + } + } + Some(result) + } + + pub(crate) fn passed(&self) -> bool { + self.digest == OUTPUT_FNV && self.foreground == 1568 && self.background == 2528 + && self.untouched == 0 && self.zero == 0 + } +} diff --git a/drivers/gpu/drm/asahi/g18/fixture/mod.rs b/drivers/gpu/drm/asahi/g18/fixture/mod.rs new file mode 100644 index 0000000000000..ce562d91f0887 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fixture/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fixed G18 diagnostic captures (bring-up stages 8 and 9). + +#[allow(dead_code)] +pub(crate) mod compute_payload; +#[allow(dead_code)] +pub(crate) mod compute_triangle; +#[allow(dead_code)] +pub(crate) mod render_pixels; diff --git a/drivers/gpu/drm/asahi/g18/fixture/render_pixels.rs b/drivers/gpu/drm/asahi/g18/fixture/render_pixels.rs new file mode 100644 index 0000000000000..018ca8f0477a1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fixture/render_pixels.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Streaming SHA-256 of the fixed H1 output. This is not a completion signal. +pub(crate) struct Pixels { h: [u32;8], count: u64, pub poison: u32, pub background: u32 } +impl Pixels { + pub(crate) fn new() -> Self { Self { h: [0x6a09e667,0xbb67ae85,0x3c6ef372,0xa54ff53a, + 0x510e527f,0x9b05688c,0x1f83d9ab,0x5be0cd19], count: 0, poison: 0, background: 0 } } + fn block(&mut self, b: &[u8]) { + const K:[u32;64]=[0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2]; + let mut w=[0u32;64]; + for (i,v) in w[..16].iter_mut().enumerate() { *v=u32::from_be_bytes(b[i*4..i*4+4].try_into().unwrap()); } + for i in 16..64 { let x=w[i-15]; let y=w[i-2]; + w[i]=w[i-16].wrapping_add(x.rotate_right(7)^x.rotate_right(18)^(x>>3)) + .wrapping_add(w[i-7]).wrapping_add(y.rotate_right(17)^y.rotate_right(19)^(y>>10)); } + let [mut a,mut b,mut c,mut d,mut e,mut f,mut g,mut h]=self.h; + for i in 0..64 { + let t=h.wrapping_add(e.rotate_right(6)^e.rotate_right(11)^e.rotate_right(25)) + .wrapping_add((e&f)^(!e&g)).wrapping_add(K[i]).wrapping_add(w[i]); + let u=(a.rotate_right(2)^a.rotate_right(13)^a.rotate_right(22)).wrapping_add((a&b)^(a&c)^(b&c)); + h=g;g=f;f=e;e=d.wrapping_add(t);d=c;c=b;b=a;a=t.wrapping_add(u); + } + for (v,x) in self.h.iter_mut().zip([a,b,c,d,e,f,g,h]) { *v=v.wrapping_add(x); } + } + pub(crate) fn page(&mut self, data: &[u8]) -> bool { + if data.len()!=0x4000 || self.count>=0x40000 { return false; } + for w in data.chunks_exact(4) { self.poison+=u32::from(w==[0xa5;4]); self.background+=u32::from(w==[16,16,16,255]); } + for b in data.chunks_exact(64) { self.block(b); } + self.count+=data.len() as u64; true + } + pub(crate) fn finish(mut self) -> Option<[u32;8]> { + if self.count!=0x40000 { return None; } + let mut padding=[0u8;64];padding[0]=0x80;padding[56..].copy_from_slice(&(self.count*8).to_be_bytes()); + self.block(&padding);Some(self.h) + } +} +pub(crate) const COLOR: [u32;8] = [0xeb547f16,0x6fc6e2b6,0x9d35aca7,0xaa219eec,0xa3405d99,0x47c52e92,0xc18a8642,0xe0298e06]; diff --git a/drivers/gpu/drm/asahi/g18/fw/compute.rs b/drivers/gpu/drm/asahi/g18/fw/compute.rs new file mode 100644 index 0000000000000..2df516579d569 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/compute.rs @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 compute command serialization, derived from the original 26A428 HAL400. +//! +//! This encodes the submission fields and registers of the 0x1020-byte command. +//! It does not create resource owners, publish a queue entry or execute a kick. +//! The caller must own and validate all GPU addresses before publication. Native +//! offsets retained in field names are deliberately not assigned guessed meanings. +//! Register overrides, optional SMAPP preemption objects and random/alternating +//! tuning policies are outside this initial contract. + +#![allow(dead_code)] + +pub(crate) const G18_COMPUTE_COMMAND_BYTES: usize = 0x1020; + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ComputeRegisterInputs { + pub command_gpu: u64, + /// Native descriptor +4d8, +480, +4e8, +4f0, +4f8, +500, in emission order. + pub program: [u64; 6], + /// Native descriptor +578 through +5a0, in emission order. + pub aliases: [u64; 6], + pub d550: u64, + pub d558: u64, + pub d548: u64, + pub d5a8: u64, + pub d5b0: u64, + pub d540: u8, + pub d5d8: u8, + pub d5d9: u8, + pub d62a: u8, + pub d62b: u8, + pub d420: u8, + pub d418: u32, + pub d774: u32, + pub d5c0: u32, + pub d5c4: u32, + pub d5c8: u32, + pub d50c: u32, + pub d510: u32, + pub d514: u32, + pub d630: u32, + pub context_id: u32, + pub pool_slot: u32, + /// 0x10241[63:16]: AGXUMADataRec+0x100, the per-dispatch SMA result + /// (kext 8c4bdb8 kind 2). Not the pool binding's FList generation. + pub sma_units: u64, + pub operands: [u64; 2], + pub support_control: u64, + pub clusters: u32, + pub cluster_width: u32, + pub a6b7: u8, + pub a6b9: u8, + pub a6d0: u8, + pub a758: u32, + pub a10aac: u64, + pub a10ab4: u64, + pub a10aec: u32, + pub a10af0: u8, + pub a10af4: u32, + pub a10b04: u8, + pub a10b08: u32, + pub a10b18: u32, + pub policy54: u8, + pub policy2c: u8, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum G18ComputeRegisterError { + InvalidTopology, + AddressOverflow, + InvalidCompletionState, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18ComputeRegisterSummary { + pub main_gpu: u64, + pub main_count: u16, + pub total_main_count: u16, + pub secondary_gpu: u64, + pub secondary_count: u16, +} + +/// Inputs to the native per-queue completion calculation. `generation` is the +/// selected stamp half's u64 generation; selection happens in the owner layer. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ComputeCompletionState { + pub stamp: u32, + pub index: u32, + pub entries: u32, + pub generation: u64, + pub base: u64, +} + +/// The remaining native submission inputs. Offset names refer to the native +/// host descriptor, not firmware command offsets. Ownership is a caller duty. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ComputeCommandInputs { + pub sequence: u64, + pub completion: G18ComputeCompletionState, + pub stamp_addresses: [u64; 2], + pub channel_ordinal: u32, + pub has_wait: bool, + pub d100: u64, + pub d150: u32, + pub d158: u64, + pub d198: u64, + pub d1b0: u64, + pub d1b8: u64, + pub d488: u64, + pub d490: u64, + pub d498: u64, + pub d4a0: u64, + pub d4a8: u64, + pub d4b0: u32, + pub d4b8: u64, + pub d4c0: u32, + pub d4d0: u32, + pub d4e0: u64, + pub d508: u8, + pub d518: u64, + pub d520: u32, + pub d524: u32, + pub d628: u8, + pub d629: u8, + pub d770: u8, + pub d778: u64, + pub d780: u64, + pub d788: u64, + pub d790: u64, + pub d7b8: u64, + pub d7d4: u8, + pub d8b0: u8, + pub extra_addresses: [u64; 4], + pub extra_policy: u8, +} + +/// Original HAL400 completion expression. Preserve the native u64 division, +/// u32 arithmetic and final 8-bit index independently; a wide modulo gives +/// different bytes around counter wrap. G18's native scratch payload is zero. +pub(crate) fn g18_compute_completion( + p: &G18ComputeCompletionState, +) -> Result { + if p.entries == 0 || p.index >= 128 { + return Err(G18ComputeRegisterError::InvalidCompletionState); + } + let value = (u64::from(p.stamp >> 8) | p.generation << 24).wrapping_sub(1); + let quotient = (value / u64::from(p.entries)) as u32; + let remainder = (value as u32).wrapping_sub(quotient.wrapping_mul(p.entries)); + let slot = (p.base as u32 & 0xff) + (remainder & 0xff); + let carry = slot / p.entries; + let index = (slot - carry * p.entries) & 0xff; + let phase = ((p.base >> 8) as u32) + .wrapping_add(quotient) + .wrapping_add(carry); + Ok(u64::from(index) | u64::from(phase) << 8) +} + +/// Encode the native submission field writer and register program. This does +/// not publish a ring entry or imply that any pointer is backed. The caller +/// initializes the command allocation, rejects errors and retains all owners. +/// Aperture entries and optional SMAPP preemption resources are not supported. +pub(crate) fn encode_g18_compute_command( + out: &mut [u8; G18_COMPUTE_COMMAND_BYTES], + p: &G18ComputeCommandInputs, + registers: &G18ComputeRegisterInputs, +) -> Result { + let completion = g18_compute_completion(&p.completion)?; + put32(out, 0, 3); + put64(out, 4, p.sequence); + put32(out, 0xc, registers.context_id); + for (offset, value) in [ + (0x10, p.d198), + (0xfb5, p.d158), + (0xe80, p.d518), + (0xeb8, p.d488), + (0xec0, p.d490), + (0xec8, p.d498), + (0xed0, p.d4a0), + (0xed8, registers.program[0]), + (0xee0, p.d4e0), + (0xee8, p.d4a8), + (0xef8, p.d4b8), + (0xf2c, 0), + (0xf34, 0), + (0xf40, p.stamp_addresses[0]), + (0xf48, p.stamp_addresses[1]), + (0xf60, completion), + (0xf68, p.d100), + (0xf74, 0), + (0xf7c, p.d1b0), + (0xf84, p.d1b8), + (0xf8c, p.d778), + (0xf94, p.d780), + (0xf9c, p.d788), + (0xfa4, p.d790), + (0xfac, p.d7b8), + ] { + put64(out, offset, value); + } + for (offset, value) in [ + (0xf70, p.d150), + (0xe88, p.d520), + (0xe90, p.d524), + (0xeac, 0), + (0xef0, p.d4b0), + (0xf00, p.d4c0), + (0xf10, p.d4d0), + (0xf1c, u32::from(p.d508 & 1)), + (0xf20, registers.d50c), + (0xf24, registers.d510), + (0xf28, registers.d514), + (0xf50, p.completion.stamp), + (0xf54, p.completion.index), + (0xf58, p.channel_ordinal), + (0xf5c, u32::from(p.has_wait)), + (0xfdd, 0), + (0xfe1, 0), + ] { + put32(out, offset, value); + } + out[0xfb4] = p.d7d4 & 1; + out[0xf3c] = p.d628; + out[0xf3d] = p.d629 & 1; + out[0xf3e] = p.d770 & 1; + out[0xf3f] = p.d8b0 & 1; + let summary = encode_g18_compute_registers(out, registers)?; + out[0x1005] &= 0xf2; + if p.extra_addresses.iter().any(|&value| value != 0) { + out[0x1005] |= 1 | ((p.extra_policy & 3) << 2); + for (index, value) in p.extra_addresses.iter().enumerate() { + put64(out, 0xfe5 + index * 8, *value); + } + } + Ok(summary) +} + +fn put16(out: &mut [u8], offset: usize, value: u16) { + out[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn read32(out: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(out[offset..offset + 4].try_into().unwrap()) +} + +fn append(out: &mut [u8], base: usize, count: &mut u16, number: u32, value: u64) { + let offset = base + usize::from(*count) * 12; + let header = (read32(out, offset) & 0xfffc0006) | (number & 0x3fff9); + put32(out, offset, header); + put64(out, offset + 4, value); + *count += 1; +} + +/// Fill only native generateRegisterList outputs; leave all other command bytes +/// untouched. The full-command builder sets command+0xf68 before this call. +pub(crate) fn encode_g18_compute_registers( + out: &mut [u8; G18_COMPUTE_COMMAND_BYTES], + p: &G18ComputeRegisterInputs, +) -> Result { + if p.clusters == 0 || p.clusters > 16 || p.cluster_width == 0 || p.cluster_width > 16 { + return Err(G18ComputeRegisterError::InvalidTopology); + } + let main_gpu = p + .command_gpu + .checked_add(0x40) + .ok_or(G18ComputeRegisterError::AddressOverflow)?; + let secondary_gpu = p + .command_gpu + .checked_add(0x760) + .ok_or(G18ComputeRegisterError::AddressOverflow)?; + let mode_table = [ + 0x154006201u64, + 0x154004201, + 0x154002201, + 0x154002201, + 0x154002201, + 0x154002201, + 0x154000201, + ]; + let mut control = 0x154024200u64; + if p.a10b04 & 1 != 0 { + control = mode_table + .get(p.a10b08.wrapping_sub(1) as usize) + .copied() + .unwrap_or(0x154002201); + control = (control | (u64::from(p.a10b18) << 17)) & 0x1ffffffee6200; + } + let cs_mode = if p.a10af0 & 1 == 0 { + p.a10aec + } else { + u32::from(p.a10af4 == 0) + }; + control = ((p.a10aac >> 28 | p.a10aac << 9) & 0x1e1fffffe30) + | ((control | u64::from(cs_mode) << 20) & 0xfffe1fffe6001) + | u64::from((!p.d540) & 1) + | ((p.a10ab4 & 0x87ffffff) << 6) + | ((p.a10ab4 >> 32) << 33); + if p.a6d0 == 1 { + control |= 0x30; + } + let mut count = 0u16; + macro_rules! emit { + ($number:expr, $value:expr) => { + append(out, 0x40, &mut count, $number, $value) + }; + } + if p.d5d8 & 1 != 0 { + emit!(0x81e9, 1); + } + if p.d5d9 & 1 != 0 { + emit!(0x81f9, 1); + } + for (number, value) in [0x1a510, 0x1a420, 0x1a4d0, 0x1a4d8, 0x1a4e0, 0x1a4e8] + .into_iter() + .zip(p.program) + { + emit!(number, value); + } + emit!(0x1a440, control); + put64(out, 0xf08, control); + emit!(0xa5e1, p.d550 | p.d558 << 16); + if p.d548 != 0 { + emit!(0xa0f1, p.d548 & !0xfff); + } + let mut mode = if p.d62a & 1 == 0 { + 0x10c08860 + } else { + 0x14c08a60 + }; + if p.d62b & 1 != 0 { + mode |= 0x80; + } + if p.clusters > 2 || p.a6b9 & 1 != 0 || p.d548 != 0 { + mode |= 0x1c; + } + emit!(0x1a458, mode); + emit!(0x101d9, u64::from(p.d774 & 0x7f)); + emit!(0xa111, 0xffff00000000); + emit!(0xa131, 0x7000000000); + emit!(0xa151, 0x7100000000); + for (number, value) in [0x1a079, 0x1a081, 0x1a049, 0x1a051, 0x1a0a9, 0x1a0b1] + .into_iter() + .zip(p.aliases) + { + emit!(number, value); + } + emit!(0x101d1, (p.d5a8 & 0xffffffffc00fffff) | p.d5b0 << 20); + emit!(0x1a0d9, u64::from(p.d5c4 & 0xfffffff8)); + let next = p.d5c8.wrapping_add(1); + emit!(0x1a109, u64::from(if next < 32 { next } else { 0 })); + let mut dispatch = if p.d5d8 & 1 == 0 { + 0xff0004 + } else { + (u64::from(p.d5c0) << 16) | 0x1000004 + }; + if p.a6b7 & 1 == 0 { + dispatch = (dispatch & 0xfffffffb) | (u64::from(p.a758) << 32); + } else { + dispatch &= 0xffffffff; + } + dispatch |= u64::from(p.d5d9 & 1) << 25; + emit!(0x107a1, dispatch); + put64(out, 0x1006, dispatch); + put32(out, 0x100e, 0); + let identity = u64::from(read32(out, 0xf68)) | u64::from(p.d50c) << 32; + emit!(0x1a540, identity); + emit!(0x11209, identity); + emit!(0xa351, identity); + if p.policy54 & 1 != 0 { + if p.policy2c & 8 != 0 { + emit!(0x1a430, 0x3c0f); + } + let dimensions = p.d510.min(0x3fff) | p.d514.min(0x3fff) << 16; + for number in [0x1a4f0, 0x1a4f8, 0x1a500, 0x1a508] { + emit!(number, u64::from(dimensions)); + } + } + put32(out, 0x20, 0); + let mode = if p.d420 & 1 == 0 { + 0 + } else if p.d418 == 1 { + 3 + } else { + 1 + }; + emit!(0xa731, mode); + let context = u64::from((p.d630 & 0x3f) | p.context_id.wrapping_shl(8)); + emit!(0x10201, context); + emit!(0x10428, context); + emit!( + 0x10241, + (p.pool_slot as i32 as i64 as u64) | p.sma_units << 16 + ); + emit!(0xa611, 0); + emit!( + 0x10229, + (p.support_control & 0x7ffffffffc0) | 0x1000000000000000 + ); + if p.d5d8 & 1 != 0 { + emit!(0x81e1, 1); + } + if p.d5d9 & 1 != 0 { + emit!(0x81f1, 1); + } + let extent = u64::from(p.cluster_width + 10 * p.clusters - 10) * 0x1800; + let a = p.operands[0] & 0x7fffffffc00; + let b = p.operands[1] & 0x7fffffffc00; + emit!(0x10099, a | 5); + emit!(0x10091, a + extent); + emit!(0xa5c1, b | 5); + emit!(0xa5c9, b + extent); + let main_count = count; + put16(out, 0x18, main_count); + emit!(0xa731, 0); + emit!(0x1a440, control | 8); + put16(out, 0x1a, count); + put16(out, 0x1c, count); + put64(out, 0x740, main_gpu); + put16(out, 0x748, count); + let mut secondary_count = 0; + for (number, value) in [ + (0x10099, b | 5), + (0x10091, b + extent), + (0xa5c1, a | 5), + (0xa5c9, a + extent), + ] { + append(out, 0x760, &mut secondary_count, number, value); + } + put64(out, 0xe60, secondary_gpu); + put16(out, 0xe68, secondary_count); + Ok(G18ComputeRegisterSummary { + main_gpu, + main_count, + total_main_count: count, + secondary_gpu, + secondary_count, + }) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/compute_memory.rs b/drivers/gpu/drm/asahi/g18/fw/compute_memory.rs new file mode 100644 index 0000000000000..07fbf7999686e --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/compute_memory.rs @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Native G18 / 26A428 compute channel allocation geometry. +//! +//! AGXCLChannel::init 0x8be3934..0x8be3970 computes the extents; +//! 0x8be399c / 0x8be39ac form the allocation size, and +//! 0x8be3ad0..0x8be3ae8 bind its four pointers. These working buffers +//! are distinct from process UAT page tables and scheduler records. + +#![allow(dead_code)] + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18ComputeChannelLayout { + pub(crate) clusters: u32, + pub(crate) width: u32, + pub(crate) operand_bytes: u64, + pub(crate) control_bytes: u64, + pub(crate) tail_bytes: u64, + pub(crate) allocation_bytes: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18ComputeChannelPointers { + /// Native channel +0x178, +0x180, in that order. + pub(crate) operands: [u64; 2], + /// Native channel +0x188 and +0x190. + pub(crate) control: u64, + pub(crate) tail: u64, +} + +impl G18ComputeChannelLayout { + pub(crate) fn new(clusters: u32, width: u32) -> Option { + // The register encoder admits the same bounded topology. Arithmetic + // cannot wrap within these bounds; native's u32 operations agree. + if clusters == 0 || clusters > 16 || width == 0 || width > 16 { + return None; + } + let operand_bytes = + (u64::from(width + 10 * clusters - 10) * 0x1800 + u64::from(clusters) * 64 + 0x3ff) + & !0x3ff; + let control_bytes = u64::from(clusters) * 0x800; + let tail_bytes = 0x400; + let allocation_bytes = (2 * operand_bytes + control_bytes) | tail_bytes; + Some(Self { + clusters, + width, + operand_bytes, + control_bytes, + tail_bytes, + allocation_bytes, + }) + } + + /// The native register fields consume lower GPU VAs. A firmware-high + /// address must not be silently truncated into a different lower address. + pub(crate) fn bind(&self, base: u64) -> Option { + let end = base.checked_add(self.allocation_bytes)?; + if base == 0 || base & 0x3ff != 0 || end > 1u64 << 42 { + return None; + } + Some(G18ComputeChannelPointers { + operands: [base + self.operand_bytes, base], + control: base + 2 * self.operand_bytes, + tail: base + 2 * self.operand_bytes + self.control_bytes, + }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/compute_params.rs b/drivers/gpu/drm/asahi/g18/fw/compute_params.rs new file mode 100644 index 0000000000000..647c496321dbf --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/compute_params.rs @@ -0,0 +1,1619 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8140/G17P compute descriptor construction. +//! +//! The descriptor is the one-page selector-3 object executed by the CL path. +//! The layout combines the m1n1 builder with inspected native producers. The +//! caller supplies queue-owned object addresses and command-owned UAPI values; +//! this module only validates and encodes them. + +pub(crate) const COMPUTE_DESCRIPTOR_SIZE: usize = 0x4000; +/// Registers a kick with no SCS scratch request emits. +pub(crate) const COMPUTE_REGISTER_COUNT: usize = 40; +/// One more slot for the SCS enable register `generateCLRegisterListG16Common` +/// PREPENDS when the descriptor carries a non-zero scratch request +/// (mac17p `0xfffffe0008a5fa90`..`0xfffffe0008a5fac8`). +pub(crate) const COMPUTE_REGISTER_MAX: usize = COMPUTE_REGISTER_COUNT + 1; +pub(crate) const COMPUTE_REGISTER_CAPACITY: usize = 128; +pub(crate) const COMPUTE_REGISTER_SIZE: usize = 0x0c; +pub(crate) const COMPUTE_QUEUE_GRAPH_SIZE: usize = 0x8000; +pub(crate) const COMPUTE_QUEUE_CONTEXT_SIZE: usize = 0x4000; +pub(crate) const COMPUTE_QUEUE_CONTEXT_EXTENT: usize = 8 * COMPUTE_QUEUE_CONTEXT_SIZE; +pub(crate) const COMPUTE_OPTIONAL_SIZE: usize = 0xc0; +pub(crate) const COMPUTE_EVENT_SIZE: usize = 0x400; +pub(crate) const COMPUTE_CHANNEL_WRITE_INDEX: u32 = 3; +pub(crate) const COMPUTE_QUEUE_POINTERS: usize = 0x100; +const COMPUTE_QUEUE_CONTEXT_ITEM: usize = 0x200; +/// Same offset, exported so a repeat can refresh the item the first bind wrote. +pub(crate) const COMPUTE_QUEUE_CONTEXT_ITEM_OFFSET: usize = COMPUTE_QUEUE_CONTEXT_ITEM; +pub(crate) const COMPUTE_ITEM_RING: usize = 0x4000; +pub(crate) const COMPUTE_ITEM_RING_SIZE: usize = 0x500 * 8; +pub(crate) const COMPUTE_OPTIONAL: usize = 0x6800; +pub(crate) const COMPUTE_EVENT: usize = 0x6c00; +/// Storage for the tag-14 `KsmKickQueueAddKicks` channel command. +/// +/// Native allocates this record from a firmware-owned pool +/// (`AGXFirmware::requestChannelCommandKSMKickQueueAddKicks`, +/// mac17p `0xfffffe0008a2d664`). Linux keeps one retained slot per queue graph, +/// placed after the tag-16 window exactly as tags 15 and 16 are placed after the +/// command-pointer ring. A000 `0xfffffc0000011710` cache-invalidates `0x18` +/// bytes of the record before reading it, so the reserved window is zeroed as a +/// whole even though the host writes only `0x15` bytes. +/// Records one steady-state submission publishes into the item ring: +/// the tag-3 command pointer, tag 15, tag 14, tag 16. +pub(crate) const COMPUTE_REPEAT_RING_RECORD_COUNT: usize = 4; +pub(crate) const COMPUTE_ADD_KICKS: usize = 0x7000; +pub(crate) const COMPUTE_ADD_KICKS_SIZE: usize = 0x40; +const _: () = assert!(COMPUTE_ITEM_RING + COMPUTE_ITEM_RING_SIZE <= COMPUTE_OPTIONAL); +const _: () = assert!(COMPUTE_OPTIONAL + COMPUTE_OPTIONAL_SIZE <= COMPUTE_EVENT); +const _: () = assert!(COMPUTE_EVENT + COMPUTE_EVENT_SIZE <= COMPUTE_ADD_KICKS); +const _: () = assert!(COMPUTE_ADD_KICKS + COMPUTE_ADD_KICKS_SIZE <= COMPUTE_QUEUE_GRAPH_SIZE); +const _: () = assert!(COMPUTE_REGISTER_MAX <= COMPUTE_REGISTER_CAPACITY); + +const COMPUTE_SELECTOR: u32 = 3; +const COMPUTE_REGISTER_START: usize = 0x40; +const COMPUTE_SECONDARY_REGISTER_START: usize = 0x760; +const COMPUTE_PRIMARY_COUNT: usize = 0x748; +const COMPUTE_ENCODER_PARAMS: usize = 0xf14; +const COMPUTE_SAMPLER_ARRAY: usize = COMPUTE_ENCODER_PARAMS + 0x18; +const COMPUTE_SAMPLER_COUNT: usize = COMPUTE_ENCODER_PARAMS + 0x20; +const COMPUTE_SAMPLER_MAX: usize = COMPUTE_ENCODER_PARAMS + 0x24; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ComputeBuildError { + BufferTooSmall, + AddressOverflow, + CdmTerminatorBeforeStream, + SubmissionIndexZero, + QueueSubmissionZero, + InvalidSksmEntryAliases, + SamplerCountOutOfRange, + SamplerPair, + SamplerAlignment, + ScsRequestOutOfRange, + RegisterProgramShape, + HardwareTimestampOutOfRange, +} + +/// Low and high GPU aliases of one retained SKSM entry backing. +/// +/// Native tag-15 ConfigUpdate reads these from the same KSM queue object at +/// `+0x80/+0x88`. Keeping them typed prevents the classic CL2 queue-context +/// allocation from being substituted for either address. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeSksmEntryAliases { + low: u64, + high: u64, + extent: usize, +} + +impl ComputeSksmEntryAliases { + pub(crate) const fn new( + low: u64, + high: u64, + extent: usize, + alignment: u64, + ) -> Result { + if low == 0 + || high == 0 + || low == high + || extent == 0 + || alignment == 0 + || alignment & (alignment - 1) != 0 + || low & (alignment - 1) != 0 + || high & (alignment - 1) != 0 + || low.checked_add(extent as u64).is_none() + || high.checked_add(extent as u64).is_none() + { + return Err(ComputeBuildError::InvalidSksmEntryAliases); + } + Ok(Self { low, high, extent }) + } + + pub(crate) const fn low(self) -> u64 { + self.low + } + + pub(crate) const fn high(self) -> u64 { + self.high + } + + pub(crate) const fn extent(self) -> usize { + self.extent + } + + /// Return corresponding addresses within both aliases of the one backing. + pub(crate) const fn range_at( + self, + offset: usize, + size: usize, + ) -> Result<[u64; 2], ComputeBuildError> { + let end = match offset.checked_add(size) { + Some(end) => end, + None => return Err(ComputeBuildError::InvalidSksmEntryAliases), + }; + if end > self.extent { + return Err(ComputeBuildError::InvalidSksmEntryAliases); + } + Ok([self.low + offset as u64, self.high + offset as u64]) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeRegister { + pub(crate) number: u32, + pub(crate) value: u64, +} + +impl ComputeRegister { + const fn new(number: u32, value: u64) -> Self { + Self { number, value } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeRegisterParameters { + pub(crate) preempt_base: u64, + pub(crate) cdm_base: u64, + pub(crate) usc_exec_base: u64, + pub(crate) helper_binary: u64, + pub(crate) helper_data: u64, + pub(crate) helper_config: u64, + pub(crate) dispatch_identity: u64, + pub(crate) context_id: u32, + pub(crate) work_ordinal: u32, + pub(crate) robustness: u64, + pub(crate) operand_state_base: u64, + /// Native CL prepare stores the allocated FList ID at descriptor+0x798; + /// generateRegisterList emits it at 0x14028. This is not an enable gate. + pub(crate) usc_flist_id: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeDescriptorObjects { + pub(crate) scheduler_record: u64, + pub(crate) descriptor_low_alias: u64, + pub(crate) dispatch_a: u64, + pub(crate) dispatch_b: u64, + pub(crate) status_a: u64, + pub(crate) status_b: u64, + pub(crate) shared_control: u64, + pub(crate) usage_record: u64, +} + +/// Descriptor offset of the completion-stamp value firmware copies to the +/// support-page stamp word (`*(descriptor + 0x0F40)`) after the kick's +/// completion cache maintenance (b1 `AGFAcceleratorProcessTimeStampQueue`, +/// see `G17P_COMPUTE_STAMP_SUPPORT_OFFSET`). +pub(crate) const COMPUTE_COMPLETION_STAMP_VALUE: usize = 0x0f50; +/// Initial stamp for bootstrap records. User publication overwrites this with +/// the physical queue's monotonic kick value, independently of TSQ slot index. +pub(crate) const COMPUTE_COMPLETION_STAMP_CONSTANT: u32 = 0x100; + +/// Native EventMachine increments a stable event index's value by 0x100. +/// B1 coalesces timestamp records at that index before cache maintenance: +/// the newest store must cover every earlier completed kick in that prefix. +pub(crate) const fn completion_stamp_value(kick_timestamp: u64) -> u32 { + (kick_timestamp as u32).wrapping_shl(8) +} + +/// Modular prefix comparison, valid while outstanding kicks span less than +/// half the 24-bit stamp sequence (the queue's admission bound is much lower). +/// Zero is a valid wrapped store. Counter/ring identity must also pass. +pub(crate) const fn completion_stamp_covers(stamp_word: u32, kick_timestamp: u64) -> bool { + let delta = stamp_word.wrapping_sub(completion_stamp_value(kick_timestamp)); + delta & 0xff == 0 && delta < 0x8000_0000 +} + +/// Bind the native allocation epoch after building any user compute record. +/// Zero is a valid wrapped epoch; never replace it with a default value. +pub(crate) fn set_context_generation(raw: &mut [u8], generation: u8) -> Result<(), ComputeBuildError> { + if raw.len() <= 0x0fd3 { + return Err(ComputeBuildError::BufferTooSmall); + } + put_u8(raw, 0x0fd3, generation); + Ok(()) +} + +/// Bind the physical queue's exact publication prefix before exposing a record. +pub(crate) fn set_completion_stamp_value(raw: &mut [u8], kick_timestamp: u64) -> Result<(), ComputeBuildError> { + if raw.len() < COMPUTE_COMPLETION_STAMP_VALUE + 4 { + return Err(ComputeBuildError::BufferTooSmall); + } + put_u32(raw, COMPUTE_COMPLETION_STAMP_VALUE, completion_stamp_value(kick_timestamp)); + Ok(()) +} + +/// CL's native completion scratch payload is zero; its low forty bits are +/// the same converted hardware timestamp used by the KSM entry. This field +/// is a u64 at +0xf60, not a u32 ordinal or a QID-packed dependency token. +pub(crate) fn set_hardware_timestamp(raw: &mut [u8], timestamp: u64) -> Result<(), ComputeBuildError> { + if raw.len() < 0x0f68 { return Err(ComputeBuildError::BufferTooSmall); } + if timestamp >> 40 != 0 { return Err(ComputeBuildError::HardwareTimestampOutOfRange); } + put_u64(raw, 0x0f60, timestamp); + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeDescriptorMetadata { + pub(crate) submit_sequence: u64, + pub(crate) context_id: u32, + /// Event stamp index (+0xf54), distinct from the TSQ pointer slot. + pub(crate) grid_index: u32, + pub(crate) work_ordinal: u32, + pub(crate) queue_submission: u32, + /// Historical name: +0xf58 selects a pointer slot within the TSQ node. + /// It is NOT an unbounded submission ordinal. User queues use slot zero. + pub(crate) queue_ordinal: u32, + /// Bootstrap seed for the low word at +0xf60; user publication binds the + /// complete forty-bit hardware timestamp with set_hardware_timestamp. + pub(crate) submission_index: u32, + pub(crate) support_control: u32, + pub(crate) support_flags: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeDescriptorCommand { + pub(crate) cdm_terminator: u64, + pub(crate) sampler_array: u64, + pub(crate) sampler_count: u32, + pub(crate) user_timestamp_start: u64, + pub(crate) user_timestamp_end: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeQueueGraphAddresses { + pub(crate) queue_record: u64, + pub(crate) pointers: u64, + pub(crate) job_list: u64, + pub(crate) item_ring: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeChannelGroupAddresses { + pub(crate) queue: u64, + pub(crate) optional: u64, + pub(crate) event: u64, + pub(crate) write_index: u32, +} + +fn add(address: u64, offset: u64) -> Result { + address + .checked_add(offset) + .ok_or(ComputeBuildError::AddressOverflow) +} + +/// The three register values that carry a compute kick's ray-traversal (SCS) +/// scratch request. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ScsRequestRegisters { + /// 0x101d1: pages in the low half, MiB in the high half. + pub(crate) size_word: u64, + /// 0xd479: the request's "extras" count. + pub(crate) extras: u64, + /// 0x107a1: window and enable, or the disabled constant. + pub(crate) window_word: u64, + /// `AGXCLCommandDescriptor+0x5c8`, which `processComputeSetup` sets to + /// `pages != 0` (mac17p `0xfffffe0008a07ab8`..`0xfffffe0008a07ac0`). + /// `generateCLRegisterListG16Common` branches on it twice: once to prepend + /// register `0x17d9 = 1` (`0xfffffe0008a5fa90`) and once to choose the + /// 0x107a0 value (`0xfffffe0008a60280`). The three request registers alone + /// do NOT tell the two arms apart, so carry the flag rather than inferring + /// it from a value. + pub(crate) enabled: bool, +} + +/// The MiB half of the SCS request, derived from the page count exactly as +/// AGXMetalG17P derives it in `ComputeContext::endComputePass` +/// (0x216ac1f5c..0x216ac1f6c: `pages << 2`, `+ 0x3ff`, `>> 10`). +fn scs_request_mib(pages: u32) -> u32 { + ((pages as u64 + 0xff) >> 8) as u32 +} + +/// Derive the per-core SCS request from Apple's packed ScsLayoutGen1 word. +/// +/// AGXMetalG17P's `getSectionEnd(ScsSection::End)` has four fixed sections +/// totaling 0x58000 bytes, followed by per-VRID ray-stack, driver-payload and +/// user-payload storage. The virtual-ray count is the packed clique byte times +/// 32. `endComputePass` rounds that total to 4 KiB and adds one guard page. +/// +/// A layout with neither user nor driver payload is the native "no traversal +/// scratch" gate even if other fields are populated. Arithmetic is checked +/// here rather than reproducing Apple's 32-bit wrapping: a wrapped request +/// would under-allocate kernel-owned backing. +pub(crate) fn scs_pages_for_layout(layout: u64) -> Result { + let depth = layout & 0xffff; + let cliques = (layout >> 24) & 0xff; + let user = (layout >> 32) & 0xffff; + let driver = (layout >> 48) & 0xffff; + + if user == 0 && driver == 0 { + return Ok(0); + } + + let vrids = cliques + .checked_mul(32) + .ok_or(ComputeBuildError::ScsRequestOutOfRange)?; + let per_vrid = depth + .checked_mul(8) + .and_then(|value| value.checked_add(driver)) + .and_then(|value| value.checked_add(user)) + .ok_or(ComputeBuildError::ScsRequestOutOfRange)?; + let bytes = vrids + .checked_mul(per_vrid) + .and_then(|value| value.checked_add(0x58000)) + .and_then(|value| value.checked_add(0x1fff)) + .ok_or(ComputeBuildError::ScsRequestOutOfRange)? + & !0xfff; + let pages = bytes >> 12; + + if pages > 0xffff { + return Err(ComputeBuildError::ScsRequestOutOfRange); + } + + Ok(pages as u32) +} + +/// Encode an SCS scratch request the way the kext's register builder does +/// (generateCLRegisterListG16Common): the page and MiB counts share 0x101d1, +/// the extras count is its own register, and 0x107a1 either carries the window +/// with the enable bit or reads back the disabled constant 0x00ff_0000. +/// +/// `mib` of zero means "derive it the way Metal does", which is what a faithful +/// request wants; pass a non-zero value only to sweep the two halves of 0x101d1 +/// independently. A request of zero pages reproduces the register program this +/// driver has always emitted, so the default path is unchanged. +pub(crate) fn scs_request_registers( + pages: u32, + mib: u32, + extras: u32, + window: u32, +) -> Result { + if pages == 0 { + return Ok(ScsRequestRegisters { + size_word: 0, + extras: 0, + window_word: 0xff_0000, + enabled: false, + }); + } + // Keep the page and MiB halves distinct, and the window below bit 24's + // enable flag. Reject unrepresentable diagnostic inputs before publishing + // any register program instead of overlapping fields or truncating them. + if pages > 0xffff || mib > 0xffff || window > 0xff { + return Err(ComputeBuildError::ScsRequestOutOfRange); + } + let mib = if mib == 0 { scs_request_mib(pages) } else { mib }; + Ok(ScsRequestRegisters { + size_word: (pages as u64) | ((mib as u64) << 16), + extras: extras as u64, + window_word: ((window as u64) << 16) | 0x0100_0000, + enabled: true, + }) +} + +/// One compute kick's primary register program. +/// +/// The length is not fixed: `generateCLRegisterListG16Common` emits one extra +/// leading register when the descriptor carries an SCS scratch request, so the +/// descriptor's own count word at `COMPUTE_PRIMARY_COUNT` has to follow. +#[derive(Debug, Copy, Clone)] +pub(crate) struct ComputeRegisterProgram { + registers: [ComputeRegister; COMPUTE_REGISTER_MAX], + len: usize, +} + +impl ComputeRegisterProgram { + pub(crate) fn as_slice(&self) -> &[ComputeRegister] { + &self.registers[..self.len] + } + + pub(crate) fn len(&self) -> usize { + self.len + } +} + +/// Build the compute register program, taking the SCS request from the module +/// parameters. +pub(crate) fn build_compute_registers( + p: ComputeRegisterParameters, +) -> Result { + let scs = scs_request_registers( + crate::g18::fw::config::COMPUTE_SCS_PAGES, + crate::g18::fw::config::COMPUTE_SCS_MIB, + crate::g18::fw::config::COMPUTE_SCS_EXTRAS, + crate::g18::fw::config::COMPUTE_SCS_WINDOW, + )?; + build_compute_registers_with_scs(p, scs) +} + +/// Build a production compute register program from the per-command SCS ABI. +/// A zero/non-RT layout is byte-identical to the historical register program. +pub(crate) fn build_compute_registers_for_layout( + p: ComputeRegisterParameters, + layout: u64, +) -> Result { + let pages = scs_pages_for_layout(layout)?; + let scs = scs_request_registers(pages, 0, 0, 0)?; + build_compute_registers_with_scs(p, scs) +} + +/// The register program for an explicit SCS request. +/// +/// Split out of `build_compute_registers` so the enabled arm is reachable from +/// a host test without a module parameter. +pub(crate) fn build_compute_registers_with_scs( + p: ComputeRegisterParameters, + scs: ScsRequestRegisters, +) -> Result { + // Native reads CommandBufferState+8 and masks its low six bits here. + // The logical compute ordinal at descriptor+0xf70 is a separate identity. + let context_word = ((p.context_id as u64) << 8) | u64::from(p.work_ordinal & 63); + + let body = [ + ComputeRegister::new(0x1a510, p.preempt_base), + ComputeRegister::new(0x1a420, p.cdm_base), + ComputeRegister::new(0x1a4d0, add(p.preempt_base, 0x1480)?), + ComputeRegister::new(0x1a4d8, add(p.preempt_base, 0x1488)?), + ComputeRegister::new(0x1a4e0, add(p.preempt_base, 0x1490)?), + ComputeRegister::new(0x1a4e8, add(p.preempt_base, 0x1498)?), + ComputeRegister::new(0x10071, p.usc_exec_base), + ComputeRegister::new(0x11841, p.helper_binary), + ComputeRegister::new(0x11849, p.helper_data), + ComputeRegister::new(0x11f81, p.helper_config), + ComputeRegister::new(0x1a440, 0x0000_0001_5402_4201), + ComputeRegister::new(0x1a458, 0x0000_0000_10c0_8860), + ComputeRegister::new(0x101d9, 0x1c), + ComputeRegister::new(0x1a089, 0), + ComputeRegister::new(0x1a091, 0), + ComputeRegister::new(0x1a059, 0), + ComputeRegister::new(0x1a061, 0), + ComputeRegister::new(0x1a0b9, 0), + ComputeRegister::new(0x1a0c1, 0), + ComputeRegister::new(0x101d1, scs.size_word), + ComputeRegister::new(0x0d479, scs.extras), + ComputeRegister::new(0x1a0e9, 8), + ComputeRegister::new(0x107a1, scs.window_word), + // `A[47:32]` is the per-thread private-context ("PTC"/DUPM) maximum. + // `generateCLRegisterListG16Common` builds this value twice and both + // times from the same accelerator constant, so the two emissions are + // always equal; this driver's 0x132 was not: + // + // first emission `0xfffffe0008a6030c`: reads + // `AGXCLCommandDescriptor+0x548` and emits + // `CL[0x548] ? 0x0040_0020 | CL[0x548] << 32 + // : 0x0000_0060_0040_0020` (`0xfffffe0008a6032c`); + // second emission `0xfffffe0008a60b4c`: reads the accelerator fields + // directly -- `accel[0x10790] ? (u32)accel[0x10790] + // : accel[0x10778]` -- and applies the + // identical expression (`0xfffffe0008a60b7c`). + // + // `CL+0x548` is seeded from `AGXUMADataRec+0x88` + // (`AGXCLCommandDescriptor::prepare`, `0xfffffe00089fb85c`), which + // `AGXUSCPrivMemPoolUMA::allocatePagesEnd` fills with that same + // `accel[0x10790] ?: accel[0x10778]` (`0xfffffe0008aa7688`.. + // `0xfffffe0008aa76a0`). + // + // For G17P that constant is 0x60: `AGXAcceleratorG16::configureDevice` + // stores `accel[0x10778] = 0x60` unconditionally + // (`0xfffffe0008a55688`: `mov w8, #0x60`; `0xfffffe0008a5568c`: + // `str w8, [x21, #0x1b8]`, x21 = accel+0x105c0), and + // `AGXAcceleratorG17P::configureDevice` calls it as its first + // instruction (`0xfffffe0008a64df8`). Nothing in com.apple.AGXG17P + // writes `accel[0x10790]`, so the `?:` selects `accel[0x10778]`, and + // Apple's own zero-source fallback constant in the same expression is + // literally `0x60` -- the two agree. The render lists already emit + // `0x0000_0060_0040_0020` for their analogues 0x0a5a1 and 0x0a5a9. + ComputeRegister::new(0x0a599, 0x0000_0060_0040_0020), + ComputeRegister::new(0x0d411, 0x0000_0002_0000_0001), + ComputeRegister::new(0x1a540, p.dispatch_identity), + ComputeRegister::new(0x014a9, p.dispatch_identity), + ComputeRegister::new(0x0a351, p.dispatch_identity), + ComputeRegister::new(0x10201, context_word), + ComputeRegister::new(0x10428, context_word), + ComputeRegister::new(0x14028, u64::from(p.usc_flist_id)), + ComputeRegister::new(0x14070, p.robustness | 1), + ComputeRegister::new(0x10229, add(p.operand_state_base, 0x12800)?), + ComputeRegister::new(0x140a8, add(p.operand_state_base, 0x13000)?), + ComputeRegister::new(0x10099, add(p.operand_state_base, 0x9405)?), + ComputeRegister::new(0x10091, add(p.operand_state_base, 0x12400)?), + ComputeRegister::new(0x0a5c1, add(p.operand_state_base, 0x0005)?), + ComputeRegister::new(0x0a5c9, add(p.operand_state_base, 0x9000)?), + ComputeRegister::new(0x1a440, 0x0000_0001_5402_4209), + ComputeRegister::new(0x0a599, 0x0000_0060_0040_0020), + ]; + + // `generateCLRegisterListG16Common` writes its first register list entry + // at buffer offset 0 only when `AGXCLCommandDescriptor+0x5c8` is clear. + // When it is set the first entry is `0x17d9 = 1` and everything else moves + // up by one 12-byte entry (mac17p `0xfffffe0008a5fa90` takes the branch, + // `0xfffffe0008a5faa0` ORs the selector `0x17d9` into the entry word, + // `0xfffffe0008a5fab0` stores the value 1, `0xfffffe0008a5fac4` sets the + // cursor to 12 so `0x1a510` lands at entry 1; the clear arm at + // `0xfffffe0008a5facc` sets the cursor to 0 instead). The TA and 3D list + // builders do the same thing with the same selector and value, each gated + // on its own descriptor flag (`0xfffffe0008a585b8` on TA `+0x598`, + // `0xfffffe0008a5a84c` on 3D `+0xb70`). + // + // This driver had never emitted `0x17d9`, which is the only difference + // left between its `g17p_compute_scs_pages=1` program and the kext's: that + // run already sent the byte-exact 0x101d1 / 0xd479 / 0x107a1 triple and its + // shader still never launched. That `0x17d9` is what the launch was + // waiting on is an INFERENCE, not a measured result. + // + // What `0x17d9` means to the hardware is not established either. None of + // the selectors this program emits appears as an immediate anywhere in the + // B100 firmware image (checked 0x17d8/0x17d9, 0x101d0, 0xd478, 0x107a0, + // 0xa598, 0x1a510, 0x14070 and 0x1748), so the firmware replays the list + // rather than interpreting it and the meaning is the GPU's, not the + // firmware's. + let mut registers = [ComputeRegister::new(0, 0); COMPUTE_REGISTER_MAX]; + let mut len = 0; + if scs.enabled { + registers[len] = ComputeRegister::new(0x17d9, 1); + len += 1; + } + for register in body { + registers[len] = register; + len += 1; + } + Ok(ComputeRegisterProgram { registers, len }) +} + +fn put_u8(raw: &mut [u8], offset: usize, value: u8) { + raw[offset] = value; +} + +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Build the retained queue record named by every compute CL entry. +pub(crate) fn build_compute_queue_graph( + base: u64, + channel_control: u64, + raw: &mut [u8], +) -> Result { + if raw.len() < COMPUTE_QUEUE_GRAPH_SIZE { + return Err(ComputeBuildError::BufferTooSmall); + } + let addresses = ComputeQueueGraphAddresses { + queue_record: base, + pointers: add(base, 0x100)?, + job_list: add(base, 0x200)?, + item_ring: add(base, 0x4000)?, + }; + let raw = &mut raw[..COMPUTE_QUEUE_GRAPH_SIZE]; + raw.fill(0); + put_u64(raw, 0x00, addresses.pointers); + put_u64(raw, 0x08, addresses.item_ring); + put_u64(raw, 0x10, addresses.job_list); + put_u32(raw, 0x24, u32::MAX); + put_u32(raw, 0x28, 2); + put_u32(raw, 0x2c, 2); + raw[0x36..0x38].fill(0xff); + put_u32(raw, 0x40, 2); + put_u32(raw, 0x44, u32::MAX); + // Bootstrap/unowned graph. A logical bind supplies the creator's TGID. + put_u32(raw, 0x48, 0); + put_u64(raw, 0x9c, channel_control); + put_u32(raw, 0x100 + 0x50, u32::MAX); + put_u32(raw, 0x100 + 0x60, compute_item_ring_entries()); + put_u64(raw, 0x200 + 0x08, addresses.job_list); + Ok(addresses) +} + +/// Usable entries in the queue's item ring, as programmed into the graph's +/// pointer block at `+0x160` and read back by both the host publisher and the +/// firmware. +/// +/// Normally the full `0x500`. `g17p_ring_limit` shrinks it so the ring WRAP can +/// be reached in a handful of submissions instead of 320: at four records per +/// submission, a 32-entry ring wraps on the eighth. Wrap behaviour is identical +/// at any size, so this turns a 400-iteration run -- which also overflows the +/// kernel log and destroys its own evidence -- into a 20-iteration one. +pub(crate) fn compute_item_ring_entries() -> u32 { + const FULL: u32 = 0x500; + let requested = crate::g18::fw::config::RING_LIMIT; + if requested == 0 { + return FULL; + } + // Must stay a multiple of the per-submission record count so a batch never + // straddles the wrap differently than it would at full size, and must leave + // room for at least two submissions plus the always-empty slot. + let clamped = requested.clamp( + 2 * COMPUTE_REPEAT_RING_RECORD_COUNT as u32 + 4, + FULL, + ); + clamped - (clamped % COMPUTE_REPEAT_RING_RECORD_COUNT as u32) +} + +/// Identity of the configuration last published on one physical CL QID. +/// +/// Logical scheduler pages are unique and retained until processor stop. +/// Their address therefore also distinguishes two root leases that reuse the +/// same numeric context ID, including queues sharing one VM. The root lease +/// is immutable while current; submit validates that before using this key. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeConfigIdentity { + pub(crate) channel_control: u64, + pub(crate) qos_id: u8, + pub(crate) context_id: Option, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeConfigUpdates { + pub(crate) qos: bool, + pub(crate) context: bool, +} + +impl ComputeConfigIdentity { + pub(crate) fn updates_from(self, published: Option) -> ComputeConfigUpdates { + match published { + None => ComputeConfigUpdates { qos: true, context: true }, + Some(previous) => ComputeConfigUpdates { + // QID membership is scoped to its logical scheduler page. + // Policy is currently immutable for this logical lease. + qos: self.channel_control != previous.channel_control + || self.qos_id != previous.qos_id, + context: self.channel_control != previous.channel_control + || self.context_id != previous.context_id, + }, + } + } +} + +/// Encode the tag-15 `KSMKickQueueConfigUpdate` record inside a retained +/// queue graph. +/// +/// A retained submission may carry this record without reinstalling the QID +/// or changing its QoS/context. Native computes +0x52 from QoS ID, QID +/// membership and policy changes, and +0x62 from the channel's cached GART +/// allocation generation. See audit_g17_native_scheduler_owner.py. +pub(crate) fn encode_compute_config_update( + graph: &mut [u8], + sksm_entries: ComputeSksmEntryAliases, + shared_control: u64, + channel_control: u64, + queue_id: u8, + hardware_buffer_id: u8, + install_queue: bool, + updates: ComputeConfigUpdates, + context_id: Option, +) -> Result<(), ComputeBuildError> { + // AGXChannel::getPid supplies a full u32 at tag15 +0x5a. The queue's + // logical-owner bind populates the same PID at record +0x48; reading that + // retained identity avoids capturing a workqueue PID during submission. + let owner_pid = u32::from_le_bytes( + graph.get(0x48..0x4c).ok_or(ComputeBuildError::BufferTooSmall)? + .try_into().map_err(|_| ComputeBuildError::BufferTooSmall)?, + ); + let optional_raw = graph + .get_mut(COMPUTE_OPTIONAL..COMPUTE_OPTIONAL + COMPUTE_OPTIONAL_SIZE) + .ok_or(ComputeBuildError::BufferTooSmall)?; + encode_compute_config_update_record( + optional_raw, + owner_pid, + sksm_entries, + shared_control, + channel_control, + queue_id, + hardware_buffer_id, + install_queue, + updates, + context_id, + ) +} + +/// Encode one tag-15 `KSMKickQueueConfigUpdate` record into `optional_raw` +/// (`COMPUTE_OPTIONAL_SIZE` bytes), wherever that record lives. +/// +/// A queue's FIRST bind encodes it in the queue graph's optional window; every +/// later publication encodes it in its own per-publication record slot. The +/// firmware reads a record through the item-ring pointer that names it, which +/// can be long after the host has published further kicks, so a record must +/// never be rewritten while a ring entry still names it. Measured on Neo +/// 2026-09-26 19:41: a repeat encoded into the graph window while the first +/// bind's ring entry still pointed there rewrote +0x1a (queue-install) to 0; +/// the firmware then consumed the first bind with the install branch skipped, +/// left the per-queue slot unpopulated, and its tag-16 handler faulted on the +/// NULL slot (pc=0x1190c far=0x140 esr=0x96000004), the same fault shape the +/// +0x1a note below records for sending 0 directly. +pub(crate) fn encode_compute_config_update_record( + optional_raw: &mut [u8], + owner_pid: u32, + sksm_entries: ComputeSksmEntryAliases, + shared_control: u64, + channel_control: u64, + queue_id: u8, + hardware_buffer_id: u8, + install_queue: bool, + updates: ComputeConfigUpdates, + context_id: Option, +) -> Result<(), ComputeBuildError> { + let optional_raw = optional_raw + .get_mut(..COMPUTE_OPTIONAL_SIZE) + .ok_or(ComputeBuildError::BufferTooSmall)?; + // Record +0x32 is the hardware context this queue's work is declared to run + // in. It has always been the literal 2, from when every compute client ran + // in the fixed pair `T8140_COMPUTE_CTXS = [2, 3]`. + // + // Those two contexts are a SINGLE GLOBAL alias pair: + // `install_t8140_compute_context_alias` publishes the current client's VM + // root into both and records one `t8140_alias_owner`. With one compute + // client that is correct. With a second, the alias is re-pointed and the + // loser's kicks -- still declaring 2 -- no longer name their own root, + // while their real root lives in the app context the execution context + // allocated (5..62). The kick already declares that real context at + // `sgx+0x10428`; only this record still says 2. + // + // Probe-gated and default-off: 0 sends the historical constant, byte + // identical to the previous driver; 1 sends the client's real context when + // one is known. See `g17p_compute_config_context`. + let declared_context = match context_id { + Some(context) if crate::g18::fw::config::COMPUTE_CONFIG_CONTEXT != 0 => { + context + } + _ => 2u16, + }; + let flushid_fix = crate::g18::fw::config::FLUSHID_FIX != 0; + let (flushid_object, flushid_slot, flushid_index) = if flushid_fix { + (0u64, 0xffffu16, 0xffffu16) + } else { + (shared_control, 0x3fu16, crate::g18::fw::config::compute_flist_id(queue_id)) + }; + optional_raw.fill(0); + put_u32(optional_raw, 0x00, 0x0f); + put_u64(optional_raw, 0x08, sksm_entries.low()); + put_u64(optional_raw, 0x10, sksm_entries.high()); + for (offset, value) in [ + (0x18, u16::from(queue_id)), + // +0x1a is the QUEUE-INSTALL flag, and it must be non-zero on a first + // bind. Firmware RE (2026-08-31): the only writer of the per-queue slot + // `0x127628 + qid*0x28` is at 0x119b8, and that store -- together with + // slot[+0x00]=1, the per-data-master active bitmap, the sgx+0x21008 + // queue-base write and the 0x40 KTrace receipt -- is reachable only when + // this field is non-zero. Sending 0 left slot[+0x08] NULL, and the tag-16 + // handler then faulted dereferencing it (firmware exception pc=0x1190c + // far=0x340 esr=0x96000004, a level-0 translation fault), which is why + // pause bit 7 taken at 0x118c0 was never cleared at 0x11948. + // + // The two captured macOS records show 0 here because both were taken on + // already-installed queues, not on a first bind. Absence of the 0x40 + // KTrace code is the receipt that the install branch was not taken. + // Queue-install flag: 1 only on a queue's FIRST bind. Native clears it + // afterwards -- leaving it set forces a disable/reprogram/enable cycle + // on the KSMFE queue for every submission, which is why a second submit + // published fine and then never completed. + (0x1a, if install_queue { 1 } else { 0 }), + (0x1e, 2), + (0x22, 2), + (0x32, declared_context), + (0x3e, flushid_slot), + (0x46, flushid_index), + (0x52, u16::from(updates.qos)), + // AGXCLChannelSKSM::submitBuffer copies AGXQOSQueue +0x10 here. + // Native retained CL queues receive distinct HardwareBufferIDs in + // allocation order (1, 2, 3 in the captured QID 4/5/7 sequence). + // This field is therefore queue identity, not the literal 1 from the + // first output-positive queue. + (0x56, u16::from(hardware_buffer_id)), + (0x5e, 2), + (0x62, u16::from(updates.context)), + (0x66, 1), + ] { + put_u16(optional_raw, offset, value); + } + put_u64(optional_raw, 0x36, flushid_object); + put_u64(optional_raw, 0x4a, channel_control); + put_u32(optional_raw, 0x5a, owner_pid); + optional_raw[0x76..0x86].fill(0xff); + Ok(()) +} + +/// Encode one queue-context item. +/// +/// The array is a sibling of the SKSM entry array, not a single record: +/// `COMPUTE_QUEUE_CONTEXT_EXTENT` is `8 * 0x4000 = 0x20000`, the item stride is +/// `0x200`, so it holds `0x100` items -- exactly the queue geometry's +/// `stamp_count`, and exactly the size and stride of the entry backing +/// (`0x100 * 0x200 = 0x20000`). The first bind writes item 1 and consumes stamp +/// index 1, which is consistent with the item being indexed by stamp. +/// +/// That reading is INFERRED from the geometry, not proven, which is why the +/// repeat path selects between "rewrite item 1" and "write item[stamp]" by +/// parameter instead of assuming. +pub(crate) fn encode_compute_queue_context_item( + queue_context: &mut [u8], + item_index: usize, + descriptor: u64, + queue_base: u64, + queue_id: u8, +) -> Result<(), ComputeBuildError> { + let offset = item_index + .checked_mul(COMPUTE_QUEUE_CONTEXT_ITEM) + .ok_or(ComputeBuildError::AddressOverflow)?; + let end = offset + .checked_add(COMPUTE_QUEUE_CONTEXT_ITEM) + .ok_or(ComputeBuildError::AddressOverflow)?; + if end > queue_context.len() { + return Err(ComputeBuildError::BufferTooSmall); + } + let context = &mut queue_context[offset..end]; + context.fill(0); + put_u64(context, 0x00, 0x1000_1000_0000_0004); + put_u64(context, 0x10, descriptor); + put_u64(context, 0x18, queue_base); + put_u64(context, 0x20, 0xffff_0801_0000_0001); + put_u64(context, 0x28, u64::from(queue_id) << 40); + put_u64(context, 0x130, 0); + put_u64(context, 0x138, 4); + put_u64(context, 0x150, 0x0001_1003_8001_a002); + put_u64(context, 0x158, 0x0000_2003_8001_a03b); + put_u64(context, 0x178, 0x003f_ffff_ffff_ffff); + Ok(()) +} + +/// Stage the first hardware-proven direct-compute group in a retained queue. +/// +/// This is the CL_2 channel graph used by the output-positive m1n1 add3 test: +/// selector-3 descriptor, selector-0x0f optional record, selector-0x0e event, +/// followed by queue write index 3. The caller publishes the outer channel +/// slot and its producer only after this function returns. +pub(crate) fn build_initial_compute_channel_group( + base: u64, + descriptor: u64, + sksm_entries: ComputeSksmEntryAliases, + shared_control: u64, + channel_control: u64, + queue_id: u8, + hardware_buffer_id: u8, + graph: &mut [u8], + queue_context: &mut [u8], + install_queue: bool, + context_id: Option, +) -> Result { + if graph.len() < COMPUTE_QUEUE_GRAPH_SIZE + || queue_context.len() < COMPUTE_QUEUE_CONTEXT_SIZE + { + return Err(ComputeBuildError::BufferTooSmall); + } + + let optional = add(base, COMPUTE_OPTIONAL as u64)?; + let event = add(base, COMPUTE_EVENT as u64)?; + + // Record +0x48 is the process PID, not a UUID. Preserve the logical + // owner's value installed when this physical graph was bound. + for (index, address) in [descriptor, optional, event].into_iter().enumerate() { + put_u64(graph, COMPUTE_ITEM_RING + index * 8, address); + } + put_u32( + graph, + COMPUTE_QUEUE_POINTERS + 0x40, + COMPUTE_CHANNEL_WRITE_INDEX, + ); + + // B1 registers an exact pointer/cookie at table[record+0x46]; later + // descriptor processing selects that same table with FList+0x08. Both + // IDs must identify our canonical compute pool, not render's ID zero. + // The cookie is stable for this retained session incarnation. All queues + // register the same pointer/cookie; their scheduler identity is separate. + encode_compute_config_update( + graph, + sksm_entries, + shared_control, + channel_control, + queue_id, + hardware_buffer_id, + install_queue, + ComputeConfigUpdates { qos: true, context: true }, + context_id, + )?; + + let event_raw = &mut graph[COMPUTE_EVENT..COMPUTE_EVENT + COMPUTE_EVENT_SIZE]; + event_raw.fill(0); + put_u32(event_raw, 0x00, 0x0e); + put_u32(event_raw, 0x04, (1u32 << 16) | u32::from(queue_id)); + put_u32(event_raw, 0x08, 0x0102); + put_u32(event_raw, 0x10, 0x0200); + + queue_context[..COMPUTE_QUEUE_CONTEXT_SIZE].fill(0); + // Item index 1, which is also the stamp index the first bind consumes + // (the geometry seeds `current_timestamp` at 1). See + // `encode_compute_queue_context_item` for why that coincidence matters. + encode_compute_queue_context_item(queue_context, 1, descriptor, base, queue_id)?; + + Ok(ComputeChannelGroupAddresses { + queue: base, + optional, + event, + write_index: COMPUTE_CHANNEL_WRITE_INDEX, + }) +} + +fn encode_registers(raw: &mut [u8], offset: usize, registers: &[ComputeRegister]) { + for (index, register) in registers.iter().enumerate() { + let entry = offset + index * COMPUTE_REGISTER_SIZE; + put_u32(raw, entry, register.number); + put_u64(raw, entry + 4, register.value); + } +} + +fn register_value(registers: &[ComputeRegister], number: u32, occurrence: usize) -> u64 { + registers + .iter() + .filter(|register| register.number == number) + .nth(occurrence) + .expect("fixed G17P register program lost a required register") + .value +} + +fn register_count_through( + registers: &[ComputeRegister], + number: u32, + occurrence: usize, +) -> Result { + let index = registers + .iter() + .enumerate() + .filter(|(_, register)| register.number == number) + .nth(occurrence) + .map(|(index, _)| index) + .ok_or(ComputeBuildError::RegisterProgramShape)?; + u16::try_from(index + 1).map_err(|_| ComputeBuildError::RegisterProgramShape) +} + +/// Resume checkpoints consumed by B1 after a nonterminal CL completion. +/// +/// The no-extra G16 common producer records the current register count after +/// the first 0x0a5c9, the final 0x1a440 (twice), and the final 0x0a599. B1 +/// reads descriptor +0x1a/+0x1c on status 2 and copies the selected low seven +/// bits into the selected SKSM kick-ring entry at +0x150. Derive the counts from the +/// actual program so an SCS 0x17d9 prefix, or a later list edit, cannot leave +/// stale resume indices in the descriptor. +pub(crate) fn compute_resume_checkpoints( + registers: &[ComputeRegister], +) -> Result<[u16; 4], ComputeBuildError> { + let first_tail = register_count_through(registers, 0x0a5c9, 0)?; + let final_control = register_count_through(registers, 0x1a440, 1)?; + let final_tail = register_count_through(registers, 0x0a599, 1)?; + Ok([first_tail, final_control, final_control, final_tail]) +} + +/// Encode a selector-3 descriptor without dropping sampler, timestamp, or CDM +/// command state. +pub(crate) fn build_compute_descriptor( + program: &ComputeRegisterProgram, + objects: ComputeDescriptorObjects, + metadata: ComputeDescriptorMetadata, + command: ComputeDescriptorCommand, + raw: &mut [u8], +) -> Result<[u16; 4], ComputeBuildError> { + let registers = program.as_slice(); + if raw.len() < COMPUTE_DESCRIPTOR_SIZE { + return Err(ComputeBuildError::BufferTooSmall); + } + if command.cdm_terminator < register_value(registers, 0x1a420, 0) { + return Err(ComputeBuildError::CdmTerminatorBeforeStream); + } + if metadata.submission_index == 0 { + return Err(ComputeBuildError::SubmissionIndexZero); + } + if metadata.queue_submission == 0 { + return Err(ComputeBuildError::QueueSubmissionZero); + } + if command.sampler_count == u32::MAX { + return Err(ComputeBuildError::SamplerCountOutOfRange); + } + if (command.sampler_array == 0) != (command.sampler_count == 0) { + return Err(ComputeBuildError::SamplerPair); + } + if command.sampler_array & 7 != 0 { + return Err(ComputeBuildError::SamplerAlignment); + } + + let raw = &mut raw[..COMPUTE_DESCRIPTOR_SIZE]; + raw.fill(0); + put_u32(raw, 0x00, COMPUTE_SELECTOR); + put_u64(raw, 0x04, metadata.submit_sequence); + put_u32(raw, 0x0c, metadata.context_id); + put_u64(raw, 0x10, objects.scheduler_record); + let resume_checkpoints = compute_resume_checkpoints(registers)?; + for (index, value) in resume_checkpoints.into_iter().enumerate() { + put_u16(raw, 0x18 + index * 2, value); + } + + encode_registers(raw, COMPUTE_REGISTER_START, registers); + put_u64( + raw, + 0x740, + add(objects.descriptor_low_alias, COMPUTE_REGISTER_START as u64)?, + ); + // The count word follows the program, which grows by one entry when the + // kick carries an SCS request -- exactly as the kext's own + // `[sp,#0x58]`/`[sp,#0x5a]` byte-size and entry-count pair does + // (`0xfffffe0008a5fab4` seeds them with 12 and 1 on the request arm). + put_u32( + raw, + COMPUTE_PRIMARY_COUNT, + ((program.len() * COMPUTE_REGISTER_SIZE) as u32) << 16 | program.len() as u32, + ); + + let secondary = [ + ComputeRegister::new(0x10099, register_value(registers, 0x0a5c1, 0)), + ComputeRegister::new(0x10091, register_value(registers, 0x0a5c9, 0)), + ComputeRegister::new(0x0a5c1, register_value(registers, 0x10099, 0)), + ComputeRegister::new(0x0a5c9, register_value(registers, 0x10091, 0)), + ]; + encode_registers(raw, COMPUTE_SECONDARY_REGISTER_START, &secondary); + put_u64( + raw, + 0x0e60, + add( + objects.descriptor_low_alias, + COMPUTE_SECONDARY_REGISTER_START as u64, + )?, + ); + put_u32(raw, 0x0e68, 0x0030_0004); + put_u32(raw, 0x0f28, u32::MAX); + + put_u64(raw, 0x0ed8, register_value(registers, 0x1a510, 0)); + put_u64(raw, 0x0ee0, command.cdm_terminator); + put_u64(raw, 0x0f08, register_value(registers, 0x1a440, 0)); + put_u32( + raw, + 0x0f20, + (register_value(registers, 0x1a540, 0) >> 32) as u32, + ); + put_u64(raw, COMPUTE_SAMPLER_ARRAY, command.sampler_array); + put_u32(raw, COMPUTE_SAMPLER_COUNT, command.sampler_count); + put_u32( + raw, + COMPUTE_SAMPLER_MAX, + if command.sampler_count == 0 { + 0 + } else { + command.sampler_count + 1 + }, + ); + put_u64(raw, 0x0f40, objects.dispatch_a); + put_u64(raw, 0x0f48, objects.dispatch_b); + // Bootstrap seed only. Every user publication route binds the actual kick + // timestamp separately; queue_ordinal is a TSQ slot, not a stamp sequence. + put_u32(raw, 0x0f50, COMPUTE_COMPLETION_STAMP_CONSTANT); + put_u32(raw, 0x0f54, metadata.grid_index); + put_u32(raw, 0x0f58, metadata.queue_ordinal); + put_u32(raw, 0x0f60, metadata.submission_index); + put_u64( + raw, + 0x0f68, + register_value(registers, 0x1a540, 0) & 0xffff_ffff, + ); + put_u32(raw, 0x0f70, metadata.work_ordinal); + put_u64(raw, 0x0f7c, objects.status_a); + put_u64(raw, 0x0f84, objects.status_b); + put_u64(raw, 0x0f8c, command.user_timestamp_start); + put_u64(raw, 0x0f94, command.user_timestamp_end); + + put_u16(raw, 0x0fb0, 0x001a); + put_u64(raw, 0x0fb2, objects.shared_control); + put_u32(raw, 0x0fba, metadata.support_control); + put_u32(raw, 0x0fbe, metadata.support_flags); + put_u8(raw, 0x0fc5, 0x9f); + put_u32(raw, 0x0fc8, (metadata.work_ordinal & 3) << 30); + put_u64(raw, 0x0fcb, objects.usage_record); + put_u8(raw, 0x0fd3, 1); + Ok(resume_checkpoints) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn register_work_state_does_not_overwrite_the_context_id() { + for word in [0, 62, 64, 0x1234_5678, u32::MAX - 1] { + let mut p = register_parameters(); + p.context_id = 9; + p.work_ordinal = word; + let program = build_compute_registers(p).unwrap(); + let expected = (9 << 8) | u64::from(word & 63); + assert_eq!(register_value(program.as_slice(), 0x10201, 0), expected); + assert_eq!(register_value(program.as_slice(), 0x10428, 0), expected); + } + } + + #[test] + fn hardware_timestamp_keeps_all_forty_bits_and_zero_payload() { + for timestamp in [0, 1, 0xffff_ffff, 0x1_0000_0000, 0xff_ffff_ffff] { + let mut raw = [0xa5u8; 0x1000]; + set_hardware_timestamp(&mut raw, timestamp).unwrap(); + assert_eq!(u64::from_le_bytes(raw[0xf60..0xf68].try_into().unwrap()), timestamp); + assert!(raw[..0xf60].iter().all(|&v| v == 0xa5)); + assert!(raw[0xf68..].iter().all(|&v| v == 0xa5)); + } + let mut raw = [0xa5u8; 0x1000]; + assert_eq!(set_hardware_timestamp(&mut raw, 1 << 40), Err(ComputeBuildError::HardwareTimestampOutOfRange)); + assert!(raw.iter().all(|&v| v == 0xa5)); + assert_eq!(set_hardware_timestamp(&mut [0; 4], 1), Err(ComputeBuildError::BufferTooSmall)); + } + + #[test] + fn allocation_epoch_changes_only_its_byte_and_accepts_wrap() { + for epoch in [0, 1, 2, 3, 127, 255] { + let mut raw = [0xa5u8; 0x1000]; + set_context_generation(&mut raw, epoch).unwrap(); + assert_eq!(raw[0xfd3], epoch); + assert!(raw[..0xfd3].iter().all(|&b| b == 0xa5)); + assert!(raw[0xfd4..].iter().all(|&b| b == 0xa5)); + } + assert_eq!(set_context_generation(&mut [0; 0xfd3], 1), Err(ComputeBuildError::BufferTooSmall)); + } + + #[test] + fn completion_stamp_covers_only_the_flushed_prefix_including_wrap() { + for kick in [1u64, 0x100, 0x3628, 0x3629, 0xff_ffff, 0x100_0000, 0x123_4567_89ab] { + let stamp = completion_stamp_value(kick); + assert!(completion_stamp_covers(stamp, kick)); + assert!(!completion_stamp_covers(stamp, kick + 1)); + assert!(completion_stamp_covers(completion_stamp_value(kick + 16), kick)); + assert!(!completion_stamp_covers(stamp | 1, kick)); + } + assert!(!completion_stamp_covers(0, 1)); + assert!(completion_stamp_covers(0, 0x100_0000)); + assert!(completion_stamp_covers(0, 0xff_ffff)); + assert!(!completion_stamp_covers(0xffff_ff00, 0x100_0000)); + assert!(!completion_stamp_covers(0x8000_0100, 1)); + let mut raw = [0u8; COMPUTE_COMPLETION_STAMP_VALUE + 4]; + set_completion_stamp_value(&mut raw, 0x3629).unwrap(); + assert_eq!( + u32::from_le_bytes(raw[0x0f50..0x0f54].try_into().unwrap()), + 0x362900, + ); + let mut raw2 = [0u8; COMPUTE_COMPLETION_STAMP_VALUE + 4]; + set_completion_stamp_value(&mut raw2, 1).unwrap(); + assert_ne!(raw2, raw); + assert_eq!(set_completion_stamp_value(&mut [0; 4], 1), Err(ComputeBuildError::BufferTooSmall)); + } + + fn fnv1a64(raw: &[u8]) -> u64 { + let mut value = 0xcbf2_9ce4_8422_2325u64; + for byte in raw { + value ^= *byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + value + } + + fn register_parameters() -> ComputeRegisterParameters { + ComputeRegisterParameters { + preempt_base: 0x100_0404_0000, + cdm_base: 0x100_0410_0000, + usc_exec_base: 0x1_0000_0000, + helper_binary: 0, + helper_data: 0, + helper_config: 0, + dispatch_identity: 0x1234_5678_0000_0001, + context_id: 3, + work_ordinal: 7, + robustness: 0x100_0430_0000, + operand_state_base: 0x70_0022_8000, + usc_flist_id: 2, + } + } + + #[test] + fn register_program_matches_the_hardware_tested_m1n1_builder() { + let program = build_compute_registers(register_parameters()).unwrap(); + let registers = program.as_slice(); + assert_eq!(program.len(), COMPUTE_REGISTER_COUNT); + let mut raw = [0u8; COMPUTE_REGISTER_COUNT * COMPUTE_REGISTER_SIZE]; + encode_registers(&mut raw, 0, registers); + // Re-pinned when 0x0a599's first emission was corrected from 0x132 + // to the 0x60 both of the kext's emissions carry. + assert_eq!(fnv1a64(&raw), 0x2d33_2bc4_ecd2_8bdd); + assert_eq!(registers[0], ComputeRegister::new(0x1a510, 0x100_0404_0000)); + assert_eq!(registers[39], ComputeRegister::new(0x0a599, 0x6000_4000_20)); + } + + #[test] + fn both_0xa599_emissions_carry_the_same_accelerator_constant() { + // `generateCLRegisterListG16Common` reads one accelerator quantity for + // both 0xa598 emissions (`0xfffffe0008a6030c` through + // `AGXCLCommandDescriptor+0x548`, and `0xfffffe0008a60b4c` straight + // from `accel[0x10790] ?: accel[0x10778]`), so they cannot disagree. + // This driver's two writes did; the first carried 0x132. + let program = build_compute_registers(register_parameters()).unwrap(); + let first = register_value(program.as_slice(), 0x0a599, 0); + let second = register_value(program.as_slice(), 0x0a599, 1); + assert_eq!(first, second); + // 0x60 is `AGXAcceleratorG16::configureDevice`'s + // `accel[0x10778] = 0x60` (`0xfffffe0008a5568c`), and also the kext's + // own zero-source fallback constant in the same expression. + assert_eq!(first, 0x0000_0060_0040_0020); + } + + #[test] + fn an_scs_request_prepends_the_0x17d9_enable_register() { + let scs = scs_request_registers(89, 1, 0, 0).unwrap(); + assert!(scs.enabled); + let program = + build_compute_registers_with_scs(register_parameters(), scs).unwrap(); + assert_eq!(program.len(), COMPUTE_REGISTER_MAX); + // Entry 0 is the enable register, and the historical program follows + // it unchanged -- the kext moves its cursor to 12 before writing + // 0x1a510 (`0xfffffe0008a5fac4`). + assert_eq!(program.as_slice()[0], ComputeRegister::new(0x17d9, 1)); + assert_eq!( + program.as_slice()[1], + ComputeRegister::new(0x1a510, 0x100_0404_0000) + ); + // Nothing else moves: the same registers in the same order, and only + // the three request registers change value. + let disabled = build_compute_registers(register_parameters()).unwrap(); + assert_eq!(program.as_slice().len(), disabled.as_slice().len() + 1); + for (enabled, disabled) in program.as_slice()[1..].iter().zip(disabled.as_slice()) { + assert_eq!(enabled.number, disabled.number); + if !matches!(enabled.number, 0x101d1 | 0x0d479 | 0x107a1) { + assert_eq!(enabled.value, disabled.value); + } + } + // The three request registers still carry the request itself. + assert_eq!(register_value(program.as_slice(), 0x101d1, 0), 0x0001_0059); + assert_eq!(register_value(program.as_slice(), 0x107a1, 0), 0x0100_0000); + assert_eq!( + compute_resume_checkpoints(program.as_slice()).unwrap(), + [39, 40, 40, 41] + ); + } + + #[test] + fn no_scs_request_emits_no_enable_register() { + let program = build_compute_registers(register_parameters()).unwrap(); + assert!(program + .as_slice() + .iter() + .all(|register| register.number != 0x17d9)); + assert_eq!(register_value(program.as_slice(), 0x107a1, 0), 0xff_0000); + } + + #[test] + fn an_empty_scs_request_reproduces_the_historical_registers() { + // Zero pages is the kext's "this pass asks for no ray-traversal + // scratch" branch, and is what this driver emitted before the request + // became settable. register_program_matches_the_hardware_tested_m1n1_builder + // above pins the same thing through the whole program's hash. + let scs = scs_request_registers(0, 0, 0, 0).unwrap(); + assert_eq!(scs.size_word, 0); + assert_eq!(scs.extras, 0); + assert_eq!(scs.window_word, 0xff_0000); + // The window is ignored while no pages are requested. + assert_eq!(scs_request_registers(0, 7, 3, 9), scs_request_registers(0, 7, 3, 0)); + } + + #[test] + fn a_nonempty_scs_request_packs_pages_mib_extras_and_window() { + // 89 pages is the four-section floor (0x58000 bytes) plus the guard + // page AGXMetalG17P adds; one MiB covers it ((89 * 4 + 0x3ff) >> 10). + let scs = scs_request_registers(89, 1, 0, 0).unwrap(); + assert_eq!(scs.size_word, 0x0001_0059); + assert_eq!(scs.extras, 0); + assert_eq!(scs.window_word, 0x0100_0000); + + let wide = scs_request_registers(815, 4, 2, 0x20).unwrap(); + assert_eq!(wide.size_word, 0x0004_032f); + assert_eq!(wide.extras, 2); + assert_eq!(wide.window_word, 0x0120_0000); + } + + #[test] + fn an_unset_mib_is_derived_from_the_page_count_like_metal_does() { + // AGXMetalG17P computes MiB as (pages * 4 + 0x3ff) >> 10 and never + // leaves it zero next to a non-zero page count, so the driver must not + // either: an operator who sets only the page count still gets the + // register value a real ray-tracing kick carries. + // 0x59 pages = the four fixed sections plus the guard page. + assert_eq!(scs_request_registers(0x59, 0, 0, 0).unwrap().size_word, 0x0001_0059); + // 0x32f pages = the intersector layout + // (traversal depth 0x70, driver payload 0x48, 96 cliques). + assert_eq!(scs_request_registers(0x32f, 0, 0, 0).unwrap().size_word, 0x0004_032f); + // 0x3c8 pages = the intersection_query<> layout (driver payload 0x114). + assert_eq!(scs_request_registers(0x3c8, 0, 0, 0).unwrap().size_word, 0x0004_03c8); + // 0x6a7 pages = intersection_query on a + // top-level structure, the Vulkan rayQueryEXT shape + // (traversal depth 0xe8, driver payload 0x128). + assert_eq!(scs_request_registers(0x6a7, 0, 0, 0).unwrap().size_word, 0x0007_06a7); + // An explicit value still wins, so the halves stay separable. + assert_eq!(scs_request_registers(0x59, 9, 0, 0).unwrap().size_word, 0x0009_0059); + // Zero pages keeps the historical register exactly. + assert_eq!(scs_request_registers(0, 0, 0, 0).unwrap().size_word, 0); + } + + #[test] + fn scs_layout_sizes_match_the_measured_metal_shapes() { + let layout = |depth: u64, driver: u64| depth | (96 << 24) | (driver << 48); + + assert_eq!(scs_pages_for_layout(layout(0x70, 0x48)).unwrap(), 0x32f); + assert_eq!(scs_pages_for_layout(layout(0x70, 0x114)).unwrap(), 0x3c8); + assert_eq!(scs_pages_for_layout(layout(0xe8, 0x128)).unwrap(), 0x6a7); + + // The native endComputePass gate suppresses a request when both + // payload sizes are zero, even if depth/clique fields are populated. + assert_eq!(scs_pages_for_layout(0x70 | (96 << 24)).unwrap(), 0); + assert_eq!(scs_pages_for_layout(0).unwrap(), 0); + } + + #[test] + fn scs_layout_rejects_an_unrepresentable_request() { + assert_eq!( + scs_pages_for_layout(u64::MAX), + Err(ComputeBuildError::ScsRequestOutOfRange) + ); + } + + #[test] + fn descriptor_preserves_cdm_sampler_timestamp_and_queue_state() { + let registers = build_compute_registers(register_parameters()).unwrap(); + let mut raw = [0u8; COMPUTE_DESCRIPTOR_SIZE]; + build_compute_descriptor( + ®isters, + ComputeDescriptorObjects { + scheduler_record: 0xffff_fc20_c099_8100, + descriptor_low_alias: 0x70_0034_0000, + dispatch_a: 0xffff_fc20_001c_8028, + dispatch_b: 0xffff_fc20_c07c_0028, + status_a: 0xffff_fc20_0002_4c68, + status_b: 0xffff_fc20_0002_4c70, + shared_control: 0xffff_fc20_c095_0000, + usage_record: 0xffff_fc20_0171_0000, + }, + ComputeDescriptorMetadata { + submit_sequence: 5, + context_id: 3, + grid_index: 2, + work_ordinal: 7, + queue_submission: 8, + queue_ordinal: 7, + submission_index: 9, + support_control: 0xe0a0_0001, + support_flags: 0, + }, + ComputeDescriptorCommand { + cdm_terminator: 0x100_0410_002c, + sampler_array: 0x100_0800_0000, + sampler_count: 4, + user_timestamp_start: 0x200_0000_4000, + user_timestamp_end: 0x200_0000_4008, + }, + &mut raw, + ) + .unwrap(); + + // Re-pinned with the register-derived B1 resume checkpoints, and again + // with the constant completion stamp at +0x0F50 (COMPUTE_COMPLETION_STAMP_CONSTANT). + assert_eq!(fnv1a64(&raw), 0x8a56_8b1c_2a27_0524); + assert_eq!(u64::from_le_bytes(raw[0x0ee0..0x0ee8].try_into().unwrap()), 0x100_0410_002c); + assert_eq!(u64::from_le_bytes(raw[COMPUTE_SAMPLER_ARRAY..COMPUTE_SAMPLER_ARRAY + 8].try_into().unwrap()), 0x100_0800_0000); + assert_eq!(u32::from_le_bytes(raw[COMPUTE_SAMPLER_COUNT..COMPUTE_SAMPLER_COUNT + 4].try_into().unwrap()), 4); + assert_eq!(u64::from_le_bytes(raw[0x0f8c..0x0f94].try_into().unwrap()), 0x200_0000_4000); + // The raw builder retains its bootstrap seed. User publication binds + // the physical queue's actual completion prefix after this step. + assert_eq!(u32::from_le_bytes(raw[0x0f50..0x0f54].try_into().unwrap()), COMPUTE_COMPLETION_STAMP_CONSTANT); + assert_eq!( + [ + u16::from_le_bytes(raw[0x18..0x1a].try_into().unwrap()), + u16::from_le_bytes(raw[0x1a..0x1c].try_into().unwrap()), + u16::from_le_bytes(raw[0x1c..0x1e].try_into().unwrap()), + u16::from_le_bytes(raw[0x1e..0x20].try_into().unwrap()), + ], + [38, 39, 39, 40] + ); + } + + #[test] + fn invalid_command_relations_are_caught_before_encoding() { + let registers = build_compute_registers(register_parameters()).unwrap(); + let objects = ComputeDescriptorObjects { + scheduler_record: 1, + descriptor_low_alias: 0x70_0000_0000, + dispatch_a: 2, + dispatch_b: 3, + status_a: 4, + status_b: 5, + shared_control: 6, + usage_record: 7, + }; + let metadata = ComputeDescriptorMetadata { + submit_sequence: 1, + context_id: 3, + grid_index: 2, + work_ordinal: 0, + queue_submission: 1, + queue_ordinal: 0, + submission_index: 1, + support_control: 0xe0a0_0001, + support_flags: 0, + }; + let mut raw = [0u8; COMPUTE_DESCRIPTOR_SIZE]; + assert_eq!( + build_compute_descriptor( + ®isters, + objects, + metadata, + ComputeDescriptorCommand { + cdm_terminator: register_parameters().cdm_base - 4, + sampler_array: 0, + sampler_count: 0, + user_timestamp_start: 0, + user_timestamp_end: 0, + }, + &mut raw, + ), + Err(ComputeBuildError::CdmTerminatorBeforeStream) + ); + assert_eq!( + build_compute_descriptor( + ®isters, + objects, + metadata, + ComputeDescriptorCommand { + cdm_terminator: register_parameters().cdm_base, + sampler_array: 0, + sampler_count: 1, + user_timestamp_start: 0, + user_timestamp_end: 0, + }, + &mut raw, + ), + Err(ComputeBuildError::SamplerPair) + ); + } + + #[test] + fn queue_graph_matches_the_hardware_tested_m1n1_builder() { + let mut raw = [0u8; COMPUTE_QUEUE_GRAPH_SIZE]; + let addresses = build_compute_queue_graph( + 0xffff_fc20_c100_0000, + 0xffff_fc20_c200_0000, + &mut raw, + ) + .unwrap(); + assert_eq!( + addresses, + ComputeQueueGraphAddresses { + queue_record: 0xffff_fc20_c100_0000, + pointers: 0xffff_fc20_c100_0100, + job_list: 0xffff_fc20_c100_0200, + item_ring: 0xffff_fc20_c100_4000, + } + ); + assert_eq!(u32::from_le_bytes(raw[0x48..0x4c].try_into().unwrap()), 0); + // Normalize only the captured process identity for the historical hash. + put_u32(&mut raw, 0x48, 0x1aa); + assert_eq!(fnv1a64(&raw), 0x3490_ef31_add0_ac88); + } + + #[test] + fn initial_channel_group_matches_the_output_positive_add3_builder() { + let base = 0xffff_fc20_c100_0000; + let descriptor = 0xffff_fc20_c200_0000; + let mut graph = [0u8; COMPUTE_QUEUE_GRAPH_SIZE]; + build_compute_queue_graph(base, 0xffff_fc20_c500_0000, &mut graph).unwrap(); + put_u32(&mut graph, 0x48, 0x1234_5678); + let mut context = vec![0u8; COMPUTE_QUEUE_CONTEXT_EXTENT]; + let addresses = build_initial_compute_channel_group( + base, + descriptor, + ComputeSksmEntryAliases::new( + 0x70_0040_0000, + 0xffff_fc20_c300_0000, + 0x8000, + 0x100, + ) + .unwrap(), + 0xffff_fc20_c400_0000, + 0xffff_fc20_c500_0000, + 4, + 1, + &mut graph, + &mut context, + true, + None, + ) + .unwrap(); + assert_eq!( + addresses, + ComputeChannelGroupAddresses { + queue: base, + optional: base + 0x6800, + event: base + 0x6c00, + write_index: 3, + } + ); + // The compute pool uses ID 1 consistently in registration and FList. + // Normalize only the captured PID for the original whole-graph golden. + assert_eq!(&graph[COMPUTE_OPTIONAL + 0x46..COMPUTE_OPTIONAL + 0x48], &[5, 0]); + assert_eq!(&graph[0x48..0x4c], &0x1234_5678u32.to_le_bytes()); + assert_eq!(&graph[COMPUTE_OPTIONAL + 0x5a..COMPUTE_OPTIONAL + 0x5e], &0x1234_5678u32.to_le_bytes()); + put_u32(&mut graph, 0x48, 0x170); + put_u32(&mut graph, COMPUTE_OPTIONAL + 0x5a, 0x170); + assert_eq!(fnv1a64(&graph), 0x66c2_81b4_ef4e_f1db); + assert_eq!(fnv1a64(&context), 0x5ba5_b355_8c90_0783); + } + + #[test] + fn config_update_uses_physical_graph_flist_index_not_qos() { + let mut graph = [0u8; COMPUTE_QUEUE_GRAPH_SIZE]; + let aliases = ComputeSksmEntryAliases::new( + 0x70_0040_0000, + 0xffff_fc20_c300_0000, + 0x8000, + 0x100, + ) + .unwrap(); + encode_compute_config_update( + &mut graph, + aliases, + 0xffff_fc20_c400_0000, + 0xffff_fc20_c500_0000, + 5, + 2, + true, + ComputeConfigUpdates { qos: true, context: true }, + None, + ) + .unwrap(); + assert_eq!(&graph[COMPUTE_OPTIONAL + 0x46..COMPUTE_OPTIONAL + 0x48], &[6, 0]); + assert_eq!( + &graph[COMPUTE_OPTIONAL + 0x56..COMPUTE_OPTIONAL + 0x58], + &[2, 0] + ); + } + + #[test] + fn sksm_entry_aliases_preserve_one_backing_offset() { + const EXTENT: usize = 0x8000; + let aliases = ComputeSksmEntryAliases::new( + 0x70_004d_8000, + 0xffff_fc20_0027_8000, + EXTENT, + 0x100, + ) + .unwrap(); + assert_eq!( + aliases.range_at(0x200, 0x200), + Ok([0x70_004d_8200, 0xffff_fc20_0027_8200]) + ); + assert_eq!( + aliases.range_at(EXTENT - 0x200, 0x200), + Ok([0x70_004d_fe00, 0xffff_fc20_0027_fe00]) + ); + assert_eq!( + aliases.range_at(EXTENT, 1), + Err(ComputeBuildError::InvalidSksmEntryAliases) + ); + assert_eq!( + ComputeSksmEntryAliases::new(0x70_004d_8000, 0x70_004d_8000, EXTENT, 0x100), + Err(ComputeBuildError::InvalidSksmEntryAliases) + ); + assert_eq!( + ComputeSksmEntryAliases::new( + 0x70_004d_8001, + 0xffff_fc20_0027_8000, + EXTENT, + 0x100, + ), + Err(ComputeBuildError::InvalidSksmEntryAliases) + ); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/compute_state.rs b/drivers/gpu/drm/asahi/g18/fw/compute_state.rs new file mode 100644 index 0000000000000..6050c0ee2d915 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/compute_state.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Fields copied by the native 26A428 compute-state writer (8bf16cc..8bf18a8). +//! This is a bounded state view, not a command-stream or ownership validator. +//! Host policy, context, pool, scheduler and completion fields stay with callers. + +#![allow(dead_code)] + +use crate::g18::fw::compute::{G18ComputeCommandInputs, G18ComputeRegisterInputs}; + +pub(crate) const G18_COMPUTE_STATE_BYTES: usize = 0x268; + +pub(crate) struct G18ComputeState<'a>(&'a [u8; G18_COMPUTE_STATE_BYTES]); + +impl<'a> G18ComputeState<'a> { + pub(crate) fn parse(bytes: &'a [u8]) -> Option { + Some(Self(bytes.try_into().ok()?)) + } + + fn q(&self, offset: usize) -> u64 { + u64::from_le_bytes(self.0[offset..offset + 8].try_into().unwrap()) + } + + fn w(&self, offset: usize) -> u32 { + u32::from_le_bytes(self.0[offset..offset + 4].try_into().unwrap()) + } + + fn flag(&self, offset: usize) -> u8 { self.0[offset] & 1 } + + /// Replace only the fields sourced from userspace compute state. This + /// deliberately does not synthesize any native host-owned resource fields. + pub(crate) fn apply(&self, c: &mut G18ComputeCommandInputs, r: &mut G18ComputeRegisterInputs) { + c.d488 = self.q(0x08); + c.d490 = self.q(0x10); + c.d498 = self.q(0x18); + c.d4a0 = self.q(0x20); + c.d4a8 = self.q(0x28); + c.d4b0 = self.w(0x30); + c.d4b8 = self.q(0x38); + c.d4c0 = self.w(0x40); + c.d4d0 = self.w(0x50); + c.d4e0 = self.q(0x60); + c.d508 = self.flag(0xc4); + c.d628 = self.0[0xd0]; + c.d629 = self.flag(0x249); + c.d770 = self.flag(0x24c); + c.d8b0 = self.flag(0x24e); + r.program = [self.q(0x58), self.q(0), self.q(0x70), self.q(0x78), self.q(0x80), self.q(0x88)]; + r.aliases = [self.q(0x1f8), self.q(0x200), self.q(0x208), self.q(0x210), self.q(0x218), self.q(0x220)]; + r.d540 = self.flag(0x1f4); + r.d548 = self.q(0xa8); + r.d5d9 = self.flag(0x24f); + r.d62a = self.flag(0x24a); + r.d62b = self.flag(0x24b); + r.d418 = self.w(0x260); + r.d774 = self.w(0xa4); + // The native 16-byte store starts at descriptor +0x5b8, not +0x5c0. + r.d5c0 = self.w(0x230); + r.d5c4 = self.w(0x234); + r.d5c8 = self.w(0x238); + r.d50c = self.w(0xc0); + r.d510 = self.w(0xc8); + r.d514 = self.w(0xcc); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/config.rs b/drivers/gpu/drm/asahi/g18/fw/config.rs new file mode 100644 index 0000000000000..b532b92938c0a --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/config.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fixed choices for the currently qualified G17P firmware path. +//! +//! Values preserve the former module-parameter defaults. Firmware ABI +//! alternatives require a code change and qualification, not a boot-time mix +//! of independent experimental switches. Debugging and timeout controls remain +//! module parameters in `asahi.rs`. + +pub(crate) const RENDER_SLOT_AFFINITY: u32 = 1; +pub(crate) const RENDER_SLOT_GLOBALS: u32 = 3; +pub(crate) const RENDER_NOP_RETIRE: u32 = 1; +pub(crate) const REG_DUMP: u32 = 0; +pub(crate) const SCAN_GATE: u32 = 0; +pub(crate) const NATIVE_ORDER: u32 = 1; +pub(crate) const SUBMIT_POLLS: u32 = 500; +pub(crate) const PROBE_LOW_RANGE: u32 = 0; +pub(crate) const LATE_SKSM: u32 = 0; +pub(crate) const FLUSHID_FIX: u32 = 0; +/// Session compute FList identity; render uses the separate pool ID zero. +pub(crate) const COMPUTE_FLIST_ID: u16 = 1; + +/// Published CL graphs retain distinct pools; render owns zero. The physical +/// QID lease makes this mapping stable across logical owner turnover. +pub(crate) const fn compute_flist_id(queue_id: u8) -> u16 { + queue_id as u16 + 1 +} +pub(crate) const SKIP: u32 = 0; +pub(crate) const DEFER_ADD_KICKS: u32 = 0; +pub(crate) const CTX2_ROOT: u32 = 3; +pub(crate) const PROGRESS_WATCHDOG: u32 = 1; +pub(crate) const POWER_HOLD: u32 = 0; +pub(crate) const POWER_WIRE: u32 = 0; +pub(crate) const IDLE_SELECTOR: u32 = 0; +pub(crate) const RENDER_ABSOLUTE_POINTERS: u32 = 0; +pub(crate) const RENDER_STATUS_FLAG: u32 = 0; +pub(crate) const RENDER_GATE_PROBE: u32 = 0; +pub(crate) const RENDER_KSM_PROBE: u32 = 0; +pub(crate) const RENDER_HEADER_FLAGS: u32 = 0; +pub(crate) const RENDER_GENERATION_BIAS: u32 = 0; +pub(crate) const RENDER_BG_PREFIX: u32 = 0; +pub(crate) const RENDER_EOT_BIND_ZERO: u32 = 1; +pub(crate) const RENDER_MCACHE_HWSID: u32 = 24; +pub(crate) const RENDER_MCACHE_MODE: u32 = 1; +pub(crate) const RENDER_MCACHE_SELECTOR: u32 = 64; +pub(crate) const RENDER_KSM_MCACHE: u32 = 1; +pub(crate) const RENDER_NO_BARRIER: u32 = 0; +pub(crate) const RENDER_SPLIT_LAUNCH: u32 = 0; +pub(crate) const RENDER_NATIVE_DOORBELLS: u32 = 0; +pub(crate) const RENDER_NATIVE_POOL_A_FIRST_RECORD: u32 = 1; +pub(crate) const RENDER_NATIVE_FRAGMENT_HWPB: u32 = 1; +pub(crate) const RENDER_3D_RCE: u32 = 0xff; +pub(crate) const SEED_WORK_CHANNEL: u32 = 0; +pub(crate) const RENDER_3D_STROBE: u32 = 0; +pub(crate) const IRQ_BURST: u32 = 0; +pub(crate) const SLOT_BURST: u32 = 0; +pub(crate) const PIPE_SWEEP: u32 = 0; +pub(crate) const RENDER_TA_DM: u32 = 0xff; +pub(crate) const RENDER_3D_DM: u32 = 0xff; +pub(crate) const FW_TRACE_CLASSES: u32 = 0; +pub(crate) const KSM_COLD_ARM: u32 = 0; +pub(crate) const ADMIT_ARM: u32 = 0; +pub(crate) const RENDER_NO_INSTALL: u32 = 0; +pub(crate) const FRAG_ORACLE: u32 = 0; +pub(crate) const TA_ORACLE: u32 = 0; +pub(crate) const RETAIN_STAGING: u32 = 1; +pub(crate) const RENDER_USC_FREELIST: u32 = 1; +pub(crate) const RECOVERY_ALLOW_ACTIVE_SCHEDULER: u32 = 1; +pub(crate) const RENDER_SERVICE_EVENTS: u32 = 1; +pub(crate) const COMPUTE_SKSM_ENTRY: u32 = 1; +pub(crate) const REPORT_DRAIN: u32 = 0; +pub(crate) const REPORT_DRAIN_BUDGET: u32 = 0; +pub(crate) const RENDER_RECOVERY_RETIRE: u32 = 0; +pub(crate) const COMPUTE_CONFIG_CONTEXT: u32 = 1; +// bit0: scope a faulted async-compute queue's admission poison to its own +// client instead of the whole device (landed with f6a71e767d55a). +// bit1: scope a lost render submission the same way. When a firmware +// stamp-blamed recovery is serviced to a clean final clear (state=0, +// epoch=1) but the driver cannot prove the exact lost render was +// firmware-retired, fail ONLY that client's VM and retire its physical +// slot; keep the session up for every other client. Without this bit the +// render settle path calls note_session_recovery_pending and the whole +// GPU session is declared crashed -- and on G17P a crashed session's +// rebuild never re-runs the RTKit HELLO handshake (4/4 ETIME), so the +// GPU is dead until reboot. Measured 2026-09-26 (07:48, 07:54, and the +// real desktop at 08:12): a single unprivileged client's first render +// submission killed the GPU for everyone. Both scopings are the correct +// fault-containment default, not a diagnostic opt-in: they never touch +// the healthy path's batching or concurrency. +pub(crate) const QUARANTINE_SCOPE: u32 = 3; +pub(crate) const RENDER_NATIVE_REGS: u32 = 0; +pub(crate) const RENDER_NATIVE_BYTES: u32 = 0; +pub(crate) const RENDER_TVB: u32 = 2; +pub(crate) const DEP_RECORDS: u32 = 0; +pub(crate) const NO_IMPLICIT_DEP: u32 = 0; +pub(crate) const RENDER_TA_QID_FIELD: u32 = 1; +pub(crate) const RENDER_SLOT_TEMPLATE: u32 = 0; +pub(crate) const RENDER_SEED_TS: u32 = 0; +pub(crate) const RENDER_KICK_QOS: u32 = 0; +pub(crate) const RENDER_TAG16: u32 = 0; +pub(crate) const RENDER_POPULATED_HIGH_ROOT: u32 = 0; +pub(crate) const RENDER_SKSM: u32 = 3; +pub(crate) const IRQ_COMPLETION: u32 = 1; +pub(crate) const RING_NORMALIZE: u32 = 0; +pub(crate) const RING_LIMIT: u32 = 0; +pub(crate) const RING_WRAP: u32 = 0; +pub(crate) const COMPUTE_SCS_PAGES: u32 = 0; +pub(crate) const COMPUTE_SCS_MIB: u32 = 0; +pub(crate) const COMPUTE_SCS_EXTRAS: u32 = 0; +pub(crate) const COMPUTE_SCS_WINDOW: u32 = 0; +pub(crate) const COMPLETION_STAMP: u32 = 1; +pub(crate) const REPEAT_CONTEXT: u32 = 1; +pub(crate) const REPEAT_RING: u32 = 1; +pub(crate) const REPEAT_ACTIVATION: u32 = 0; +pub(crate) const QUEUE_RECYCLE: u32 = 2; +/// Diagnostic only: 0 allocates a fresh QID graph for each new async-compute +/// owner so retained graph rebinding can be isolated from execution faults. +/// Production must remain 1 once the handoff defect is fixed. +pub(crate) const ASYNC_COMPUTE_REBIND: u32 = 1; +/// Diagnostic only: retain the pre-publication B1 descriptor/FList snapshot +/// so the first halted fault can be compared with the exact bytes it consumed. +pub(crate) const COMPUTE_RESOURCE_SNAPSHOT: u32 = 0; +pub(crate) const ZERO_DURATION: u32 = 1; +pub(crate) const GATE_QUIESCE_MS: u32 = 0; +pub(crate) const RECOVERY_RESYNC: u32 = 1; +pub(crate) const ASC_RESTART: u32 = 0; +pub(crate) const ASC_GATE_SETTLE_MS: u32 = 10; +pub(crate) const ASC_TEARDOWN: u32 = 0; +pub(crate) const ABANDON_RECOVERY: u32 = 1; +pub(crate) const SKSM_MMIO: u32 = 0; + +/// Default-off diagnostic: ordinary Job graphs with a shared per-submit TSQ +/// use native pending-IO-fence-free TA policy. This defers the early TA drain; +/// it does not remove maintenance or relax any post-maintenance fence gate. +/// Bootstrap/private-node paths retain their qualified policy. Do not promote +/// without mixed/failure/chunking liveness and throughput qualification. +pub(crate) const RENDER_NATIVE_TA_DRAIN_PROBE: bool = cfg!(g17p_native_ta_drain_probe); diff --git a/drivers/gpu/drm/asahi/g18/fw/events.rs b/drivers/gpu/drm/asahi/g18/fw/events.rs new file mode 100644 index 0000000000000..123543037ddd0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/events.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Decoder of the firmware completion-stamp event that G18 reads from its report ring. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. + +pub(crate) const G17P_FIRMWARE_EVENT_ENTRY_SIZE: usize = 0x48; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum CompletionDecodeError { + RecordSize, + ChannelNumber, + EventType, + TraceShape, + CompletionSelector, + CompletionCommandType, + CompletionCapacity, + CompletionCursor, + CursorRange, + QueueCounterOrder, + QueueCounterRegression, + PublishedPrefixBeyondWrite, + RenderWindow, + RenderCursorOutsideWindow, + CompletionIrqSource, + CompletionOrdinal, + CompletionNotificationCount, + CompletionQueueId, + CompletionTimestamp, + CompletionTimestampOrder, + FirmwareEventType, + FirmwareEventDiagnosticCount, + FirmwareEventStampIndex, + FirmwareEventStampMissing, + FirmwareEventUmaPoolIndex, + FirmwareEventTvbRequest, + FirmwareEventUmaAddress, + CompletionOutstanding, + CompletionRouteUnavailable, + FirmwareEventEndpoint, + FirmwareEventMessage, + FirmwareEventCursor, + FirmwareEventEmpty, +} + +fn get_u16(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) +} + +fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) +} + +/// Firmware event type 1, delivered through the ordinary EP0x20 event ring. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PFirmwareStampEvent { + pub(crate) masks: [u64; 2], + pub(crate) diagnostic_pair_count: u16, +} + +pub(crate) fn decode_g17p_firmware_stamp_event( + raw: &[u8], +) -> Result { + if raw.len() != G17P_FIRMWARE_EVENT_ENTRY_SIZE { + return Err(CompletionDecodeError::RecordSize); + } + if get_u32(raw, 0x00) != 1 { + return Err(CompletionDecodeError::FirmwareEventType); + } + let diagnostic_pair_count = get_u16(raw, 0x14); + if diagnostic_pair_count >= 6 { + return Err(CompletionDecodeError::FirmwareEventDiagnosticCount); + } + Ok(G17PFirmwareStampEvent { + masks: [get_u64(raw, 0x04), get_u64(raw, 0x0c)], + diagnostic_pair_count, + }) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/graphics_payload.rs b/drivers/gpu/drm/asahi/g18/fw/graphics_payload.rs new file mode 100644 index 0000000000000..5e60e8bf37b84 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/graphics_payload.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded private pre-submit color capture for G18 graphics bring-up. +//! +//! This reader preserves user virtual pages and the userspace render segment. +//! It neither creates a GPU VM nor submits a job. Native TA/3D command builders, +//! kernel-owned graphics resources and completion remain separate prerequisites. +//! Resource usage bits are capture metadata, NOT native PTE permissions. + +#![allow(dead_code)] + +pub(crate) const HEADER_BYTES: usize = 64; +pub(crate) const PAGE_BYTES: usize = 0x4000; +pub(crate) const RECORD_BYTES: usize = 16 + PAGE_BYTES; +pub(crate) const COMMAND_BYTES: usize = 0xac0; +pub(crate) const GEARS_COMMAND_BYTES: usize = 0xad8; +pub(crate) const PAGE_COUNT: usize = 214; +pub(crate) const GEARS_PAGE_COUNT: usize = 231; +pub(crate) const WITNESS_PAGE_COUNT: usize = 215; +pub(crate) const WITNESS: u64 = 0x100_000a_0000; +pub(crate) const WITNESS_DATA_BYTES: usize = 4 + 256 * 256 / 8; +pub(crate) const OUTPUT: u64 = 0x100_0005_8000; +pub(crate) const OUTPUT_BYTES: u64 = 256 * 256 * 4; + +// Root allocation address, length, and captured usage for submission 1 in the +// two independent native color runs. Their shader heaps need not be identical. +const RANGES: [(u64, usize, u32); 24] = [ + (0, 0x10000, 128), + (0x18000, 0x8000, 1), (0x28000, 0x8000, 1), + (0x38000, 0x8000, 1), (0x48000, 0x8000, 1), + (0x58000, 0x8000, 1024), (0x68000, 0xc000, 1024), + (0x100_0000_0000, 0x10000, 128), + (0x100_0001_8000, 0x20000, 129), + (0x100_0004_0000, 0x10000, 128), (OUTPUT, 0x40000, 12), + (0x100_000a_0000, 0x20000, 128), + (0x100_000c_8000, 0x6c000, 128), + (0x100_0013_8000, 0x8000, 1), + (0x100_0014_8000, 0x8000, 4), + (0x100_0015_8000, 0x8000, 1), + (0x100_0016_8000, 0x8000, 4), + (0x100_0017_8000, 0x100000, 4), + (0x100_0028_0000, 0x8000, 4), + (0x100_0029_0000, 0x8000, 1), + (0x100_002a_0000, 0x8000, 1), + (0x100_002b_0000, 0x8000, 1), + (0x100_002c_0000, 0x8000, 1), + (0x100_002d_0000, 0xc0000, 4), +]; + +const GEARS_RANGES: [(u64, usize, u32); 25] = [ + (0, 0x10000, 128), (0x18000, 0x8000, 1), (0x28000, 0x8000, 1), + (0x38000, 0x8000, 1), (0x48000, 0x8000, 1), (0x58000, 0x8000, 1024), + (0x68000, 0xc000, 1024), (0x100_0000_0000, 0x10000, 128), + (0x100_0001_8000, 0x20000, 129), (0x100_0004_0000, 0x10000, 128), + (OUTPUT, 0x40000, 12), (0x100_000a_0000, 0x20000, 128), + (0x100_000c_8000, 0x44000, 12), (0x100_0011_0000, 0x6c000, 128), + (0x100_0018_0000, 0x8000, 1), (0x100_0019_0000, 0x8000, 4), + (0x100_001a_0000, 0x8000, 1), (0x100_001b_0000, 0x8000, 4), + (0x100_001c_0000, 0x100000, 4), (0x100_002c_8000, 0x8000, 4), + (0x100_002d_8000, 0x8000, 1), (0x100_002e_8000, 0x8000, 1), + (0x100_002f_8000, 0x8000, 1), (0x100_0030_8000, 0x8000, 1), + (0x100_0031_8000, 0xc0000, 4), +]; + +// Separate native fragment-witness capture: its extra 16-KiB buffer shifts +// every subsequent pipeline allocation. Never mix these with colour-a/b. +const WITNESS_RANGES: [(u64, usize, u32); 25] = [ + (0, 0x10000, 128), (0x18000, 0x8000, 1), (0x28000, 0x8000, 1), + (0x38000, 0x8000, 1), (0x48000, 0x8000, 1), (0x58000, 0x8000, 1024), + (0x68000, 0xc000, 1024), (0x100_0000_0000, 0x10000, 128), + (0x100_0001_8000, 0x20000, 129), (0x100_0004_0000, 0x10000, 128), + (OUTPUT, 0x40000, 12), (WITNESS, 0x4000, 12), + (0x100_000a_8000, 0x20000, 128), (0x100_000d_0000, 0x6c000, 128), + (0x100_0014_0000, 0x8000, 1), (0x100_0015_0000, 0x8000, 4), + (0x100_0016_0000, 0x8000, 1), (0x100_0017_0000, 0x8000, 4), + (0x100_0018_0000, 0x100000, 4), (0x100_0028_8000, 0x8000, 4), + (0x100_0029_8000, 0x8000, 1), (0x100_002a_8000, 0x8000, 1), + (0x100_002b_8000, 0x8000, 1), (0x100_002c_8000, 0x8000, 1), + (0x100_002d_8000, 0xc0000, 4), +]; + +/// Every colour-a page (address, captured usage) in file order, for the +/// stage-10 layout that is mapped without a capture file. +pub(crate) fn color_a_pages() -> impl Iterator { + RANGES.iter().flat_map(|&(base, length, usage)| + (0..length).step_by(PAGE_BYTES).map(move |offset| (base + offset as u64, usage))) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Capture { ColorA = 1, ColorB = 2, GearsG1 = 3, FragmentWitness = 4 } + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum PayloadError { Length, Header, Checksum, Page, Command, Output, Witness } + +pub(crate) struct GraphicsPayload<'a> { + bytes: &'a [u8], capture: Capture, command_bytes: usize, page_count: usize, +} +pub(crate) struct PayloadPage<'a> { + pub(crate) address: u64, + pub(crate) usage: u32, + pub(crate) bytes: &'a [u8], +} + +fn u32_at(b: &[u8], off: usize) -> u32 { + u32::from_le_bytes(b[off..off + 4].try_into().unwrap()) +} +fn u64_at(b: &[u8], off: usize) -> u64 { + u64::from_le_bytes(b[off..off + 8].try_into().unwrap()) +} + +/// Transfer integrity only; the offline receipt carries SHA256 provenance. +pub(crate) fn fnv64(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |h, b| + (h ^ u64::from(*b)).wrapping_mul(0x100_0000_01b3)) +} + +impl<'a> GraphicsPayload<'a> { + pub(crate) fn parse(bytes: &'a [u8]) -> Result { + if bytes.len() < HEADER_BYTES { return Err(PayloadError::Length); } + let capture = match u64_at(bytes, 56) { + 1 => Capture::ColorA, 2 => Capture::ColorB, 3 => Capture::GearsG1, + 4 => Capture::FragmentWitness, + _ => return Err(PayloadError::Header), + }; + let gears = capture == Capture::GearsG1; + let witness = capture == Capture::FragmentWitness; + let command_bytes = if gears || witness { GEARS_COMMAND_BYTES } else { COMMAND_BYTES }; + let page_count = if gears { GEARS_PAGE_COUNT } else if witness { WITNESS_PAGE_COUNT } else { PAGE_COUNT }; + let size = HEADER_BYTES + command_bytes + page_count * RECORD_BYTES; + if bytes.len() != size { return Err(PayloadError::Length); } + if &bytes[..8] != b"G18GFX01" || u32_at(bytes, 8) != 2 + || u32_at(bytes, 12) != HEADER_BYTES as u32 + || u32_at(bytes, 16) != PAGE_BYTES as u32 + || u32_at(bytes, 20) != page_count as u32 + || u32_at(bytes, 24) != 256 || u32_at(bytes, 28) != 256 + || u32_at(bytes, 32) != 1024 + || u32_at(bytes, 36) != command_bytes as u32 + || u64_at(bytes, 40) != OUTPUT + { return Err(PayloadError::Header); } + if fnv64(&bytes[HEADER_BYTES..]) != u64_at(bytes, 48) { + return Err(PayloadError::Checksum); + } + let p = Self { bytes, capture, command_bytes, page_count }; + let cmd = p.command(); + // Fixture identity is bound to the actual userspace command; the two + // colour shader heaps must never be interchanged by relabelling a file. + let command_fnv = match capture { + Capture::ColorA => 0x89e7_aca5_9572_cb78, + Capture::ColorB => 0x75e0_2b53_1fcc_46f2, + Capture::GearsG1 => 0x23d3_27af_95ce_9eb7, + Capture::FragmentWitness => 0x5eed_ef2d_0bb3_31af, + }; + if u32_at(cmd, 0) != 0x10000 || u32_at(cmd, 4) != command_bytes as u32 + || u64_at(cmd, command_bytes - 24) != OUTPUT || fnv64(cmd) != command_fnv + { return Err(PayloadError::Command); } + let mut index = 0; + let mut output_pages = 0; + let ranges: &[(u64, usize, u32)] = if gears { &GEARS_RANGES } + else if witness { &WITNESS_RANGES } else { &RANGES }; + for &(base, length, usage) in ranges { + for offset in (0..length).step_by(PAGE_BYTES) { + let page = p.page(index).ok_or(PayloadError::Page)?; + let record = HEADER_BYTES + command_bytes + index * RECORD_BYTES; + if page.address != base + offset as u64 || page.usage != usage + || u32_at(bytes, record + 12) != PAGE_BYTES as u32 + { return Err(PayloadError::Page); } + if (OUTPUT..OUTPUT + OUTPUT_BYTES).contains(&page.address) { + if page.bytes.iter().any(|&b| b != 0xa5) { + return Err(PayloadError::Output); + } + output_pages += 1; + } + if witness && page.address == WITNESS + && (page.bytes[..WITNESS_DATA_BYTES].iter().any(|&b| b != 0) + || page.bytes[WITNESS_DATA_BYTES..].iter().any(|&b| b != 0xa5)) + { return Err(PayloadError::Witness); } + index += 1; + } + } + if index != page_count || output_pages != 16 { return Err(PayloadError::Page); } + Ok(p) + } + + pub(crate) fn capture(&self) -> Capture { self.capture } + pub(crate) fn body_digest(&self) -> u64 { u64_at(self.bytes, 48) } + pub(crate) fn page_count(&self) -> usize { self.page_count } + pub(crate) fn command(&self) -> &'a [u8] { + &self.bytes[HEADER_BYTES..HEADER_BYTES + self.command_bytes] + } + pub(crate) fn page(&self, index: usize) -> Option> { + if index >= self.page_count { return None; } + let off = HEADER_BYTES + self.command_bytes + index * RECORD_BYTES; + Some(PayloadPage { address: u64_at(self.bytes, off), usage: u32_at(self.bytes, off + 8), + bytes: &self.bytes[off + 16..off + RECORD_BYTES] }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/initdata.rs b/drivers/gpu/drm/asahi/g18/fw/initdata.rs new file mode 100644 index 0000000000000..ae404f123b4fb --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/initdata.rs @@ -0,0 +1,6549 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! Evidence-backed T8140/G17P firmware init-data layout. +//! +//! HISTORY / PROVENANCE. The previous revision of this module was derived from +//! the **M5 / G17S** on-disk firmware `GFX.a010.macho` (parser at VA +//! 0xfffffc000002127c) — a different chip and a different firmware build from +//! the target, **A18 Pro / T8140 / G17P**. G17S and G17P share a generation +//! name but not an ABI: every structure size in that model was wrong, the five +//! `+0x254` pointers were modeled as independent allocations instead of views +//! into one bundle, and the M5 magic 0x0c8d_e339_0728_04c0 is rejected by the +//! T8140 firmware with "Host and FW Interface is mismatched". This revision is +//! re-derived from: +//! - live T8140 hardware runs (Mac17,5 / J700 / T8140), which booted both +//! firmware instances with a source-constructed descriptor; +//! - our own J700 firmware image: the version-word gate is the MOVZ/MOVK +//! chain at `__TEXT+0x1e8a8` (`mov x10,#0x4c0; movk #0x396,lsl 16; +//! movk #0xa322,lsl 32; movk #0xc8a,lsl 48`), and the same routine +//! confirms the endpoint-0x20 doorbell contract +//! (`(msg & 0xbf<<48) == 0x81<<48`, address rebuilt as +//! `msg | 0xfffff0_0000_0000_0000`); +//! - the executable serializers in the m1n1 tree +//! (`proxyclient/m1n1/agx/g17p.py`, `g17p_initdata.py`) and +//! `docs/t8140-g17p-firmware-abi-spec.md`, both validated by byte-exact +//! rebuilds of pre-handoff captures. +//! +//! The object graph: each firmware instance is handed one root (`0xb8` bytes +//! for the primary, `0xc8` for the secondary). The root carries the version +//! tuple, the full UAT geometry (page size, level count, and three 0x20-byte +//! level descriptors), and points at a per-instance main configuration +//! (`0x600`), the shared region A (16 KiB), the shared region C (`0x1000`), +//! and per-instance status blocks. The main configuration points at the shared +//! hardware-data object (`0x3db4`, at the base of a contiguous bundle +//! allocation of at least `0x10000`) and carries a 17-entry channel table. +//! The five bare pointers at main `+0x254` are views into that one bundle; +//! allocating them independently breaks the overlap and offset relations +//! firmware relies on. +//! +//! Byte slices are used instead of packed Rust structures because several +//! proven fields are unaligned (main `+0x254`, main `+0x471`, and the region +//! record address at `+0x0c`). + +use core::ops::Range; +use core::result::Result; + +/// The recovered fields are sufficient for offline validation and encoding. +pub(crate) const GROUNDED_LAYOUT_AVAILABLE: bool = true; +/// The init-data object graph the T8140 firmware *accepts and boots with* — +/// per-role root sizes, the version tuple, the UAT geometry block, the main +/// configuration, the bundle-view relations, the 17-entry channel table, the +/// hardware-data acceptance block, region C, and the status blocks — is +/// grounded by live T8140 boots of a source-constructed descriptor. This is +/// what `INITDATA_GRAPH` gates. +pub(crate) const PARSE_ACCEPTED_GRAPH_AVAILABLE: bool = true; +/// Whether every hardware-data scalar the firmware reads is grounded to a +/// concrete *derivation* (rather than a recorded value). It is not: the +/// performance ladders/voltage columns and the recorded opaque platform +/// constants ([`HW_DATA_OPAQUE_RUNS`], [`REGION_C_RUNS`]) still lack an ADT +/// derivation, and this module does not encode the performance tables at all +/// (see [`PERF_TABLE_VALUES_ENCODED`]). +pub(crate) const ALL_HW_GLOBALS_VALUES_GROUNDED: bool = false; +/// The complete hardware-validated T8140 performance tables are encoded. +/// Their eventual ADT derivation remains future work, so they are retained as +/// target-scoped measured values rather than inferred from an AGX2 config. +pub(crate) const PERF_TABLE_VALUES_ENCODED: bool = true; +/// The sparse *content* of the five bundle views and of the bundle's third +/// page (`MAIN_ADDR_OBJECTS` / `HWDATA_BUNDLE_STATIC_RUNS` in `g17p.py`, +/// several KiB of recorded runs) is not yet carried by this module; only the +/// view topology is. A live boot needs those runs staged into the bundle. +pub(crate) const BUNDLE_VIEW_CONTENT_CARRIED: bool = true; +/// The primary status-B pointer names a large (`0xeb00`) status/configuration +/// object, not merely the 0x80-byte block modeled here: it holds FW-control +/// state/ring pointers at `+0x48e0`/`+0x48e8`, a lifecycle header at +/// `+0xe434`, and sparse configuration from `+0xe440` +/// (`NATIVE_PRIMARY_STATUS_B_CONFIG_RUNS` in `g17p.py`). Not yet carried. +pub(crate) const PRIMARY_STATUS_B_FULL_OBJECT_CARRIED: bool = true; + +// --- Object sizes ------------------------------------------------------------ + +/// Root size for the primary (`gfx-asc`) instance. +pub(crate) const ROOT_SIZE_PRIMARY: usize = 0x0b8; +/// Root size for the secondary (`gfx1-asc`) instance; the extra 0x10 bytes are +/// the two secondary-only pointers at `+0xb8`/`+0xc0`. +pub(crate) const ROOT_SIZE_SECONDARY: usize = 0x0c8; +pub(crate) const MAIN_CONFIG_SIZE: usize = 0x600; +pub(crate) const HW_DATA_SIZE: usize = 0x3db4; +/// Native QoS manager block published through hardware data. +pub(crate) const G17P_QOS_RESOURCE_OFFSET: usize = 0x27a4; +/// The QoS table pointer as the LIVE 25G83 build reads it. +/// +/// KEXT_PROVEN (25G83 decompile): the live initdata object is 0x2708 bytes and +/// no instruction in that build references 0x27a4 or 0x27ac at all -- both of +/// our writes land past its end, in space the firmware never reads. The real +/// field is +0x26ac, written from `AGXQOSManager + 0x28`. The neighbouring +/// live fields are +0x2614 (the 4 x 0x20 KSM completion table, which we do +/// populate), +0x2694 (a 0x40-entry table fwva), +0x2604 (deferred-kick-mode +/// selector) and the flags at +0x2600/+0x2608/+0x2610. +/// +/// We keep writing 0x27a4 as well: if that offset really is dead space the +/// extra store is inert, and leaving it in place keeps this a single-variable +/// change. +pub(crate) const G17P_QOS_RESOURCE_LIVE_OFFSET: usize = 0x26ac; +pub(crate) const G17P_QOS_RESOURCE_SIZE: usize = 0xe40; +/// Native global SKSM/QID configuration block in hardware data. +pub(crate) const G17P_SKSM_QID_CONFIG_2700_OFFSET: usize = 0x2700; +pub(crate) const G17P_SKSM_QID_CONFIG_2708_OFFSET: usize = 0x2708; +pub(crate) const G17P_SKSM_QID_RESOURCE_OFFSET: usize = 0x27ac; +/// Host->firmware transport selector, `initdata[0x26f8] = accel[0x4f8] & 1` +/// (`AGXFirmware::initFirmwareData` @0xfffffe0008a20940). 1 = the host +/// publishes channel commands; 0 = the host drives the MMIO kick bridge. +const HW_DATA_TRANSPORT_SELECTOR_OFFSET: usize = 0x26f8; +/// MEASURED, live 25G83 hardware-data block: native reads 0 here. +/// initFirmwareData clears the u32 at +0x26fe, which covers +0x2700..+0x2701, +/// and never sets it. Our 1 was a non-native write inside the live object -- +/// the last non-perf difference in the whole 0x2708-byte structure. +pub(crate) const G17P_SKSM_QID_CONFIG_2700: u64 = 0; +pub(crate) const G17P_SKSM_QID_CONFIG_2708: u32 = 0x100; +/// Firmware log enable word inside main config. +pub(crate) const MAIN_FIRMWARE_LOG_ENABLE: usize = 0x250; + +pub(crate) const G17P_SKSM_QID_RESOURCE_SIZE: usize = 0x2000; +pub(crate) const REGION_A_SIZE: usize = 0x4000; +pub(crate) const REGION_C_SIZE: usize = 0x1000; +pub(crate) const STATUS_BLOCK_SIZE: usize = 0x80; + +/// The hardware-data bundle: one contiguous allocation with the hardware-data +/// object at its base. Its principal content spans `0xc000`, but the repeated +/// writable region at `+0xc500` (named by main `+0x08`/`+0x10`) lies past +/// that, so the allocation must reach at least `0x10000`. +pub(crate) const HW_DATA_BUNDLE_SIZE: usize = 0xc000; +pub(crate) const HW_DATA_BUNDLE_MIN_ALLOC: usize = 0x10000; +/// The native placement additionally hosts the twelve work rings and both main +/// configuration objects inside the same cluster, which then spans this much +/// (see [`WORK_RING_BUNDLE_OFFSETS`], [`NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET`]). +pub(crate) const NATIVE_SHARED_CLUSTER_SIZE: usize = 0x28000; +/// Firmware-high base of the complete hardware-data/main-config/work-ring +/// cluster in every output-positive G17P capture. +pub(crate) const NATIVE_SHARED_CLUSTER_GPU_VA: u64 = 0xffff_fc20_c078_8000; + +// --- Version tuple ------------------------------------------------------------ + +/// Root `+0x00`: four u16 version words, `{0x04c0, 0x0396, 0xa322, 0x0c8a}`. +/// Grounded twice: read out of every live T8140 root, and independently +/// re-derived from the MOVZ/MOVK chain at `__TEXT+0x1e8a8` of our own J700 +/// firmware image. A mismatch is rejected with "Host and FW Interface is +/// mismatched" (the M5/G17S image wants 0x0c8d_e339_0728_04c0 instead — that +/// value must never be sent to T8140 firmware). +pub(crate) const INITDATA_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; +/// The same tuple as the little-endian u64 the firmware compares in one go. +pub(crate) const INITDATA_VERSION_WORD: u64 = 0x0c8a_a322_0396_04c0; + +/// Target/build-specific root encoding. This is a serialization choice, not +/// permission to start firmware or reuse the other target's hardware tables. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RootProfile { + J700G17P, + /// J813 Mac17,3 / 25G76, RTKit-3255.160.4.release. Both roots were captured + /// before INBOX1 publication. Independently confirmed by the loaded GFX + /// version comparison at +0x1fa5c and GFX1 at +0x21140. + J813G17G25G76, + /// J873 Mac18,5 / T8152 / G18G / 26A428, RTKit_release-3514.0.15 + /// (armfw_g18g-b000 on GFX, armfw1_g18g-b010 on GFX1). No capture of a + /// native root exists; the encoding is source-constructed from the 26A428 + /// kext and firmware (see the "J873 / G18G" section below). Selecting it + /// changes nothing for the other two profiles. + J873G18G26A428, +} + +impl RootProfile { + pub(crate) const fn version_word(self) -> u64 { + match self { + Self::J700G17P => INITDATA_VERSION_WORD, + Self::J813G17G25G76 => 0x0c8a_a322_039e_04c0, + Self::J873G18G26A428 => G18G_26A428_INITDATA_VERSION_WORD, + } + } + + /// Bytes of the root object for `role`. J700/J813 keep the historical + /// 0xb8/0xc8 split. G18G sizes both roots to 0xc8 because both firmware + /// images copy 0xc8 bytes from each root (FW_PROVEN b000 0x21664-0x216a8). + pub(crate) const fn root_size(self, role: InstanceRole) -> usize { + match self { + Self::J873G18G26A428 => G18G_26A428_ROOT_SIZE, + Self::J700G17P | Self::J813G17G25G76 => root_size(role), + } + } + + /// Opcode of the one opening device-control record staged at main + /// `+0x4c0`. G18G renumbered the opening handler: 0x16 is now an + /// unrelated handler and 0x15 reaches the opening (FW_PROVEN jump table + /// 0xfffffc0000006c60, handler b000 0x3ed0). The secondary keeps 0x2a. + pub(crate) const fn control_opening_opcode(self, role: InstanceRole) -> u32 { + match (self, role) { + (Self::J873G18G26A428, InstanceRole::Primary) => { + G18G_26A428_CONTROL_OPENING_OPCODE_PRIMARY + } + (Self::J873G18G26A428, InstanceRole::Secondary) => { + G18G_26A428_CONTROL_OPENING_OPCODE_SECONDARY + } + (_, InstanceRole::Primary) => CONTROL_OPENING_OPCODE_PRIMARY, + (_, InstanceRole::Secondary) => CONTROL_OPENING_OPCODE_SECONDARY, + } + } + + /// The "primary register" device-control opcode (0x20 on J700/M5). + /// FW_PROVEN: G18G opcode 0x20 is in the invalid set of the 56-entry + /// jump table, so the profile has none and callers must not emit one. + pub(crate) const fn control_register_opcode(self) -> Option { + match self { + Self::J873G18G26A428 => None, + Self::J700G17P | Self::J813G17G25G76 => Some(CONTROL_OPENING_PRIMARY_REGISTER_OPCODE), + } + } + + /// Bundle offsets of the five views published at main `+0x254`. + pub(crate) const fn bundle_view_offsets(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => G18G_26A428_BUNDLE_VIEW_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_OFFSETS, + } + } + + /// Extents each view must fit inside the bundle allocation. + pub(crate) const fn bundle_view_extents(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => G18G_26A428_BUNDLE_VIEW_EXTENTS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_EXTENTS, + } + } + + /// Bundle offset named twice by main `+0x08`/`+0x10` (the empty BRN + /// table and the 0x88 block, which the allocator places together). + pub(crate) const fn repeated_region_bundle_offset(self) -> usize { + match self { + Self::J873G18G26A428 => G18G_26A428_SHARED_LAYOUT.block_88, + Self::J700G17P | Self::J813G17G25G76 => MAIN_REPEATED_REGION_BUNDLE_OFFSET, + } + } + + /// Bundle offsets of the twelve work rings in channel-table order. + pub(crate) const fn work_ring_bundle_offsets(self) -> [usize; WORK_CHANNEL_COUNT] { + match self { + Self::J873G18G26A428 => G18G_26A428_WORK_RING_BUNDLE_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => WORK_RING_BUNDLE_OFFSETS, + } + } +} + +// --- Address representability -------------------------------------------------- + +// The endpoint-0x20 doorbell carries the root address in a 44-bit field; the +// J700 firmware rebuilds it as `msg | 0xfffff0_0000_0000_0000`. Every pointer +// placed in the graph must therefore be either bare or carry exactly that +// kernel-half prefix, or it silently truncates. +const INITDATA_ADDR_MASK: u64 = 0x0000_0fff_ffff_ffff; +const INITDATA_ADDR_PREFIX: u64 = 0xffff_f000_0000_0000; + +// --- Root layout --------------------------------------------------------------- + +const ROOT_VERSION: usize = 0x00; +const ROOT_REGION_A: usize = 0x08; +/// Root `+0x10` is zero at handoff on both instances; semantics unknown. +const ROOT_ZERO_10: usize = 0x10; +const ROOT_MAIN_CONFIG: usize = 0x18; +const ROOT_REGION_C: usize = 0x20; +const ROOT_INSTANCE_KIND: usize = 0x28; +const ROOT_CONSTANT_2C: usize = 0x2c; +const ROOT_PAGE_SIZE: usize = 0x30; +const ROOT_PAGE_BITS: usize = 0x32; +const ROOT_LEVEL_COUNT: usize = 0x33; +const ROOT_LEVELS: usize = 0x34; +/// Root `+0xa8`: Status A pointer. (The M5-derived model misused this slot as +/// a role pointer.) +const ROOT_STATUS_A: usize = 0xa8; +/// Root `+0xb0`: Status B pointer. Real for the primary, zero for the +/// secondary. (The M5-derived model misused it as a cross-role pointer.) +const ROOT_STATUS_B: usize = 0xb0; +const ROOT_SECONDARY_EXTRA_0: usize = 0xb8; +const ROOT_SECONDARY_EXTRA_1: usize = 0xc0; + +// --- UAT geometry (root +0x30..+0x94) ----------------------------------------- + +pub(crate) const UAT_PAGE_SIZE: u16 = 0x4000; +pub(crate) const UAT_PAGE_BITS: u8 = 14; +pub(crate) const UAT_LEVEL_COUNT: usize = 3; +/// `(index_shift, entry_count)` for the three levels below the root, +/// confirmed against a live table walk. +pub(crate) const UAT_LEVELS: [(u8, u16); UAT_LEVEL_COUNT] = [(36, 64), (25, 2048), (14, 2048)]; +pub(crate) const UAT_LEVEL_DESC_SIZE: usize = 0x20; +/// Level descriptor `+0x10`: physical mask, identical in all three. +pub(crate) const UAT_LEVEL_PHYS_MASK: u64 = 0x0000_03ff_ffff_c000; + +// --- Main configuration layout -------------------------------------------------- + +const MAIN_HW_DATA: usize = 0x00; +/// The same repeated-region address appears at `+0x08` and `+0x10`; it is the +/// bundle base plus [`MAIN_REPEATED_REGION_BUNDLE_OFFSET`], not the bundle +/// base itself. It reads zero at handoff (firmware writes into it), so the +/// host only has to provide it mapped. +const MAIN_REPEATED_0: usize = 0x08; +const MAIN_REPEATED_1: usize = 0x10; +pub(crate) const MAIN_REPEATED_REGION_BUNDLE_OFFSET: usize = 0xc500; +const MAIN_CHANNEL_TABLE: usize = 0x20; +/// Main `+0x254`: five unaligned u64 views into the hardware-data bundle. +const MAIN_BUNDLE_VIEWS: usize = 0x254; +/// Main `+0x2d0`: six qwords of region views. +const MAIN_REGION_VIEWS: usize = 0x2d0; +/// Main `+0x300`: u32 4, secondary only. +const MAIN_SECONDARY_KIND: usize = 0x300; +const MAIN_SECONDARY_KIND_VALUE: u32 = 4; +/// Main `+0x3e0`: u32 0xff, primary only. +const MAIN_PRIMARY_MASK: usize = 0x3e0; +const MAIN_PRIMARY_MASK_VALUE: u32 = 0xff; +const MAIN_PRIMARY_SCALAR_32C: usize = 0x32c; +const MAIN_PRIMARY_SCALAR_344: usize = 0x344; +/// Main `+0x471`: unaligned u64, secondary only, pointing *into* bundle view 2 +/// at [`SECONDARY_EXTRA_VIEW_OFFSET`] (not a fresh allocation — the M5-derived +/// model had this at `+0x481` backed by an invented 0x12688-byte buffer; both +/// were wrong). +const MAIN_SECONDARY_EXTRA_PTR: usize = 0x471; +pub(crate) const SECONDARY_EXTRA_VIEW_INDEX: usize = 2; +pub(crate) const SECONDARY_EXTRA_VIEW_OFFSET: usize = 0xe40; +/// Main `+0x4c0` is both a scalar and a place: the leading u32 of the +/// device-control ring, which is embedded in the main configuration object at +/// exactly this offset on both instances. Writing the opening opcode here *is* +/// staging the opening device-control record (records are 0x40 bytes; fields +/// past the opcode are zero). The control channel's producer counter must then +/// be set to the role-specific value returned by +/// [`control_opening_producer`] before the descriptor is handed over. +const MAIN_CONTROL_RING: usize = 0x4c0; +pub(crate) const CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x16; +pub(crate) const CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; +pub(crate) const CONTROL_RECORD_SIZE: usize = 0x40; +pub(crate) const CONTROL_OPENING_PRIMARY_RECORDS: usize = 1; +pub(crate) const CONTROL_OPENING_SECONDARY_RECORDS: usize = 1; +pub(crate) const CONTROL_OPENING_PRIMARY_PRODUCER: u32 = 1; +pub(crate) const CONTROL_OPENING_SECONDARY_PRODUCER: u32 = 1; +pub(crate) const CONTROL_OPENING_PRIMARY_REGISTER_OPCODE: u32 = 0x20; +pub(crate) const CONTROL_OPENING_SECONDARY_CONTINUE_OPCODE: u32 = 0x22; +/// Hardware-buffer release emitted after the opening UMA grow request retires. +pub(crate) const CONTROL_BOOTSTRAP_UMA_RELEASE_OPCODE: u32 = 0x2e; +/// Scheduler-state (channel control) release. +/// +/// Decoded from AGXG17P 353.14 (25G83) `AGXArmFirmware::submitReleaseResource` +/// at `0xfffffe00088a722c`: the record's first word is `0x14` on this image +/// (the G17X image audited for the G17 native detach plan +/// carries `0x13`; the G17P opcode route table at `0xfffffe0007136180` sends +/// `0x14` to ring 0, the same primary control ring as `0x2e`, and `0x13` to +/// ring 1). `AGXCommandQueue::freeReclaimableResources` (`0xfffffe0008865354`) +/// submits it with the four identity bytes read back from the live scheduler +/// state page and then waits for the ring consumer to pass the record before +/// it frees the work queues and their parameter management. +pub(crate) const CONTROL_SCHEDULER_STATE_RELEASE_OPCODE: u32 = 0x14; +/// `AGXCommandQueue::processExitTimeoutHandler` (@fffffe000886a494): kill +/// the work of one scheduler-state page; acknowledged by firmware event 12 +/// (`kAGFIFirmwareEventTypeProcessExitComplete`) carrying the cookie. +pub(crate) const CONTROL_CONTEXT_KILL_OPCODE: u32 = 0x29; +/// Device-control ring capacity. FW_PROVEN (b1 +/// `AGFAcceleratorDrainDeviceControlRing` @ `0xfffffc0000003a74`): the drain +/// walks `ring + index * 0x40` and advances its consumer with +/// `(consumer + 1) & 0xff`, so the ring is exactly 256 records. The placement +/// agrees byte for byte: the primary ring starts at +/// [`NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET`] `+ 0x4c0` = bundle `+0x1ea80` and the +/// secondary main configuration begins at bundle `+0x22a80`, exactly +/// `0x100 * 0x40` bytes later. +pub(crate) const CONTROL_RING_ENTRIES: usize = 0x100; +pub(crate) const CONTROL_RING_INDEX_MASK: u32 = CONTROL_RING_ENTRIES as u32 - 1; +/// Device control `IdlePowerOff(arg)`. FW_PROVEN: the opcode jump table at +/// `0xfffffc0000005380` (indexed by `opcode - 4`) sends `0x0a` to +/// `0xfffffc0000003fd8`, which stores the record's `arg` into the idle-power-off +/// enable word `[0xfffffc0000177838]` and, **only when that arg is zero**, does +/// an unconditional `bl 0xfffffc0000021960`. That routine is the one thing in +/// the image that releases KSM pause reason bit 1 (mask `0x2`) in steady state: +/// it latches `[0xfffffc00001771f8] |= 2` and tail-calls +/// `gpu_core_power(0, 1, 0)` @ `0xfffffc000000e004`, whose power-on arm clears +/// the reason bit at `0xe488` and emits KTrace `0x46 (0, 0x2, mask)` at +/// `0xe4cc`. It is gated by nothing -- unlike the EP-0x21 work doorbell, which +/// handler `0x261a4` drops silently while `[0x1771f8] & 0x18` (RTKit +/// SLEEP/NAP) -- and it is idempotent, because `0x21960` returns immediately +/// when the powered latch is already set. +pub(crate) const DEVICE_CONTROL_OPCODE_IDLE_POWER_OFF: u32 = 0x0a; +/// Disable ordinary idle power-off and request that the GPU cores wake now. +/// +/// FW_PROVEN (25F80 B100, SHA256 83af9ae0...): opcode 0x0a stores this value +/// in `[0xfffffc0000177838]` and calls the wake path only for zero. The +/// scheduler's 0x20000000 idle event skips its descent while that policy word +/// is zero. A cold wake may reload the word from Region-C +0x30, so callers +/// which need a scoped hold must retire a second zero after the core is up. +pub(crate) const DEVICE_CONTROL_IDLE_POWER_OFF_ARG_DISABLE_IDLE_AND_WAKE: u32 = 0; +/// Restore the firmware's ordinary idle power-off policy. A non-zero value +/// permits the scheduler idle event to descend; it is not a power hold. +pub(crate) const DEVICE_CONTROL_IDLE_POWER_OFF_ARG_ENABLE_IDLE: u32 = 1; +/// Separate retained compute resources from the render pool. Native +/// populateInitialFirmwareState (8aa258c) publishes resource-derived +0x14 +/// page-list and +0x30 run-list addresses; neither is firmware-fixed. +pub(crate) const COMPUTE_FLIST_RELOCATION: u64 = 0x0200_0000; +pub(crate) const COMPUTE_FLIST_PAGE_LIST_ADDRESS: u64 = 0x70_0000_0000 + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_FLIST_RUN_TABLE_ADDRESS: u64 = CONTROL_OPERAND_TABLE_ADDRESS + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_FLIST_BUFFER_BASE: u64 = CONTROL_OPERAND_BUFFER_BASE + COMPUTE_FLIST_RELOCATION; + +pub(crate) const COMPUTE_RUNTIME_CONTROL_RECORDS: usize = 65; +pub(crate) const COMPUTE_RUNTIME_CONTROL_FINAL_COUNTER: u32 = + CONTROL_OPENING_PRIMARY_PRODUCER + COMPUTE_RUNTIME_CONTROL_RECORDS as u32; +pub(crate) const COMPUTE_RUNTIME_CLASS1_RECORD_INDEX: usize = 54; +pub(crate) const COMPUTE_RUNTIME_CLASS2_RECORD_INDEX: usize = 57; +pub(crate) const COMPUTE_RUNTIME_SUPPORT_ADDRESS: u64 = 0xffff_fc20_c086_8000; +pub(crate) const COMPUTE_RUNTIME_STATE_ADDRESS: u64 = 0xffff_fc20_0163_0000; +pub(crate) const COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS: u64 = 0x70_01ad_8000 + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_RUNTIME_PAGE_LIST_ADDRESS: u64 = 0x70_018d_0000 + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_RUNTIME_ZERO_BUFFER_0_ADDRESS: u64 = 0x70_0000_0000 + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_RUNTIME_ZERO_BUFFER_1_ADDRESS: u64 = 0x70_0010_8000 + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_RUNTIME_SUPPORT_SIZE: usize = 0x4000; +pub(crate) const COMPUTE_RUNTIME_CLASS1_TABLE_SIZE: usize = 0x4000; +pub(crate) const COMPUTE_RUNTIME_PAGE_LIST_SIZE: usize = 0xa800; +pub(crate) const COMPUTE_DISPATCH_RECORD_SIZE: usize = 0x20; + +/// Output-positive first-CL readiness suffix. These six records follow the +/// clean three-record QoS opening. Native publishes and retires each slot +/// before exposing the next. +pub(crate) const COMPUTE_READINESS_RECORDS: usize = 6; +/// Output-positive final compute graph maps and names exactly operand entries +/// 0..20. The retained opening still owns 28 backing objects. +pub(crate) const COMPUTE_READINESS_OPERAND_ENTRY_COUNT: usize = 21; +pub(crate) const COMPUTE_READINESS_FINAL_COUNTER: u32 = + CONTROL_OPENING_PRIMARY_PRODUCER + COMPUTE_READINESS_RECORDS as u32; +pub(crate) const COMPUTE_READINESS_CLASS1_SUPPORT_ADDRESS: u64 = 0xffff_fc20_c087_8000; +pub(crate) const COMPUTE_READINESS_CLASS1_STATE_ADDRESS: u64 = 0xffff_fc20_0164_8000; +pub(crate) const COMPUTE_READINESS_CLASS3_SUPPORT_ADDRESS: u64 = 0xffff_fc20_c085_0000; +pub(crate) const COMPUTE_READINESS_CLASS3_STATE_ADDRESS: u64 = 0xffff_fc20_0165_0000; +pub(crate) const COMPUTE_READINESS_PAGE_SIZE: usize = 0x4000; +pub(crate) const COMPUTE_READINESS_ACTIVATION_RECORD_INDEX: usize = 2; +pub(crate) const COMPUTE_READINESS_ACTIVATION_RECORD_OFFSET: usize = + COMPUTE_READINESS_ACTIVATION_RECORD_INDEX * CONTROL_RECORD_SIZE; + +/// One native first-CL device-control publication boundary. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeReadinessRecordBoundary { + pub(crate) index: u8, + pub(crate) producer_before: u32, + pub(crate) producer_after: u32, + pub(crate) opcode: u32, + pub(crate) sequence: u32, + pub(crate) context: u32, +} + +pub(crate) const COMPUTE_READINESS_BOUNDARIES: + [ComputeReadinessRecordBoundary; COMPUTE_READINESS_RECORDS] = [ + ComputeReadinessRecordBoundary { + index: 0, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 1, + opcode: 0x20, + sequence: 35, + context: 1, + }, + ComputeReadinessRecordBoundary { + index: 1, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER + 1, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 2, + opcode: 0x2e, + sequence: 35, + context: 1, + }, + ComputeReadinessRecordBoundary { + index: 2, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER + 2, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 3, + opcode: 0x2e, + sequence: 36, + context: 1, + }, + ComputeReadinessRecordBoundary { + index: 3, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER + 3, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 4, + opcode: 0x20, + sequence: 37, + context: 2, + }, + ComputeReadinessRecordBoundary { + index: 4, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER + 4, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 5, + opcode: 0x2e, + sequence: 37, + context: 2, + }, + ComputeReadinessRecordBoundary { + index: 5, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER + 5, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 6, + opcode: 0x2e, + sequence: 38, + context: 2, + }, +]; + +pub(crate) const fn compute_readiness_boundary( + index: usize, +) -> Option { + if index < COMPUTE_READINESS_RECORDS { + Some(COMPUTE_READINESS_BOUNDARIES[index]) + } else { + None + } +} + +/// Host progress for the native per-record publication handshake. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeReadinessProgress { + retired_records: u8, + pending_target: Option, +} + +impl ComputeReadinessProgress { + pub(crate) const fn new() -> Self { + Self { + retired_records: 0, + pending_target: None, + } + } + + pub(crate) const fn retired_records(self) -> usize { + self.retired_records as usize + } + + pub(crate) const fn pending_target(self) -> Option { + self.pending_target + } + + pub(crate) const fn next_boundary(self) -> Option { + if self.pending_target.is_some() { + None + } else { + compute_readiness_boundary(self.retired_records as usize) + } + } + + pub(crate) const fn begin_publication( + self, + boundary: ComputeReadinessRecordBoundary, + ) -> Option { + if self.pending_target.is_some() || boundary.index != self.retired_records { + return None; + } + Some(Self { + retired_records: self.retired_records, + pending_target: Some(boundary.producer_after), + }) + } + + pub(crate) const fn retire(self, target: u32) -> Option { + match self.pending_target { + Some(pending) if pending == target => Some(Self { + retired_records: self.retired_records + 1, + pending_target: None, + }), + _ => None, + } + } + + pub(crate) const fn complete(self) -> bool { + self.retired_records as usize == COMPUTE_READINESS_RECORDS + && self.pending_target.is_none() + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ComputeReadinessPublicationStep { + ControlRecord(u8), + InitializeContext2Activation, +} + +pub(crate) const COMPUTE_READINESS_PUBLICATION_ORDER: [ComputeReadinessPublicationStep; 7] = [ + ComputeReadinessPublicationStep::ControlRecord(0), + ComputeReadinessPublicationStep::ControlRecord(1), + ComputeReadinessPublicationStep::ControlRecord(2), + ComputeReadinessPublicationStep::InitializeContext2Activation, + ComputeReadinessPublicationStep::ControlRecord(3), + ComputeReadinessPublicationStep::ControlRecord(4), + ComputeReadinessPublicationStep::ControlRecord(5), +]; + +pub(crate) const CONTROL_SHARED_ADDRESS: u64 = 0xffff_fc20_c083_0000; +pub(crate) const CONTROL_SHARED_INNER_ADDRESS: u64 = 0xffff_fc20_0160_8000; +pub(crate) const CONTROL_SHARED_OBJECT_SIZE: usize = 0x4000; +pub(crate) const CONTROL_SHARED_CURSOR_OFFSET: usize = 0x48; +pub(crate) const CONTROL_SHARED_CURSOR_BEFORE: u32 = 0x88; +pub(crate) const CONTROL_SHARED_INNER_PRESENTED: u64 = 2; +/// Separate compact class-2 object consumed by the first-partial opening. +/// Descriptor tails and later compute control continue to use +/// [`CONTROL_SHARED_ADDRESS`]. +pub(crate) const PARTIAL_OPENING_CONTROL_SHARED_ADDRESS: u64 = 0xffff_fc20_c082_8000; +pub(crate) const PARTIAL_OPENING_CONTROL_SHARED_INNER_ADDRESS: u64 = + 0xffff_fc20_0160_0000; +/// Hardware-proven compact compute cursor. Twenty-one consecutive run qwords +/// occupy slots 0..20 and the next eight-qword boundary is 0x18. +pub(crate) const COMPUTE_PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE: u32 = 0x18; +/// Initial compact FList cursor in qwords. +/// +/// `AGXUSCPrivMemFList::prepareBufferResources` encodes one 1-MiB run for +/// each of the 17 initial pool blocks, rounding to an eight-qword slot after +/// each block. The first grow starts at qword 17 * 8 = 0x88 (byte +0x440), +/// exactly as the complete 25G83 first-render opcode-0x20 record publishes. +pub(crate) const PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE: u32 = 0x88; +pub(crate) const PARTIAL_OPENING_CONTROL_SHARED_INNER_PRESENTED: u64 = 0; +/// Linux owns one USC private-memory FList. HardwareBufferIDManager permits +/// slot zero (native's generic allocator pops 0 first); macOS observed ID 1 +/// only because another live FList already owned slot zero in that session. +pub(crate) const PARTIAL_OPENING_FREELIST_HARDWARE_BUFFER_ID: u32 = 0; +pub(crate) const CONTROL_OPERAND_TABLE_ADDRESS: u64 = 0x70_0020_8000; +pub(crate) const CONTROL_OPERAND_TABLE_SIZE: usize = 0x4000; +pub(crate) const CONTROL_OPERAND_SLOT_OFFSET: u64 = 0x440; +pub(crate) const PARTIAL_OPENING_CONTROL_OPERAND_SLOT_OFFSET: u64 = 0x640; +pub(crate) const PARTIAL_OPENING_CONTROL_REGISTER_COUNT: u32 = 0x18; +pub(crate) const PARTIAL_OPENING_CONTROL_OPERAND_ENTRY_COUNT: usize = 28; +pub(crate) const CONTROL_OPERAND_BUFFER_BASE: u64 = 0x70_0022_0000; +pub(crate) const CONTROL_OPERAND_BUFFER_STRIDE: u64 = 0x10_8000; +pub(crate) const CONTROL_OPERAND_BUFFER_SIZE: u64 = 0x10_0000; +pub(crate) const CONTROL_OPERAND_ENTRY_COUNT: usize = 22; +pub(crate) const CONTROL_OPERAND_ENTRY_STRIDE: usize = 0x40; +pub(crate) const CONTROL_OPERAND_ENTRY_FLAG: u64 = 1 << 60; +pub(crate) const COMPUTE_RUNTIME_BUFFER_ADDRESSES: [u64; 21] = [ + 0x70_0022_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_0032_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_0043_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_0053_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_0064_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_0074_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_0085_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_0095_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_00a6_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_00b6_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_00c7_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_00d7_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_00e8_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_00f8_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_0109_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_0119_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_012a_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_013a_8000 + COMPUTE_FLIST_RELOCATION, + COMPUTE_RUNTIME_ZERO_BUFFER_0_ADDRESS, + COMPUTE_RUNTIME_ZERO_BUFFER_1_ADDRESS, + 0x70_014b_0000 + COMPUTE_FLIST_RELOCATION, +]; + +pub(crate) const fn control_opening_producer(role: InstanceRole) -> u32 { + match role { + InstanceRole::Primary => CONTROL_OPENING_PRIMARY_PRODUCER, + InstanceRole::Secondary => CONTROL_OPENING_SECONDARY_PRODUCER, + } +} + +pub(crate) const fn control_opening_size(role: InstanceRole) -> usize { + match role { + InstanceRole::Primary => CONTROL_OPENING_PRIMARY_RECORDS * CONTROL_RECORD_SIZE, + InstanceRole::Secondary => CONTROL_OPENING_SECONDARY_RECORDS * CONTROL_RECORD_SIZE, + } +} + +fn encode_control_registration( + out: &mut [u8], + slot: usize, + control_class: u32, + sequence: u32, + object: u64, + operand: u64, + slot_offset: u64, + count: u32, + context_word: u32, +) { + let base = slot * CONTROL_RECORD_SIZE; + put_u32(out, base, CONTROL_OPENING_PRIMARY_REGISTER_OPCODE); + put_u32(out, base + 0x04, control_class); + put_u32(out, base + 0x08, 0x3f); + put_u32(out, base + 0x0c, sequence); + put_u64(out, base + 0x14, object); + put_u64(out, base + 0x1c, operand); + put_u64(out, base + 0x24, operand + slot_offset); + put_u32(out, base + 0x2c, count); + put_u32(out, base + 0x30, context_word); + put_u32(out, base + 0x34, 1); +} + +fn encode_control_tick(out: &mut [u8], slot: usize, sequence: u32) { + let base = slot * CONTROL_RECORD_SIZE; + put_u32(out, base, CONTROL_BOOTSTRAP_UMA_RELEASE_OPCODE); + put_u32(out, base + 0x04, sequence); +} + +/// Encode the exact release record produced when an Async FList grow request +/// drops its last HardwareBuffer ID reference. The captured opening uses +/// sequence zero and ID zero; accepting them as operands keeps the encoder +/// tied to the retained host ownership rather than to a raw type-13 tuple. +pub(crate) fn encode_bootstrap_uma_release_record( + sequence: u64, + hardware_buffer_id: u32, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, CONTROL_BOOTSTRAP_UMA_RELEASE_OPCODE); + put_u64(out, 0x04, sequence); + put_u32(out, 0x0c, hardware_buffer_id); + Ok(()) +} + +/// Encode the scheduler-state release record exactly as +/// `AGXArmFirmware::submitReleaseResource(va, a, b, c, d)` writes it into its +/// 64-byte device-control entry: opcode at `+0`, zero at `+4`, byte `d` at +/// `+8`, `a` at `+9`, `b` at `+0xa`, `c` at `+0xb`, and the resource's +/// firmware VA as an unaligned u64 at `+0xc`. `identity` is `[a, b, c, d]`, +/// i.e. bytes `[0]`, `[1]`, `[4]` and `[0x26]` of the live scheduler-state +/// page, as `freeReclaimableResources` loads them at `0xfffffe0008865398`. +pub(crate) fn encode_scheduler_state_release_record( + resource_va: u64, + identity: [u8; 4], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, CONTROL_SCHEDULER_STATE_RELEASE_OPCODE); + out[0x08] = identity[3]; + out[0x09] = identity[0]; + out[0x0a] = identity[1]; + out[0x0b] = identity[2]; + put_u64(out, 0x0c, resource_va); + Ok(()) +} + +/// 64-byte process-exit kill record exactly as `processExitTimeoutHandler` +/// builds it on its stack (record base sp+8): opcode at `+0`, the +/// scheduler-state page VA (queue+0x8b8) as an unaligned u64 at `+4`, and +/// the cookie as an unaligned u64 at `+0xc`; everything else zero. Firmware +/// 0x29 (fw 0x49cc) reads exactly `[x24,#4]` and `[x24,#0xc]`. +pub(crate) fn encode_context_kill_record( + scheduler_state_va: u64, + cookie: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, CONTROL_CONTEXT_KILL_OPCODE); + put_u64(out, 0x04, scheduler_state_va); + put_u64(out, 0x0c, cookie); + Ok(()) +} + +fn encode_control_tick_with_context( + out: &mut [u8], + slot: usize, + sequence: u32, + context_word: u32, +) { + encode_control_tick(out, slot, sequence); + let base = slot * CONTROL_RECORD_SIZE; + put_u32(out, base + 0x0c, context_word); +} + +/// Encode one of the six exact records immediately preceding the first CL. +pub(crate) fn encode_compute_readiness_record( + index: usize, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE || index >= COMPUTE_READINESS_RECORDS { + return Err(InitdataError::BufferSize); + } + out.fill(0); + match index { + 0 => encode_control_registration( + out, + 0, + 1, + 35, + COMPUTE_READINESS_CLASS1_SUPPORT_ADDRESS, + COMPUTE_FLIST_RUN_TABLE_ADDRESS, + 0x440, + 0x20, + 1, + ), + 1 => encode_control_tick_with_context(out, 0, 35, 1), + 2 => encode_control_tick_with_context(out, 0, 36, 1), + 3 => encode_control_registration( + out, + 0, + 3, + 37, + COMPUTE_READINESS_CLASS3_SUPPORT_ADDRESS, + COMPUTE_FLIST_RUN_TABLE_ADDRESS, + 0x5c0, + 0x28, + 2, + ), + 4 => encode_control_tick_with_context(out, 0, 37, 2), + 5 => encode_control_tick_with_context(out, 0, 38, 2), + _ => return Err(InitdataError::BufferSize), + } + Ok(()) +} + +fn encode_compute_readiness_support( + out: &mut [u8], + class: u32, + active: u32, + resource_class: u64, + cursor: u32, + state: u64, + final_kind: u32, +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_READINESS_PAGE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0x00, class); + put_u32(out, 0x08, active); + put_u32(out, 0x10, class); + put_u64(out, 0x14, COMPUTE_FLIST_PAGE_LIST_ADDRESS); + put_u32(out, 0x1c, 0x4_0000); + put_u64(out, 0x20, resource_class << 40); + put_u64(out, 0x28, resource_class << 40); + put_u64(out, 0x30, COMPUTE_FLIST_RUN_TABLE_ADDRESS); + put_u32(out, 0x40, 4); + put_u32(out, 0x48, cursor); + put_u64(out, 0x4c, state); + put_u32(out, 0x54, 0); + put_u32(out, 0x5c, 0); + put_u32(out, 0x60, final_kind); + Ok(()) +} + +pub(crate) fn encode_compute_readiness_class1_support( + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_compute_readiness_support( + out, + 1, + 1, + 0x11, + 0x88, + COMPUTE_READINESS_CLASS1_STATE_ADDRESS, + 2, + ) +} + +pub(crate) fn encode_compute_readiness_class3_support( + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_compute_readiness_support( + out, + 3, + 2, + 0x17, + 0xb8, + COMPUTE_READINESS_CLASS3_STATE_ADDRESS, + 3, + ) +} + +pub(crate) fn encode_compute_readiness_state( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_READINESS_PAGE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, 1); + Ok(()) +} + +pub(crate) fn encode_compute_readiness_activation_record( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u64(out, 0x00, 0x0000_0100_0000_ffff); + put_u64(out, 0x20, 0x0002_0000_0000_0000); + put_u64(out, 0x30, 0x0000_0000_ff00_0000); + Ok(()) +} + +pub(crate) fn encode_compute_readiness_operand_table( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_OPERAND_TABLE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for index in 0..COMPUTE_READINESS_OPERAND_ENTRY_COUNT { + let address = COMPUTE_FLIST_BUFFER_BASE + .checked_add(index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + .ok_or(InitdataError::AddressOutOfRange)?; + put_u64( + out, + index * CONTROL_OPERAND_ENTRY_STRIDE, + address | CONTROL_OPERAND_ENTRY_FLAG, + ); + } + Ok(()) +} + +/// Encode the complete control ring prefix consumed during the compute opening. +pub(crate) fn encode_control_opening( + role: InstanceRole, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != control_opening_size(role) { + return Err(InitdataError::BufferSize); + } + out.fill(0); + + match role { + InstanceRole::Primary => { + for slot in 0..CONTROL_OPENING_PRIMARY_RECORDS { + put_u32( + out, + slot * CONTROL_RECORD_SIZE, + CONTROL_OPENING_OPCODE_PRIMARY, + ); + } + } + InstanceRole::Secondary => { + let initial_records = core::cmp::min(3, CONTROL_OPENING_SECONDARY_RECORDS); + for slot in 0..initial_records { + put_u32( + out, + slot * CONTROL_RECORD_SIZE, + CONTROL_OPENING_OPCODE_SECONDARY, + ); + } + for slot in initial_records..CONTROL_OPENING_SECONDARY_RECORDS { + put_u32( + out, + slot * CONTROL_RECORD_SIZE, + CONTROL_OPENING_SECONDARY_CONTINUE_OPCODE, + ); + } + } + } + Ok(()) +} + +/// [`encode_control_opening`] for an explicit firmware build. J700/J813 are +/// byte-identical to [`encode_control_opening`]. G18G stages exactly one +/// record per role, 0x15 (primary) or 0x2a (secondary), and never a +/// continuation or register opcode: G18G 0x20 is invalid and 0x22 has not +/// been re-derived. +pub(crate) fn encode_control_opening_for_profile( + profile: RootProfile, + role: InstanceRole, + out: &mut [u8], +) -> Result<(), InitdataError> { + if profile != RootProfile::J873G18G26A428 { + return encode_control_opening(role, out); + } + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, profile.control_opening_opcode(role)); + Ok(()) +} + +/// Encode one bare device-control record: `u32 opcode @ +0x00`, +/// `u32 arg @ +0x04`, and 0x38 zero bytes. +/// +/// FW_PROVEN (b1 `0x3a74` drain): a record is 0x40 bytes at +/// `ring + index * 0x40`; the drain loads the opcode with `ldr w?, [x24]` and +/// the argument with `ldr w8, [x24, #4]`. Every opcode that carries no further +/// operands leaves the remaining 0x38 bytes untouched, exactly as the opening +/// records this module already stages do. +pub(crate) fn encode_device_control_record( + opcode: u32, + arg: u32, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0x00, opcode); + put_u32(out, 0x04, arg); + Ok(()) +} + +/// Encode one record in the output-positive 65-record runtime control tail. +/// Record zero follows the clean three-record QoS opening. +pub(crate) fn encode_compute_runtime_control_record( + index: usize, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE || index >= COMPUTE_RUNTIME_CONTROL_RECORDS { + return Err(InitdataError::BufferSize); + } + out.fill(0); + + match index { + 0..=53 => encode_control_tick(out, 0, index as u32), + COMPUTE_RUNTIME_CLASS1_RECORD_INDEX => encode_control_registration( + out, + 0, + 1, + 54, + COMPUTE_RUNTIME_SUPPORT_ADDRESS, + COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS, + 0x480, + 0x18, + 0, + ), + 55..=56 => encode_control_tick(out, 0, index as u32 - 1), + COMPUTE_RUNTIME_CLASS2_RECORD_INDEX => encode_control_registration( + out, + 0, + 2, + 56, + COMPUTE_RUNTIME_SUPPORT_ADDRESS, + COMPUTE_FLIST_RUN_TABLE_ADDRESS, + 0x5c0, + 0x28, + 0, + ), + 58..=64 => encode_control_tick(out, 0, index as u32 - 2), + _ => return Err(InitdataError::BufferSize), + } + Ok(()) +} + +fn encode_compute_runtime_support( + out: &mut [u8], + control_class: u32, + low_buffer: u64, + operand_table: u64, + resource_class: u64, + cursor: u32, + final_kind: u32, +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_RUNTIME_SUPPORT_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0x00, control_class); + put_u32(out, 0x10, control_class); + put_u64(out, 0x14, low_buffer); + put_u32(out, 0x1c, 0x4_0000); + let resource = resource_class << 40; + put_u64(out, 0x20, resource); + put_u64(out, 0x28, resource); + put_u64(out, 0x30, operand_table); + put_u32(out, 0x40, 4); + put_u32(out, 0x48, cursor); + put_u64(out, 0x4c, COMPUTE_RUNTIME_STATE_ADDRESS); + put_u32(out, 0x60, final_kind); + Ok(()) +} + +pub(crate) fn encode_compute_runtime_class1_support( + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_compute_runtime_support( + out, + 1, + COMPUTE_RUNTIME_PAGE_LIST_ADDRESS, + COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS, + 0x12, + 0x90, + 2, + ) +} + +pub(crate) fn encode_compute_runtime_class2_support( + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_compute_runtime_support(out, 2, COMPUTE_FLIST_PAGE_LIST_ADDRESS, COMPUTE_FLIST_RUN_TABLE_ADDRESS, 0x17, 0xb8, 3) +} + +pub(crate) fn encode_compute_runtime_class1_table( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_RUNTIME_CLASS1_TABLE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for (index, address) in COMPUTE_RUNTIME_BUFFER_ADDRESSES.into_iter().enumerate() { + put_u64( + out, + index * CONTROL_OPERAND_ENTRY_STRIDE, + address | CONTROL_OPERAND_ENTRY_FLAG, + ); + } + Ok(()) +} + +pub(crate) fn encode_compute_runtime_page_inventory( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_RUNTIME_PAGE_LIST_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + let mut index = 0usize; + for base in COMPUTE_RUNTIME_BUFFER_ADDRESSES { + for offset in (0..CONTROL_OPERAND_BUFFER_SIZE).step_by(0x1000) { + put_u64(out, index * 8, base + offset); + index += 1; + } + } + if index * 8 != COMPUTE_RUNTIME_PAGE_LIST_SIZE { + return Err(InitdataError::BufferSize); + } + Ok(()) +} + +/// Populate the 21-block pool advertised by the compute UMA descriptor. +/// Like render's initial page list, this contains 4-KiB addresses, not compact +/// run descriptors. Publishing nonzero UMA counters over a zero-filled list +/// makes the allocator return address zero on its first consumption. +pub(crate) fn encode_compute_flist_page_list(out: &mut [u8]) -> Result<(), InitdataError> { + let pages_per_buffer = (CONTROL_OPERAND_BUFFER_SIZE / 0x1000) as usize; + let count = COMPUTE_READINESS_OPERAND_ENTRY_COUNT * pages_per_buffer; + if out.len() < count * 8 { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for buffer in 0..COMPUTE_READINESS_OPERAND_ENTRY_COUNT { + let base = COMPUTE_FLIST_BUFFER_BASE + buffer as u64 * CONTROL_OPERAND_BUFFER_STRIDE; + for page in 0..pages_per_buffer { + put_u64(out, (buffer * pages_per_buffer + page) * 8, base + page as u64 * 0x1000); + } + } + Ok(()) +} + +pub(crate) fn encode_compute_dispatch_record( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_DISPATCH_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + // halLoadUMADesc packs the page-list VA in units of 128 bytes below + // unit_pages << 41. The native literal names render's unrelocated pool; + // slot 1 must instead name the independently backed compute page list. + put_u64(out, 0x00, (0x4_0000u64 << 41) | (COMPUTE_FLIST_PAGE_LIST_ADDRESS >> 7)); + put_u64(out, 0x08, 0x1500u64 << 33); + put_u64(out, 0x10, 0x1500); + Ok(()) +} + +/// Encode the generic shared object named by descriptor tails and later +/// compute-control records. +pub(crate) fn encode_control_shared(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != CONTROL_SHARED_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for (offset, value) in [ + (0x00, 0x0000_0000_0000_0001), + (0x10, 0x0000_0000_0000_0001), + (0x18, 0x0004_0000_0000_0070), + (0x20, 0x0000_1100_0000_0000), + (0x28, 0x0000_1100_0000_0000), + (0x30, CONTROL_OPERAND_TABLE_ADDRESS), + (0x40, 0x0000_0000_0000_0004), + (0x60, 0x0000_0000_0000_0002), + ] { + put_u64(out, offset, value); + } + put_u32(out, CONTROL_SHARED_CURSOR_OFFSET, CONTROL_SHARED_CURSOR_BEFORE); + put_u64(out, 0x4c, CONTROL_SHARED_INNER_ADDRESS); + Ok(()) +} + +/// Encode the distinct first-partial support object before opcode 0x20. +pub(crate) fn encode_partial_opening_control_shared( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_SHARED_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + // AGXUSCPrivMemFList::populateInitialFirmwareState seeds this hardware + // buffer id with -1, but AGXUSCPrivMemPool::prepareLocked replaces it with + // the allocated id before publishing the first tag-15 record. Linux owns + // the cold-opening slot zero statically, so encode the post-allocation + // state that the firmware actually receives. + put_u32( + out, + 0x08, + PARTIAL_OPENING_FREELIST_HARDWARE_BUFFER_ID, + ); + for (offset, value) in [ + (0x00, 0x0000_0000_0000_0001), + (0x10, 0x0000_0000_0000_0002), + (0x18, 0x0004_0000_0000_0070), + // The complete first-render trace has 17 initial one-MiB blocks: + // 17 MiB / 4 KiB = 0x1100 pages. Opcode 0x20 appends five blocks. + (0x20, 0x0000_1100_0000_0000), + (0x28, 0x0000_1100_0000_0000), + (0x30, CONTROL_OPERAND_TABLE_ADDRESS), + (0x40, 0x0000_0000_0000_0004), + (0x60, 0x0000_0000_0000_0003), + ] { + put_u64(out, offset, value); + } + put_u32( + out, + CONTROL_SHARED_CURSOR_OFFSET, + PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE, + ); + put_u64( + out, + 0x4c, + PARTIAL_OPENING_CONTROL_SHARED_INNER_ADDRESS, + ); + Ok(()) +} + +/// Encode the live render FList state at the native c0830000 owner address. +/// +/// The page-pool fields are the same prepared 17-block plus first-grow state +/// used by the first-partial encoder, but +0 is a pointer to the distinct +/// c01608000 inner object. Valid cold opcode-0x20, both render tag-15 records, +/// and descriptor +0x21ce all name this one object; c0828000 remains the +/// separate compact compute/first-partial state. +pub(crate) fn encode_render_control_shared(out: &mut [u8]) -> Result<(), InitdataError> { + encode_partial_opening_control_shared(out)?; + put_u64(out, 0x00, CONTROL_SHARED_INNER_ADDRESS); + put_u64(out, 0x4c, CONTROL_SHARED_INNER_ADDRESS); + Ok(()) +} + +/// Encode a per-pair render FList whose whole pool is populated up front. +/// +/// Same object as `encode_render_control_shared`, but the page counts at +/// +0x24/+0x2c and the run cursor at +0x48 cover every block, as +/// AGXUSCPrivMemFList::populateInitialFirmwareState writes them for Apple's +/// 3D pool (0x1600 pages, cursor 0xb0 for 22 blocks). Nothing is left for a +/// first opcode-0x20 grow to append. +pub(crate) fn encode_presized_render_control_shared( + out: &mut [u8], + blocks: u32, +) -> Result<(), InitdataError> { + let pages = blocks.checked_mul(0x100).ok_or(InitdataError::BufferSize)?; + encode_render_control_shared(out)?; + put_u64(out, 0x20, u64::from(pages) << 32); + put_u64(out, 0x28, u64::from(pages) << 32); + put_u32(out, CONTROL_SHARED_CURSOR_OFFSET, blocks * 8); + Ok(()) +} + +/// Encode the hardware-proven compute form of the compact class-2 object. +/// +/// This is deliberately separate from the render UMA form above. Compute's +/// successful opening uses 21 compact consecutive run descriptors, a cursor of +/// 0x18 and the original unallocated HardwareBufferID sentinel. Substituting +/// render's 22-block plus asynchronous-grow state makes compute retire without +/// executing. +pub(crate) fn encode_compute_partial_opening_control_shared( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_SHARED_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0x08, u32::MAX); + for (offset, value) in [ + (0x00, 0x0000_0000_0000_0001), + (0x10, 0x0000_0000_0000_0002), + (0x18, 0x0004_0000_0000_0070), + (0x20, 0x0000_1900_0000_0000), + (0x28, 0x0000_1900_0000_0000), + (0x30, COMPUTE_FLIST_RUN_TABLE_ADDRESS), + (0x40, 0x0000_0000_0000_0004), + (0x60, 0x0000_0000_0000_0003), + ] { + put_u64(out, offset, value); + } + put_u64(out, 0x14, COMPUTE_FLIST_PAGE_LIST_ADDRESS); + put_u32( + out, + CONTROL_SHARED_CURSOR_OFFSET, + COMPUTE_PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE, + ); + put_u64( + out, + 0x4c, + PARTIAL_OPENING_CONTROL_SHARED_INNER_ADDRESS, + ); + Ok(()) +} + +/// Keep the first-partial operand page blank through its opening and first +/// control-done notification. The host publishes the 28-entry form before +/// making either first-work producer visible. +pub(crate) fn encode_partial_opening_operand_table_pre_control( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_OPERAND_TABLE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + Ok(()) +} + +pub(crate) fn encode_partial_opening_operand_table_post_control( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_OPERAND_TABLE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for index in 0..PARTIAL_OPENING_CONTROL_OPERAND_ENTRY_COUNT { + let address = CONTROL_OPERAND_BUFFER_BASE + .checked_add(index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + .ok_or(InitdataError::AddressOutOfRange)?; + put_u64( + out, + index * CONTROL_OPERAND_ENTRY_STRIDE, + address | CONTROL_OPERAND_ENTRY_FLAG, + ); + } + Ok(()) +} + +/// Encode the 22 flagged buffer pointers used by the later compute tail. +pub(crate) fn encode_control_operand_table(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != CONTROL_OPERAND_TABLE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for index in 0..CONTROL_OPERAND_ENTRY_COUNT { + let address = COMPUTE_FLIST_BUFFER_BASE + .checked_add(index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + .ok_or(InitdataError::AddressOutOfRange)?; + put_u64( + out, + index * CONTROL_OPERAND_ENTRY_STRIDE, + address | CONTROL_OPERAND_ENTRY_FLAG, + ); + } + Ok(()) +} + +/// Bundle offsets of the five views named at main `+0x254`. They are views of +/// the common bundle, not independent objects: view 0 overlaps the tail of +/// the hardware-data object, and views 3 and 4 overlap each other by all but +/// 0x80 bytes. +pub(crate) const BUNDLE_VIEW_OFFSETS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; +/// Logical extents of the five views. +pub(crate) const BUNDLE_VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; + +/// The context-0 mappings of the PB descriptor and UMA page-pool descriptor +/// tables published at main `+0x2d8` and `+0x2e8`, respectively. The matching +/// firmware-high mappings are published at `+0x2e0` and `+0x2f0`. +pub(crate) const REGION_VIEW_LOW_ADDRS: [u64; 2] = [0x70_0183_8000, 0x70_0184_0000]; + +// --- Channel table --------------------------------------------------------------- + +pub(crate) const CHANNEL_TABLE_ENTRIES: usize = 17; +pub(crate) const CHANNEL_ENTRY_SIZE: usize = 0x20; +/// Channel entry layout: three *separate* 32-bit state-counter addresses at +/// `+0x00`/`+0x08`/`+0x10` (the counters themselves sit at 0x10 spacing in +/// their state block — do not collapse them into one state object), then the +/// ring base at `+0x18`. +pub(crate) const CHANNEL_ENTRY_STATE_COUNT: usize = 3; +pub(crate) const CHANNEL_STATE_SPACING: usize = 0x10; +pub(crate) const CHANNEL_STATE_STRIDE: usize = 0x40; +/// For a work channel, state 0 is the consumer and state 2 the producer. +pub(crate) const CHANNEL_STATE_CONSUMER: usize = 0; +pub(crate) const CHANNEL_STATE_PRODUCER: usize = 2; +/// Entries 0..11 are ordered TA_0, 3D_0, CL_0, TA_1, ... CL_3. +pub(crate) const WORK_CHANNEL_COUNT: usize = 12; +/// Entry 12 is device control. It continues the compact control-state grid +/// ([`WORK_STATE_GRID_OFFSETS`] has 13 entries, one more than there are work +/// channels), and its ring is embedded at main `+0x4c0`. The M5-derived model +/// refused this index, claiming a "cursor pair at +0x1a0/+0x1a8"; that was a +/// misread — those offsets are exactly this entry's state-counter addresses 0 +/// and 1. +pub(crate) const CONTROL_CHANNEL_INDEX: usize = 12; +/// Entries 13 and 14 are report channels with split state objects: their +/// state addresses alias off status A rather than the compact grid, and the +/// firmware peer of each host-owned counter is at `+0x20` within its object. +pub(crate) const REPORT_CHANNEL_INDICES: [usize; 2] = [13, 14]; +/// Entry 15 is partial: only its first state address, no ring. Entry 16 is +/// empty. Filling them like the rest declares channels that do not exist. +pub(crate) const PARTIAL_CHANNEL_INDEX: usize = 15; + +/// Offsets of the channel state blocks (channels 0..=12) inside the compact +/// state grid. Native state is *not* in channel-table order: within each +/// workload group the work queues sit at descending offsets, and the control +/// channel's block follows at `+0x300`. Preserving these address identities is +/// part of the descriptor ABI. +pub(crate) const WORK_STATE_GRID_OFFSETS: [usize; 13] = [ + 0x0c0, 0x1c0, 0x2c0, // TA_0, 3D_0, CL_0 + 0x080, 0x180, 0x280, // TA_1, 3D_1, CL_1 + 0x040, 0x140, 0x240, // TA_2, 3D_2, CL_2 + 0x000, 0x100, 0x200, // TA_3, 3D_3, CL_3 + 0x300, // device control +]; +/// The primary root's status-B pointer is the word immediately after that +/// compact grid: `status_b == state_grid + 0x340` in the native layout. +pub(crate) const PRIMARY_STATUS_B_STATE_GRID_OFFSET: usize = 0x340; +/// The primary status-B object runs from there to status A: `0xeb00` bytes. +pub(crate) const PRIMARY_STATUS_B_OBJECT_SIZE: usize = 0xeb00; + +/// State-counter addresses of report channels 13/14, as offsets from status A. +pub(crate) const REPORT_STATE_STATUS_A_OFFSETS: [[usize; 3]; 2] = + [[0x00040, 0x002c0, 0x00080], [0x00240, 0xa6ac0, 0x00280]]; +/// Ring bases of report channels 13/14, as offsets from status A. +pub(crate) const REPORT_RING_STATUS_A_OFFSETS: [usize; 2] = [0x04ac0, 0xafac0]; +/// Entry 15's sole state address, as an offset from status A. +pub(crate) const PARTIAL_STATE_STATUS_A_OFFSET: usize = 0x2d2c0; + +/// Native bundle offsets of the twelve work rings (0x1800 each), in +/// channel-table order. These extend past [`HW_DATA_BUNDLE_MIN_ALLOC`]; a host +/// using this placement sizes the cluster [`NATIVE_SHARED_CLUSTER_SIZE`]. +pub(crate) const WORK_RING_BUNDLE_OFFSETS: [usize; 12] = [ + 0x10dc0, 0x16dc0, 0x1cdc0, 0x0f5c0, 0x155c0, 0x1b5c0, 0x0ddc0, 0x13dc0, 0x19dc0, 0x0c5c0, + 0x125c0, 0x185c0, +]; +/// Native bundle offsets of the two main configuration objects. +pub(crate) const NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET: usize = 0x1e5c0; +pub(crate) const NATIVE_SECONDARY_MAIN_BUNDLE_OFFSET: usize = 0x22a80; + +// --- Hardware-data object --------------------------------------------------------- + +/// Register-mapping array: 53 slots of 0x28 bytes at `+0x640`. Slots may also +/// carry only the flag at `+0x20` (declared but unmapped). Register windows +/// come from the ADT; do not replace ADT-defined sizes or addresses. +/// The five qwords `AGXFirmware::initFirmwareData` writes at the head of the +/// hardware-data object: `{range-3 base, range-3 size, GART window base, GART +/// window base, reserved top-of-range-5 page}`. +pub(crate) const HW_FIRMWARE_RANGE_HEAD: &[(usize, u64)] = &[ + (0x00, 0x0000_006f_0000_0000), + (0x08, 0x0000_0000_ffc0_0000), + (0x10, 0x0000_0010_0000_0000), + (0x18, 0x0000_0010_0000_0000), + (0x20, 0x0000_02ff_ffff_8000), +]; + +pub(crate) const HW_REGISTER_MAP_OFFSET: usize = 0x640; +pub(crate) const HW_REGISTER_MAP_ENTRY_SIZE: usize = 0x28; +pub(crate) const HW_REGISTER_MAP_SLOTS: usize = 53; +/// FW device-VA aperture the register windows live in. +pub(crate) const HW_REGISTER_APERTURE: Range = 0xffff_fc21_8000_0000..0xffff_fc21_9000_0000; + +/// `+0xe90`: u32 chip identifier. +pub(crate) const HW_CHIP_ID_OFFSET: usize = 0xe90; +pub(crate) const HW_CHIP_ID_T8140: u32 = 0x8140; + +// The acceptance block at +0x258e. Bisecting by zeroing showed every *other* +// unexplained hardware-data byte can be zero, so this is the part that has to +// be right for the descriptor to be accepted. It decodes as the work-channel +// count, two {0,1,1,1} flag groups, one 0xffffffff "unassigned" word per work +// channel, and three trailing ones. +pub(crate) const HW_ACCEPT_CHANNEL_COUNT: usize = 0x258e; // u16 = 12 +pub(crate) const HW_ACCEPT_FLAG_GROUPS: [usize; 2] = [0x2590, 0x25a0]; +pub(crate) const HW_ACCEPT_FLAG_PATTERN: [u32; 4] = [0, 1, 1, 1]; +pub(crate) const HW_ACCEPT_UNASSIGNED_TABLE: usize = 0x25b4; // 12 x u32 0xffffffff +pub(crate) const HW_ACCEPT_TRAILING_ONES: [usize; 3] = [0x25f4, 0x2600, 0x2608]; +/// `+0x2630` is firmware state, not host configuration (the one byte that +/// differed between a pre-start and a running capture). It must start zero. +pub(crate) const HW_FIRMWARE_STATE_OFFSET: usize = 0x2630; + +/// Four inline G17P KSM completion descriptors at hardware-data `+0x2614`. +/// Only ordinals 0 and 2 are present. Each names low/high mappings of one +/// retained `0x800 * 0x40 == 0x20000` backing; ordinals 1 and 3 stay zero. +pub(crate) const G17P_KSM_COMPLETION_TABLE_OFFSET: usize = 0x2614; +pub(crate) const G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE: usize = 0x20; +pub(crate) const G17P_KSM_COMPLETION_PRESENT_ORDINALS: [u32; 2] = [0, 2]; +pub(crate) const G17P_KSM_COMPLETION_CAPACITY: u32 = 0x800; +pub(crate) const G17P_KSM_COMPLETION_ENTRY_SIZE: u32 = 0x40; +pub(crate) const G17P_KSM_COMPLETION_BACKING_SIZE: usize = + G17P_KSM_COMPLETION_CAPACITY as usize * G17P_KSM_COMPLETION_ENTRY_SIZE as usize; +pub(crate) const G17P_KSM_COMPLETION_LOW_VAS: [u64; 2] = + [0x70_03f0_0000, 0x70_03f2_8000]; +pub(crate) const G17P_KSM_COMPLETION_HIGH_BUNDLE_OFFSETS: [usize; 2] = [0x48000, 0x70000]; +const G17P_KSM_COMPLETION_TABLE_PREFIX_OFFSET: usize = 0x2610; +const G17P_KSM_COMPLETION_TABLE_PREFIX: u32 = 0x100; + +/// Two GPU mappings of one present completion-queue backing. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmCompletionAliases { + pub(crate) low: u64, + pub(crate) high: u64, +} + +pub(crate) const M5_25G76_COMPLETION_QUEUE_COUNT: usize = 4; +pub(crate) const M5_25G76_COMPLETION_BACKING_SIZE: usize = 0x800 * 0x40; + +/// M5 publishes all four descriptors at hardware-data +0x2614, with ordinals +/// 0,1,2,3. Each names low/high mappings of its own retained 128-KiB backing. +/// The two native pre-handoff captures agree on every descriptor byte. +/// Construction only: this does not allocate, map, publish, or poll a queue. +pub(crate) fn encode_m5_25g76_completion_table( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_COMPLETION_QUEUE_COUNT * 0x20 { + return Err(InitdataError::BufferSize); + } + let extent = M5_25G76_COMPLETION_BACKING_SIZE as u64; + // Validate every range before mutating the output. Different queues must + // not alias each other in either address space. + for (i, alias) in aliases.iter().enumerate() { + for address in [alias.low, alias.high] { + required_address(address)?; + let end = address.checked_add(extent).ok_or(InitdataError::AddressOutOfRange)?; + required_address(end - 1)?; + } + for prior in &aliases[..i] { + if (alias.low < prior.low + extent && prior.low < alias.low + extent) + || (alias.high < prior.high + extent && prior.high < alias.high + extent) + { + return Err(InitdataError::PairAliasedObject); + } + } + } + out.fill(0); + for (ordinal, alias) in aliases.iter().enumerate() { + let offset = ordinal * 0x20; + put_u64(out, offset, alias.low); + put_u64(out, offset + 8, alias.high); + put_u32(out, offset + 0x10, 0x800); + put_u32(out, offset + 0x14, 0x40); + put_u32(out, offset + 0x18, ordinal as u32); + } + Ok(()) +} + +// Performance-table geometry (values deliberately not encoded; see +// [`PERF_TABLE_VALUES_ENCODED`]). Two groups with identical internal layout, +// one per 11-entry ladder: frequency ladder, then a core-voltage column 0x40 +// on, then a memory-voltage column 0x440 on, stored as 0x40-stride per-state +// blocks. Group 0 repeats each per-state value across 16 words; group 1 +// stores it once. +pub(crate) const HW_PERF_GROUP_BASES: [usize; 2] = [0xfc8, 0x1cdc]; +pub(crate) const HW_PERF_GROUP_REPEATS: [usize; 2] = [16, 1]; +pub(crate) const HW_PERF_LADDER_ENTRIES: usize = 11; +pub(crate) const HW_PERF_VOLTAGE_DELTA: usize = 0x40; +pub(crate) const HW_PERF_MEMORY_VOLTAGE_DELTA: usize = 0x440; +pub(crate) const HW_PERF_STATE_BLOCK_STRIDE: usize = 0x40; +pub(crate) const HW_PERF_FREQ_LADDER_B: usize = 0x1808; +pub(crate) const HW_PERF_SCALE_LADDER_B: usize = 0x1848; +pub(crate) const HW_PERF_RELATIVE_LADDER_A: usize = 0x18c8; +pub(crate) const HW_PERF_RELATIVE_LADDER_B: usize = 0x1908; +pub(crate) const HW_PERF_INDEX_MAP_A: usize = 0x19c8; +pub(crate) const HW_PERF_INDEX_MAP_B: usize = 0x1a08; +const HW_PERF_FREQ_A: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 338, 492, 618, 796, 928, 1056, 1170, 1278, 1338, 1470]; +const HW_PERF_FREQ_B: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 338, 492, 618, 796, 928, 952, 1053, 1152, 1204, 1326]; +const HW_PERF_SCALE_B: [u32; HW_PERF_LADDER_ENTRIES] = [1_065_520_988; 11]; +/// Relative-performance ladder A. Native's live block reads +/// 0f 16 1d 2a 38 4a 56 at +0x18cc (states 1..7) = 15 22 29 42 56 74 86, +/// against our 16 23 30 43 57 74 85 -- off by one nearly everywhere. States +/// 8..10 are not covered by that read and keep our values. +const HW_PERF_RELATIVE_A_VALUES: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 15, 22, 29, 42, 56, 74, 86, 90, 92, 100]; +const HW_PERF_RELATIVE_B_VALUES: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 0, 13, 24, 40, 52, 63, 73, 83, 88, 100]; +const HW_PERF_INDEX_A_VALUES: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 1, 2, 3, 4, 5, 7, 9, 11, 13, 15]; +const HW_PERF_INDEX_B_VALUES: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 14]; +/// Per-perf-state GPU core voltage, in mV. +/// +/// MEASURED against native's live hardware-data block (25G83, hwdata.d reading +/// *(AGXArmFirmware+0x2b8)): native carries +/// 605 630 650 695 745 800 845 885 905 975 +/// for states 1..10, and this driver derived +/// 595 615 635 685 730 780 815 865 885 955 +/// -- systematically LOW at every state, by 10 to 30 mV. Both machines run the +/// same ADT, so this is our derivation and not chip binning. +/// +/// This is not cosmetic. The register this tree calls "GPC core power", +/// sgx+0xe01000 & 0xf, is G16::AGXFenderCommon::getGPCPerformanceState -- the +/// GPU PERFORMANCE state, where 0 means off. The firmware selects a state from +/// exactly these tables and raises the GPC to it, so a table it judges invalid +/// or below its floor is a direct mechanism for "firmware runs, tables loaded, +/// GPC stays 0, nothing ever executes". +const HW_PERF_CORE_VOLTAGE: [u32; HW_PERF_LADDER_ENTRIES] = + [125, 605, 630, 650, 695, 745, 800, 845, 885, 905, 975]; +/// Per-perf-state SRAM/memory-rail voltage, in mV. +/// +/// Same defect as the core table: our tail tracked our own (wrong) core values. +/// Native's live block carries 765 x6 then 800 845 885 905 975, which is the +/// ADT perf-states-sram column mapped through HW_PERF_INDEX_A. The correct +/// derivation is memory[i] = ADT perf-states-sram[INDEX_A[i]].volt, not "the +/// memory tail tracks core". +const HW_PERF_MEMORY_VOLTAGE: [u32; HW_PERF_LADDER_ENTRIES] = + [765, 765, 765, 765, 765, 765, 800, 845, 885, 905, 975]; + +// --- Region C --------------------------------------------------------------------- + +/// Byte immediately preceding the unaligned host power-assert counter. +pub(crate) const REGION_C_REQUIRED_OFFSET: usize = 0xe50; +pub(crate) const REGION_C_REQUIRED_VALUE: u32 = 0; +/// Unaligned host-owned GPU-core power-assert counter. +/// +/// FW_PROVEN (b1 `0x3aa8`/`0x3b20`): the firmware reads this **unaligned** u32 +/// and compares it with its own tally `status_b[0xe438]`, which it increments +/// once per drained record whose opcode flag byte (`__const 0x45430 + opcode`) +/// is non-zero. When the GPU is unpowered and the two disagree, `0x3b2c` calls +/// `0x21960` and powers the cores on before draining. Equal values mean "no +/// unaccounted power-asserting work", so a value pinned forever at 1 — the old +/// static initializer — can never trigger that arm after the first flagged +/// record. +/// +/// Was `0xe55`, which is off by four: that offset is read nowhere in b1. +/// The flag byte is `0` (i.e. no power assert) for opcodes `0x00`, `0x0e`, +/// `0x15`, `0x16`, `0x1b`, `0x22`, `0x2a`, `0x2e` and `0x3a..=0x3f`; note that +/// includes `0x2e`, contradicting the older comment here that named it as a +/// core-power claimant. +pub(crate) const REGION_C_GPU_CORE_PWR_ASSERT_COUNTER_OFFSET: usize = 0xe51; +/// Native cold value before the host publishes its first flag-1 control. +pub(crate) const REGION_C_GPU_CORE_PWR_ASSERT_COUNT: u32 = 0; + +/// Whether a device-control opcode advances the host GPU-core power-assert +/// counter. The 64-entry table is exhaustive in B1; values outside it are not +/// valid device-control opcodes. +pub(crate) const fn device_control_asserts_gpu_power(opcode: u32) -> Option { + if opcode > 0x3f { + return None; + } + Some(!matches!( + opcode, + 0x00 | 0x0e | 0x15 | 0x16 | 0x1b | 0x22 | 0x2a | 0x2e | 0x3a..=0x3f + )) +} + +// --- Status blocks ----------------------------------------------------------------- + +/// Status block `+0x04` holds u32 1 pre-init. `+0x10`/`+0x14` are *post-ack +/// firmware state* and must be zero at handoff. +pub(crate) const STATUS_BLOCK_ONE_OFFSET: usize = 0x04; +pub(crate) const STATUS_BLOCK_POST_ACK_OFFSETS: [usize; 2] = [0x10, 0x14]; +pub(crate) const PRIMARY_STATUS_B_FWCTL_STATE: usize = 0x48e0; +pub(crate) const PRIMARY_STATUS_B_FWCTL_RING: usize = 0x48e8; +pub(crate) const PRIMARY_STATUS_B_CONFIG_HEADER: usize = 0xe434; +pub(crate) const PRIMARY_STATUS_B_CONFIG_OFFSET: usize = 0xe440; + +// --- Instance separation ----------------------------------------------------------- + +/// The secondary root sits exactly this far above the primary root; the +/// secondary's initialisation message carries the primary's address plus this +/// delta in the same field, so the pair is not placed freely. +pub(crate) const SECONDARY_ROOT_DELTA: u64 = 0x8000; +/// Each instance maps the ADT shared region from its own half: the secondary +/// maps it from base plus this delta, and walks a top table at the same +/// offset into its half. Mappings installed only in the primary's tables +/// leave the secondary unable to translate anything it is handed. +pub(crate) const SECONDARY_SHARED_REGION_DELTA: u64 = 0x40000; + +/// Physical firmware instance encoded at root `+0x28`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum InstanceRole { + /// `gfx-asc`: scheduler, device control, and all work channels. + Primary = 0, + /// `gfx1-asc`: control-only peer. Linux starts it after primary RTKit is ready. + Secondary = 1, +} + +pub(crate) const fn root_size(role: InstanceRole) -> usize { + match role { + InstanceRole::Primary => ROOT_SIZE_PRIMARY, + InstanceRole::Secondary => ROOT_SIZE_SECONDARY, + } +} + +/// GPU virtual addresses of the objects both instances share. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedAddresses { + /// Shared 16 KiB region A (root `+0x08`). + pub(crate) region_a: u64, + /// Shared region C (root `+0x20`), 0x1000 bytes. + pub(crate) region_c: u64, + /// Base of the contiguous hardware-data bundle. The hardware-data object + /// sits at the base; the five main-config views, the repeated region, and + /// the secondary's `+0x471` pointer are all derived offsets into it. + pub(crate) hw_data_bundle: u64, + /// Size of that one allocation; must be at least + /// [`HW_DATA_BUNDLE_MIN_ALLOC`]. + pub(crate) hw_data_bundle_alloc: usize, +} + +/// GPU virtual addresses of one instance's private objects. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct InstanceAddresses { + /// This instance's root; its address rides the endpoint-0x20 doorbell. + pub(crate) root: u64, + pub(crate) main_config: u64, + pub(crate) status_a: u64, + /// Required for the primary; must be zero for the secondary (whose root + /// still has the field, written zero). + pub(crate) status_b: u64, + /// Root `+0xb8`/`+0xc0`. Required for the secondary; the primary root is + /// only 0xb8 bytes and cannot even hold them, so they must be zero there. + pub(crate) secondary_extras: [u64; 2], +} + +/// One channel-table entry: three state-counter addresses, then the ring base. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ChannelEntry { + pub(crate) states: [u64; 3], + pub(crate) ring: u64, +} + +impl ChannelEntry { + pub(crate) const EMPTY: ChannelEntry = ChannelEntry { + states: [0; 3], + ring: 0, + }; +} + +/// The primary's global region views at main `+0x2d0`: a globally retained, +/// initially zero HWDSID accounting table (256 pairs of u32), followed by +/// the high halves of two low/high alias pairs (the low halves +/// are the fixed context-0 addresses in +/// [`REGION_VIEW_LOW_ADDRS`]). Each pair must map one physical page from both +/// address spaces — an invariant of the UAT mapping layer that cannot be +/// checked from the addresses alone. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrimaryRegionViews { + pub(crate) sentinel_high: u64, + pub(crate) alias_high: [u64; 2], +} + +/// One register-window declaration for the hardware-data mapping array. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWindow { + pub(crate) slot: usize, + pub(crate) phys: u64, + pub(crate) device_va: u64, + /// Byte count; written at both `+0x10` and `+0x14`. Not always a granule + /// multiple. + pub(crate) size: u32, + /// Low 24 bits of the physical address for accelerator-internal windows, + /// else zero. + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +/// M5 25G76 descriptor fields +0x10 and +0x14 are independent. In slot 39 +/// native publishes 0x10000 and 0x8000 respectively. The older RegisterWindow +/// encoder duplicates one size and must not serialize this target's table. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RegisterWindow { + pub(crate) slot: usize, + pub(crate) phys: u64, + pub(crate) device_va: u64, + pub(crate) size_10: u32, + pub(crate) size_14: u32, + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +impl M5RegisterWindow { + /// Actual M5 PTE walks cover size_10 rounded to 16 KiB, including the + /// four pages of slot 39. size_14 is independently serialized firmware + /// metadata and must not truncate that mapping to two pages. + pub(crate) fn mapped_pages(&self) -> Result<(u64, u64, u64), InitdataError> { + let mask = UAT_PAGE_SIZE as u64 - 1; + if self.size_10 == 0 || self.phys & mask != self.device_va & mask { + return Err(InitdataError::RegisterWindow); + } + let physical = self.phys & !mask; + let virtual_address = self.device_va & !mask; + let end = self.device_va.checked_add(self.size_10 as u64) + .and_then(|value| value.checked_add(mask)) + .ok_or(InitdataError::AddressOutOfRange)? & !mask; + let span = end.checked_sub(virtual_address).ok_or(InitdataError::AddressOutOfRange)?; + physical.checked_add(span).ok_or(InitdataError::AddressOutOfRange)?; + Ok((physical, virtual_address, span)) + } +} + +/// Source-constructed register descriptors from both identical pre-handoff +/// J813/25G76 captures. These are firmware virtual mappings, not a request to +/// access or power the devices. No runtime currently consumes this profile. +pub(crate) const M5_25G76_REGISTER_WINDOWS: [M5RegisterWindow; 22] = { + const fn w(slot: usize, phys: u64, va: u64, size_10: u32, size_14: u32, + unk_18: u64, flag: u32) -> M5RegisterWindow { + M5RegisterWindow { slot, phys, device_va: 0xffff_fc21_8000_0000 + va, + size_10, size_14, unk_18, flag } + } + [ + w(0, 0x381014000, 0x000000, 0x04000, 0x04000, 0, 2), + w(3, 0x220104000, 0x008000, 0x18000, 0x18000, 0, 2), + w(8, 0x3a1350000, 0x028000, 0x04000, 0x04000, 0, 2), + w(9, 0x3803d0000, 0x030000, 0x04000, 0x04000, 0, 2), + w(10, 0x3803c0000, 0x038000, 0x04000, 0x04000, 0, 0), + w(12, 0x42165c000, 0x040000, 0x04000, 0x04000, 0, 2), + w(14, 0x380280000, 0x048000, 0x08000, 0x08000, 0, 0), + w(15, 0x38c840000, 0x058000, 0x24000, 0x24000, 0, 2), + w(17, 0x580000000, 0x080000, 0x21400, 0x21400, 0, 2), + w(22, 0x581000000, 0x0a8000, 0x08000, 0x08000, 0, 2), + w(24, 0x580ec0000, 0x0b8000, 0x10000, 0x10000, 0, 2), + w(26, 0x580d04000, 0x0d0000, 0x08000, 0x08000, 0xd04000, 2), + w(27, 0x580d0d000, 0x0e1000, 0x01000, 0x01000, 0xd0d000, 2), + w(28, 0x580d50000, 0x0e8000, 0x10000, 0x10000, 0xd50000, 2), + w(29, 0x580d10000, 0x100000, 0x04000, 0x04000, 0xd10000, 2), + w(31, 0x580d40000, 0x108000, 0x04000, 0x04000, 0xd40000, 2), + w(32, 0x580d60000, 0x110000, 0x08000, 0x08000, 0xd60000, 2), + w(35, 0x580e00000, 0x120000, 0x04000, 0x04000, 0xe00000, 2), + w(39, 0x580e08000, 0x128000, 0x10000, 0x08000, 0, 2), + w(41, 0x580e33800, 0x143800, 0x04000, 0x04000, 0, 2), + w(42, 0x580e35000, 0x151000, 0x04000, 0x04000, 0, 2), + w(50, 0x580014400, 0x160400, 0x00400, 0x00400, 0x14400, 0), + ] +}; + +pub(crate) fn encode_m5_25g76_register_map(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for window in M5_25G76_REGISTER_WINDOWS { + let base = window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, window.phys); + put_u64(out, base + 8, window.device_va); + put_u32(out, base + 0x10, window.size_10); + put_u32(out, base + 0x14, window.size_14); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + for slot in [2, 5, 6, 7, 30, 33, 34, 37, 38, 40, 43, 46, 47, 48, 49, 52] { + put_u32(out, slot * HW_REGISTER_MAP_ENTRY_SIZE + 0x20, 2); + } + Ok(()) +} + +pub(crate) const M5_25G76_HW_DATA_SIZE: usize = 0x2708; + +/// Native sharedDataDescriptor sizes loaded at AGXFirmware::allocFirmwareData +/// +0x8b8 onwards from 25G76 __const 0xfffffe0007139820. The allocator rounds +/// each next object to 64 bytes. These are independent allocations, not the +/// overlapping views used while investigating T8140. +pub(crate) const M5_25G76_SHARED_ALLOCATIONS: [(usize, usize); 7] = [ + (0x0000, 0x2708), (0x2740, 0x0c18), (0x3380, 0x1048), + (0x4400, 0x0e10), (0x5240, 0x6a30), (0xbc80, 0x0068), + (0xbd00, 0x0800), +]; +pub(crate) const M5_25G76_SHARED_PREFIX_SIZE: usize = 0xc500; +pub(crate) const M5_25G76_AON_PTD_SIZE: usize = 0xe4; + +/// J813/25G76 AON PTD backing, captured twice before each role's handoff and +/// unchanged after a passing Metal workload. The three recorded words are +/// scoped to this platform's telemetry description; they are not generic +/// M5 constants. The allocation and GPU binding must remain caller-owned. +pub(crate) fn encode_m5_25g76_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_25G76_AON_PTD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, 0x6730_5452); + put_u32(out, 4, 0x001d_0001); + put_u32(out, 8, 0x0000_1401); + Ok(()) +} + +/// Native independent primary status/config allocation sizes, from the +/// 25G76 descriptors at 0xfffffe0007139880 / 0xfffffe0007139888. +pub(crate) const M5_25G76_STATUS_CONFIG_SIZE: usize = 0x6a8; +const M5_25G76_STATUS_CONFIG_WORDS: &[(usize, u32)] = &[ + (0x010, 0x0000001f), (0x014, 0x3e0a3d71), (0x018, 0x401ae148), + (0x01c, 0x00000028), (0x020, 0x00000001), (0x0c8, 0x00002af8), + (0x0cc, 0x00001f40), (0x0d8, 0x41980000), (0x0dc, 0x3d0f5c28), + (0x0f0, 0x000026ac), (0x0f4, 0x000000c8), (0x108, 0x42c80000), + (0x10c, 0x43c80000), (0x110, 0x000000c8), (0x128, 0x00000001), + (0x12c, 0x00005eb0), (0x130, 0x3e800000), (0x134, 0x3f99999a), + (0x158, 0x0000000e), (0x1c0, 0x00000001), (0x1c4, 0x00000001), + (0x1c8, 0x00000002), (0x1cc, 0x00000001), (0x1d0, 0x00000001), + (0x1d4, 0x00000001), (0x1d8, 0x00000001), (0x588, 0x00000001), + (0x58c, 0x0000000a), (0x590, 0x00000dac), (0x594, 0x000003e8), + (0x598, 0x00000bb8), (0x59c, 0x000002bc), (0x5cc, 0x00000001), + (0x5d0, 0x00000004), (0x5dc, 0x00001f40), (0x5e0, 0x000000c8), + (0x5e4, 0x00000fa0), (0x5e8, 0x0000012c), (0x5ec, 0x000007d0), + (0x5f0, 0x0000012c), (0x5f4, 0x000003e8), (0x5f8, 0x0000012c), + (0x600, 0x00000001), (0x604, 0x41700000), (0x608, 0x40a00000), + (0x60c, 0x00000020), (0x610, 0x000004b0), (0x620, 0x0000044c), +]; + +/// Build the primary status object before either firmware role is started. +/// Secondary root+0xb8 aliases status+0xe440. GFX1 copies that pointer to +/// firmware global 0x176248 at text+0x211d4 and later writes config+0x670, +/// +0x678 and +0x680 at text+0x1af94..0x1afc4. Those are firmware-owned +/// runtime state, not platform constants, and must start at zero even when +/// a primary pre-handoff capture (taken after secondary startup) has values. +pub(crate) fn encode_m5_25g76_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != PRIMARY_STATUS_B_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, PRIMARY_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, PRIMARY_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, PRIMARY_STATUS_B_CONFIG_HEADER, 1); + for &(offset, value) in M5_25G76_STATUS_CONFIG_WORDS { + put_u32(out, PRIMARY_STATUS_B_CONFIG_OFFSET + offset, value); + } + Ok(()) +} + + +/// Shared private arena observed before primary initialization on J813. +/// The status and secondary-grid pointers name overlapping views of this +/// one backing. This extent is a measured mapped arena, not a claim about +/// the sizes of the native host's individual suballocations. +pub(crate) const M5_25G76_PRIVATE_CLUSTER_SIZE: usize = 0x178000; + +pub(crate) fn encode_m5_25g76_private_cluster( + fwctl_state: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_PRIVATE_CLUSTER_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + let fwctl_ring = fwctl_state.checked_add(0x40).ok_or(InitdataError::AddressOutOfRange)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_m5_25g76_primary_status_b(fwctl_state, fwctl_ring, &mut out[0x340..0xee40])?; + for offset in [0xee40, 0xc3100] { + encode_status_block(&mut out[offset..offset + STATUS_BLOCK_SIZE])?; + } + encode_secondary_root_extra_1_for_profile(RootProfile::J813G17G25G76, + &mut out[0x177400..0x177480])?; + // The main-config encoders carry one measured opening record (0x16 / + // 0x2a). Publish exactly one producer entry for each control channel. + // All work-channel and report counters remain zero. + put_u32(out, 0x320, 1); + put_u32(out, 0x1773e0, 1); + Ok(()) +} + +/// Caller-owned bindings in the power/performance policy allocation. +/// The host stores mach_absolute_time() at policy+0x651c. Supply the live +/// platform counter in its native timebase, never a captured timestamp. +#[derive(Clone, Copy)] +pub(crate) struct M5PolicyBindings { + pub state: u64, + pub aon_ptd: u64, + pub boot_timestamp: u64, +} + +/// Pre-handoff J813/25G76 scalars, identical across two cold boots after +/// removing owned pointers and the native boot timestamp. The repeated +/// 512-entry u16 table is encoded separately. Calibration fields whose +/// meanings remain unknown are scoped to this measured ten-core target. +const M5_25G76_POLICY_WORDS: &[(usize, u32)] = &[ + (0x3d98, 0xffffffff), (0x3db0, 0xffffffff), (0x5244, 0x00030d40), + (0x5248, 0x00030d40), (0x5250, 0x00000004), (0x5254, 0x3f800000), + (0x526c, 0x00000001), (0x5270, 0x00000001), (0x5280, 0x00000064), + (0x5284, 0x00000001), (0x5288, 0x000004b0), (0x5290, 0x00000064), + (0x52a0, 0x00000004), (0x52a4, 0x3f800000), (0x52ac, 0x00000001), + (0x52b0, 0x00000001), (0x52c0, 0x00000064), (0x52c4, 0x00000001), + (0x52c8, 0x000004b0), (0x52d0, 0x00000064), (0x5300, 0x3f828f5c), + (0x5304, 0x3f828f5c), (0x5308, 0x3f828f5c), (0x530c, 0x3f828f5c), + (0x5310, 0x3f828f5c), (0x5314, 0x3f828f5c), (0x5318, 0x3f828f5c), + (0x531c, 0x3f828f5c), (0x5320, 0x3f828f5c), (0x5324, 0x3f828f5c), + (0x5328, 0x3f828f5c), (0x532c, 0x3f828f5c), (0x5330, 0x3f828f5c), + (0x5bf8, 0x00000271), (0x5c04, 0x3f7f2e9f), (0x5c0c, 0x3b516154), + (0x5c14, 0x3ca59586), (0x5c20, 0x46bd6000), (0x5c24, 0x40a90fe4), + (0x5c2c, 0xbd4ac5b4), (0x5c30, 0x00000028), (0x5c34, 0x000004b0), + (0x5c38, 0x000004b0), (0x5c44, 0x00005eb0), (0x5c78, 0x000004b0), + (0x5cb0, 0x00000008), (0x5cbc, 0x3f000000), (0x5cc4, 0x3f000000), + (0x5ccc, 0x3f99999a), (0x5cd8, 0x47800000), (0x5cdc, 0x3e800000), + (0x5ce8, 0x00000028), (0x5cec, 0x000004b0), (0x5cfc, 0x00005eb0), + (0x5d08, 0x00000010), (0x5d10, 0x0005dc00), (0x5d68, 0x0000004a), + (0x5d70, 0x0000005a), (0x5d74, 0x00000032), (0x5d78, 0x00000006), + (0x5d80, 0x00000006), (0x5d84, 0x00000001), (0x5d8c, 0x3f4ccccd), + (0x5d90, 0x3f7eb852), (0x5d94, 0x3e4ccccd), (0x5d98, 0x3ba3d70a), + (0x5d9c, 0x3f144b3c), (0x5da0, 0x3f144b3c), (0x5da8, 0x42be0000), + (0x5dac, 0x4029eb85), (0x5db0, 0x4029eb85), (0x5db4, 0x4139435e), + (0x5db8, 0x00000064), (0x5dbc, 0x000004b0), (0x5dc0, 0x00000064), + (0x5dcc, 0x0000004a), (0x5dd4, 0x00000001), (0x5e00, 0x00000064), + (0x5e20, 0x00000401), (0x5e24, 0x00006400), (0x5e2c, 0x001f4000), + (0x5e30, 0x0000c800), (0x5e34, 0x000fa000), (0x5e38, 0x00012c00), + (0x5e3c, 0x0007d000), (0x5e40, 0x00012c00), (0x5e44, 0x0003e800), + (0x5e48, 0x00012c00), (0x5e4c, 0x00000100), (0x5ea4, 0x00000001), + (0x5ed0, 0x00000001), (0x74d8, 0x47800000), (0x74dc, 0x42200000), + (0x74e0, 0x44960000), (0x74e4, 0x0000062a), (0x74f8, 0x00000028), + (0x7500, 0x42c80000), (0x7504, 0x000004b0), (0x750c, 0x3f4ccccd), + (0x7510, 0x3e4ccccd), (0x7550, 0xdcdcfbfa), (0x7558, 0x00001f40), + (0x7560, 0x00000004), (0x7574, 0x3f800000), (0x757c, 0x3d0f5c28), + (0x7588, 0x47800000), (0x758c, 0x41980000), (0x7598, 0x00000028), + (0x759c, 0x000004b0), (0x75a0, 0x000004b0), (0x75ac, 0x00002af8), + (0x75e0, 0x000004b0), (0x77f8, 0x00000002), (0x77fc, 0x42000006), + (0x7a10, 0xb8520000), (0x7a14, 0x00003f7e), (0x7a18, 0xd70a0000), + (0x7a1c, 0x00003ba3), (0x7a24, 0x000043c8), (0x7a30, 0x00004780), + (0x7a34, 0x000042c8), (0x7a3c, 0x0384bac8), (0x7a40, 0x03e80000), + (0x7a44, 0x03e80000), (0x7a4c, 0x80000000), (0x7a50, 0x26ac453b), + (0x7a84, 0x03e80000), (0x7aa0, 0x00c80000), (0x7b58, 0x000004b0), + (0x7b6c, 0x00000001), (0x7ba0, 0x00000101), (0x7ba4, 0x00000200), + (0x7bbc, 0x00023f00), (0x7bc0, 0x00000100), (0x7bc4, 0x00000100), + (0x7bdc, 0x00024000), (0x7be0, 0x00000100), (0x7be4, 0x00000100), + (0x7bfc, 0x00024100), (0x7c48, 0x00000001), (0x7c50, 0x00005eb0), + (0x7c54, 0x00005eb0), (0x7c58, 0x00005eb0), (0x7c68, 0x3f800000), + (0x7c6c, 0x00000004), (0x7c70, 0x00000008), (0x7c74, 0x0002ee00), + (0x7d44, 0x0000007c), (0x7d50, 0x3f77bdef), (0x7d58, 0x3d042108), + (0x7d60, 0x3e0a3d71), (0x7d6c, 0x47800000), (0x7d70, 0x401ae148), + (0x7d7c, 0x00000028), (0x7d80, 0x000004b0), (0x7d84, 0x000004b0), + (0x7d8c, 0x46bd6000), (0x7d90, 0x00005eb0), (0x7d9c, 0x000000f8), + (0x7da4, 0x005ad200), (0x7dc4, 0x000004b0), (0x7e04, 0x00000001), + (0x7ed4, 0x00000032), (0x7ed8, 0x00000001), (0x7ee0, 0x3f638e39), + (0x7ee4, 0x3f2aaaab), (0x7ee8, 0x3de38e39), (0x7eec, 0x3eaaaaab), + (0x7ef0, 0xbecccccd), (0x7ef4, 0xbf4ccccd), (0x7efc, 0x47800000), + (0x7f00, 0xc0a00000), (0x7f04, 0xc0a00000), (0x7f0c, 0x00000064), + (0x7f10, 0x000004b0), (0x7f14, 0x00000064), (0x7f1c, 0x45fa0000), + (0x7f20, 0x00000578), (0x7f28, 0x0000000e), (0x7f2c, 0x00000048), + (0x7f30, 0x00000018), (0x7f34, 0x001a5e00), (0x7f3c, 0x0008ca00), + (0x7f48, 0x47800000), (0x7f70, 0x00001f40), (0x7f74, 0x00000004), + (0x7f78, 0x000009c4), (0x7f7c, 0x00000258), (0x7f80, 0x00000002), + (0x7f84, 0x00000004), (0x7f88, 0x00000032), (0x7fb8, 0x47800000), + (0x7fc8, 0x00000028), (0x7fcc, 0x000004b0), (0x8098, 0x00000001), + (0x80ac, 0x0000000a), (0x80b8, 0x00000001), (0x8174, 0xac000000), + (0x8178, 0xe800000d), (0x817c, 0xb8000003), (0x8180, 0xb000000b), + (0x8184, 0xb0000004), (0x8188, 0xbc000004), (0x818c, 0x20000002), + (0x8198, 0x00000001), (0x81b4, 0x00000004), (0x81c0, 0x3f780000), + (0x81c8, 0x3d000000), (0x81d0, 0x40a00000), (0x81dc, 0x47800000), + (0x81e0, 0x41700000), (0x81ec, 0x0000044c), (0x81f0, 0x000004b0), + (0x81f4, 0x000004b0), (0x8200, 0x000004b0), (0x8234, 0x000004b0), + (0x827c, 0x00000064), (0x8290, 0xdcdcfbfa), (0x8298, 0x0000007d), + (0x829c, 0x00000001), (0x8ee0, 0xdcdcfbfb), (0x8ee8, 0x00000001), + (0xaeec, 0x41c80000), (0xaef0, 0x41c80000), (0xb6a8, 0x67305452), + (0xb6e8, 0x00000001), (0xb728, 0x5243464c), (0xb72c, 0x4758464c), + (0xb738, 0x47584c50), (0xb748, 0x00000001), (0xb7a8, 0x00000001), + (0xb7b4, 0x42b73333), (0xb7f4, 0x449b8000), (0xb834, 0x42bc0000), + (0xb8e0, 0x0000186a), +]; + +/// Initialize the six allocations following M5 hardware-data. Preserve the +/// hardware block and subsequent channel resources. No firmware is started. +pub(crate) fn encode_m5_25g76_bundle_policy( + bindings: M5PolicyBindings, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() < M5_25G76_SHARED_PREFIX_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(bindings.state)?; + required_address(bindings.aon_ptd)?; + let state_end = bindings.state.checked_add(STATUS_BLOCK_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + let aon_end = bindings.aon_ptd.checked_add(M5_25G76_AON_PTD_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(state_end - 1)?; + required_address(aon_end - 1)?; + if bindings.state < aon_end && bindings.aon_ptd < state_end { + return Err(InitdataError::PairAliasedObject); + } + out[0x2740..M5_25G76_SHARED_PREFIX_SIZE].fill(0); + for &(offset, value) in M5_25G76_POLICY_WORDS { + put_u32(out, offset, value); + } + for index in 0..512 { + put_u16(out, 0x6ed7 + index * 2, 0x7fff); + } + put_u64(out, 0x8ed8, bindings.state); + put_u64(out, 0xb6a0, bindings.aon_ptd); + put_u64(out, 0xb75c, bindings.boot_timestamp); + Ok(()) +} + + +/// Recorded platform words outside the explicit register, performance and +/// owned-pointer fields. Meanings not yet derived are deliberately kept as +/// target-scoped measured values. Never insert runtime completion counters +/// or captured allocation addresses here. Reference hardware-data SHA-256: +/// 6f9ac0c4664f898a0ddb4e5df9ad160a23c1b52d3fd750750779cbdebe446be9. +const M5_25G76_HW_PLATFORM_WORDS: &[(usize, u32)] = &[ + (0x0004, 0x0000006f), (0x0008, 0xffc00000), (0x0014, 0x00000010), + (0x001c, 0x00000010), (0x0020, 0xffff8000), (0x0024, 0x000002ff), + (0x0028, 0x81400000), (0x002c, 0xfffffc21), (0x00e0, 0x00002008), + (0x00e8, 0x20080000), (0x00f4, 0x00002008), (0x00f8, 0x000024cb), + (0x00fc, 0xc98a2cfa), (0x0100, 0xf4f624cb), (0x0104, 0x1877e917), + (0x0108, 0x38d924cb), (0x010c, 0xbdab0000), (0x02d8, 0xfff82000), + (0x02dc, 0xd32d2cdb), (0x02e0, 0xf5002000), (0x02e4, 0x21dae926), + (0x02e8, 0x38b62000), (0x02ec, 0xc7420008), (0x03e0, 0x00007fe0), + (0x03e8, 0x7fe00000), (0x03f4, 0x00007fe0), (0x03f8, 0x00008000), + (0x0400, 0x80000000), (0x040c, 0x00008000), (0x05d8, 0x4b232645), + (0x05dc, 0x00000e98), (0x05e0, 0xd5a2ea5f), (0x05e4, 0x40003fff), + (0x05e8, 0xca5e4000), (0x05ec, 0x4000f5a2), (0x0e90, 0x00008142), + (0x0e94, 0x00000002), (0x0ea0, 0x00000001), (0x0ea4, 0x00000001), + (0x0eb8, 0x00000001), (0x0ec0, 0x00000001), (0x0ec8, 0x00000001), + (0x0ed0, 0x00005dc0), (0x0ed4, 0x00000001), (0x0ed8, 0x00000008), + (0x0ee0, 0x00000001), (0x0ee4, 0x00000001), (0x0ee8, 0x00000001), + (0x0f04, 0x0000001f), (0x0f24, 0x07800004), (0x0f28, 0x00000006), + (0x0f34, 0x00000001), (0x0f38, 0x00000001), (0x0f4c, 0x00000031), + (0x0f6c, 0x00000001), (0x0f88, 0x0000000a), (0x0f8c, 0x00000001), + (0x0fac, 0x00000001), (0x0fb8, 0x00000020), (0x0fbc, 0x00000005), + (0x0fc0, 0x0000000a), (0x0fc4, 0x0000000c), (0x2544, 0x00000004), + (0x2548, 0x00000006), (0x254c, 0x00000003), (0x2550, 0x00000007), + (0x2554, 0x00000007), (0x2568, 0x0000000a), (0x2570, 0x00000001), + (0x258c, 0x00180000), (0x2594, 0x00000001), (0x2598, 0x00000001), + (0x259c, 0x00000001), (0x25a4, 0x00000001), (0x25a8, 0x00000001), + (0x25ac, 0x00000001), (0x25b4, 0xffffffff), (0x25b8, 0xffffffff), + (0x25bc, 0xffffffff), (0x25c0, 0xffffffff), (0x25c4, 0xffffffff), + (0x25c8, 0xffffffff), (0x25cc, 0xffffffff), (0x25d0, 0xffffffff), + (0x25d4, 0xffffffff), (0x25d8, 0xffffffff), (0x25dc, 0xffffffff), + (0x25e0, 0xffffffff), (0x25f4, 0x00000001), (0x2600, 0x00000001), + (0x2608, 0x00000001), (0x2610, 0x00000100), (0x26b4, 0x00000001), + (0x26b8, 0x00000001), (0x26e0, 0x000003e8), (0x26e8, 0x00000001), +]; + +/// Reconstruct the M5 reference hardware-data block with caller-owned queue +/// and QoS bindings. Performance values are measured on the ten-core J813 +/// reference, not derived from ADT and not a generic M5 power configuration. +/// Both pre-handoff firmware roles supplied identical reference bytes. +/// This encoder grants no runtime admission and performs no MMIO accesses. +pub(crate) fn encode_m5_25g76_hw_data( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + qos_high: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(qos_high)?; + let qos_end = qos_high.checked_add(0xe40).ok_or(InitdataError::AddressOutOfRange)?; + required_address(qos_end - 1)?; + let mut completion = [0; 0x80]; + encode_m5_25g76_completion_table(aliases, &mut completion)?; + for alias in aliases { + if qos_high < alias.high + M5_25G76_COMPLETION_BACKING_SIZE as u64 + && alias.high < qos_end { + return Err(InitdataError::PairAliasedObject); + } + } + + out.fill(0); + for &(offset, value) in M5_25G76_HW_PLATFORM_WORDS { + put_u32(out, offset, value); + } + encode_m5_25g76_register_map(&mut out[0x640..0xe88])?; + out[0x2614..0x2694].copy_from_slice(&completion); + put_u64(out, 0x26ac, qos_high); + + // Native includes index zero plus twelve operating points. Each voltage + // occupies all sixteen lanes of its 0x40-byte state row. + const FREQUENCY: [u32; 13] = [0, 338, 486, 636, 796, 888, 988, 1084, 1182, 1278, 1374, 1470, 1578]; + const CORE_VOLTAGE: [u32; 13] = [125, 615, 630, 660, 705, 735, 755, 800, 830, 865, 905, 945, 1010]; + const MEMORY_VOLTAGE: [u32; 13] = [770, 770, 770, 770, 770, 770, 770, 800, 830, 865, 905, 945, 1010]; + const RELATIVE_A: [u32; 13] = [0, 17, 23, 30, 42, 50, 58, 71, 82, 85, 89, 93, 100]; + const RELATIVE_B: [u32; 13] = [0, 0, 11, 24, 36, 44, 52, 60, 68, 75, 83, 91, 100]; + for index in 0..13 { + put_u32(out, 0xfc8 + index * 4, FREQUENCY[index]); + put_u32(out, 0x1848 + index * 4, 0x3f828f5c); + put_u32(out, 0x18c8 + index * 4, RELATIVE_A[index]); + put_u32(out, 0x1908 + index * 4, RELATIVE_B[index]); + for lane in 0..16 { + put_u32(out, 0x1008 + index * 0x40 + lane * 4, CORE_VOLTAGE[index]); + put_u32(out, 0x1408 + index * 0x40 + lane * 4, MEMORY_VOLTAGE[index]); + } + } + for index in 0..16 { + put_u32(out, 0x19c8 + index * 4, index as u32); + } + Ok(()) +} + +/// T8140 register windows mapped into the firmware MMIO aperture. +pub(crate) const T8140_REGISTER_WINDOWS: [RegisterWindow; 18] = [ + RegisterWindow { + slot: 0, + phys: 0x0030_1014_000, + device_va: 0xffff_fc21_8000_0000, + size: 0x004000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 3, + phys: 0x0022_0104_000, + device_va: 0xffff_fc21_8000_8000, + size: 0x018000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 9, + phys: 0x0030_03d0_000, + device_va: 0xffff_fc21_8002_8000, + size: 0x001000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 10, + phys: 0x0030_03c0_000, + device_va: 0xffff_fc21_8003_0000, + size: 0x002000, + unk_18: 0, + flag: 0, + }, + RegisterWindow { + slot: 12, + phys: 0x0040_165c_000, + device_va: 0xffff_fc21_8003_8000, + size: 0x004000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 14, + phys: 0x0030_0280_000, + device_va: 0xffff_fc21_8004_0000, + size: 0x008000, + unk_18: 0, + flag: 0, + }, + RegisterWindow { + slot: 17, + phys: 0x0048_0000_000, + device_va: 0xffff_fc21_8005_0000, + size: 0x021400, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 22, + phys: 0x0048_1000_000, + device_va: 0xffff_fc21_8007_8000, + size: 0x008000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 26, + phys: 0x0048_0d04_000, + device_va: 0xffff_fc21_8008_8000, + size: 0x008000, + unk_18: 0xd04000, + flag: 2, + }, + RegisterWindow { + slot: 27, + phys: 0x0048_0d0d_000, + device_va: 0xffff_fc21_8009_9000, + size: 0x001000, + unk_18: 0xd0d000, + flag: 2, + }, + RegisterWindow { + slot: 28, + phys: 0x0048_0d58_000, + device_va: 0xffff_fc21_800a_0000, + size: 0x008000, + unk_18: 0xd58000, + flag: 2, + }, + RegisterWindow { + slot: 29, + phys: 0x0048_0d10_000, + device_va: 0xffff_fc21_800b_0000, + size: 0x004000, + unk_18: 0xd10000, + flag: 2, + }, + RegisterWindow { + slot: 31, + phys: 0x0048_0d40_000, + device_va: 0xffff_fc21_800b_8000, + size: 0x004000, + unk_18: 0xd40000, + flag: 2, + }, + RegisterWindow { + slot: 32, + phys: 0x0048_0d60_000, + device_va: 0xffff_fc21_800c_0000, + size: 0x004000, + unk_18: 0xd60000, + flag: 2, + }, + RegisterWindow { + slot: 35, + phys: 0x0048_0e00_000, + device_va: 0xffff_fc21_800c_8000, + size: 0x004000, + unk_18: 0xe00000, + flag: 2, + }, + RegisterWindow { + slot: 39, + phys: 0x0048_0e08_000, + device_va: 0xffff_fc21_800d_0000, + size: 0x008000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 40, + phys: 0x0048_0e1c_000, + device_va: 0xffff_fc21_800e_0000, + size: 0x004000, + unk_18: 0xe1c000, + flag: 2, + }, + RegisterWindow { + slot: 41, + phys: 0x0048_0e1f_800, + device_va: 0xffff_fc21_800e_b800, + size: 0x004000, + unk_18: 0, + flag: 2, + }, +]; + +pub(crate) const T8140_REGISTER_FLAG_ONLY_SLOTS: [(usize, u32); 17] = [ + (2, 2), + (5, 2), + (6, 2), + (7, 2), + (8, 2), + (30, 2), + (33, 2), + (34, 2), + (37, 2), + (38, 2), + (42, 2), + (43, 2), + (46, 2), + (47, 2), + (48, 2), + (49, 2), + (52, 2), +]; + +/// Fail-closed validation errors for the grounded layout. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum InitdataError { + BufferSize, + AddressOutOfRange, + NullRequiredAddress, + RoleSpecificAddress, + BundleAllocationTooSmall, + ChannelTableShape, + RegisterWindow, + PairRootSeparation, + PairAliasedObject, + SharedRegionSeparation, + InterfaceMismatch, + HostMappedAllocationsDisabled, +} + +fn address_representable(address: u64) -> bool { + let high = address & !INITDATA_ADDR_MASK; + high == 0 || high == INITDATA_ADDR_PREFIX +} + +fn required_address(address: u64) -> Result<(), InitdataError> { + if address == 0 { + Err(InitdataError::NullRequiredAddress) + } else if !address_representable(address) { + Err(InitdataError::AddressOutOfRange) + } else { + Ok(()) + } +} + +fn put_u16(bytes: &mut [u8], offset: usize, value: u16) { + bytes[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn get_u32(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) +} + +fn write_runs(bytes: &mut [u8], runs: &[(usize, &[u8])]) { + for (offset, data) in runs { + bytes[*offset..*offset + data.len()].copy_from_slice(data); + } +} + +fn validate_shared(shared: &SharedAddresses) -> Result<(), InitdataError> { + required_address(shared.region_a)?; + required_address(shared.region_c)?; + required_address(shared.hw_data_bundle)?; + if shared.hw_data_bundle_alloc < HW_DATA_BUNDLE_MIN_ALLOC { + return Err(InitdataError::BundleAllocationTooSmall); + } + Ok(()) +} + +fn validate_instance(role: InstanceRole, inst: &InstanceAddresses) -> Result<(), InitdataError> { + required_address(inst.root)?; + required_address(inst.main_config)?; + required_address(inst.status_a)?; + match role { + InstanceRole::Primary => { + // The primary root has no +0xb8/+0xc0 fields at all; carrying + // secondary extras there is a role-confusion bug at the caller. + if inst.secondary_extras != [0, 0] { + return Err(InitdataError::RoleSpecificAddress); + } + required_address(inst.status_b)?; + } + InstanceRole::Secondary => { + // Every hardware-booting host hands the secondary a zero status-B; + // a nonzero value there is ungrounded, so fail closed on it. + if inst.status_b != 0 { + return Err(InitdataError::RoleSpecificAddress); + } + for extra in inst.secondary_extras { + required_address(extra)?; + } + } + } + Ok(()) +} + +/// Encode one 0x20-byte UAT level descriptor. Every field is derivable from +/// the level's index shift and entry count; the index mask +/// `(entries - 1) << shift` reproduces all three captured masks exactly. +fn encode_uat_level_descriptor(bytes: &mut [u8], offset: usize, shift: u8, entries: u16) { + bytes[offset] = 8; + bytes[offset + 1] = UAT_PAGE_BITS; + bytes[offset + 2] = UAT_PAGE_BITS; + bytes[offset + 3] = shift; + put_u16(bytes, offset + 0x04, entries); + put_u16(bytes, offset + 0x06, UAT_PAGE_SIZE); + put_u64(bytes, offset + 0x08, 1); + put_u64(bytes, offset + 0x10, UAT_LEVEL_PHYS_MASK); + put_u64(bytes, offset + 0x18, (entries as u64 - 1) << shift); +} + +/// Encode one instance's root. +/// +/// The output is zeroed first, which makes unknown bytes deterministic and, +/// critically, prevents stale host memory from becoming invented firmware +/// ABI. Root `+0x10` stays zero: it is zero at handoff on both instances and +/// its semantics are unknown. +pub(crate) fn encode_root( + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_root_for_profile(RootProfile::J700G17P, role, shared, inst, out) +} + +/// Encode the common root layout with an explicitly selected firmware build. +/// The existing runtime continues to call the J700-only wrapper above. +pub(crate) fn encode_root_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + out: &mut [u8], +) -> Result<(), InitdataError> { + // Profile-sized: identical to root_size(role) for J700/J813. + if out.len() != profile.root_size(role) { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + + out.fill(0); + put_u64(out, ROOT_VERSION, profile.version_word()); + put_u64(out, ROOT_REGION_A, shared.region_a); + // ROOT_ZERO_10 deliberately left zero. + put_u64(out, ROOT_MAIN_CONFIG, inst.main_config); + put_u64(out, ROOT_REGION_C, shared.region_c); + put_u32(out, ROOT_INSTANCE_KIND, role as u32); + put_u32(out, ROOT_CONSTANT_2C, 1); + put_u16(out, ROOT_PAGE_SIZE, UAT_PAGE_SIZE); + out[ROOT_PAGE_BITS] = UAT_PAGE_BITS; + out[ROOT_LEVEL_COUNT] = UAT_LEVEL_COUNT as u8; + for (index, (shift, entries)) in UAT_LEVELS.iter().enumerate() { + encode_uat_level_descriptor( + out, + ROOT_LEVELS + index * UAT_LEVEL_DESC_SIZE, + *shift, + *entries, + ); + } + put_u64(out, ROOT_STATUS_A, inst.status_a); + put_u64(out, ROOT_STATUS_B, inst.status_b); + if role == InstanceRole::Secondary { + put_u64(out, ROOT_SECONDARY_EXTRA_0, inst.secondary_extras[0]); + put_u64(out, ROOT_SECONDARY_EXTRA_1, inst.secondary_extras[1]); + } + Ok(()) +} + +/// Byte range of one channel-table entry inside the main configuration. +pub(crate) fn channel_table_entry_range(index: usize) -> Option> { + if index >= CHANNEL_TABLE_ENTRIES { + return None; + } + let start = MAIN_CHANNEL_TABLE + index * CHANNEL_ENTRY_SIZE; + Some(start..start + CHANNEL_ENTRY_SIZE) +} + +/// Derive the native channel table for one instance. +/// +/// `state_grid` is this instance's compact channel-state block (0x40-stride, +/// permuted per [`WORK_STATE_GRID_OFFSETS`]); `status_a` anchors the trailing +/// report/partial channels; the ring for entry 12 is `main_config + 0x4c0`; +/// work rings for the primary sit in the bundle at +/// [`WORK_RING_BUNDLE_OFFSETS`] (which requires the +/// [`NATIVE_SHARED_CLUSTER_SIZE`] cluster). The secondary leaves entries +/// 0..11 empty: it is a control-only peer with no work channels. +pub(crate) fn derive_channel_table( + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = [ChannelEntry::EMPTY; CHANNEL_TABLE_ENTRIES]; + let grid_states = |index: usize| { + let base = state_grid + WORK_STATE_GRID_OFFSETS[index] as u64; + [ + base, + base + CHANNEL_STATE_SPACING as u64, + base + 2 * CHANNEL_STATE_SPACING as u64, + ] + }; + if role == InstanceRole::Primary { + for index in 0..WORK_CHANNEL_COUNT { + table[index] = ChannelEntry { + states: grid_states(index), + ring: bundle + WORK_RING_BUNDLE_OFFSETS[index] as u64, + }; + } + } + table[CONTROL_CHANNEL_INDEX] = ChannelEntry { + states: grid_states(CONTROL_CHANNEL_INDEX), + ring: main_config + MAIN_CONTROL_RING as u64, + }; + for (slot, index) in REPORT_CHANNEL_INDICES.iter().enumerate() { + let mut states = [0u64; 3]; + for (word, offset) in states.iter_mut().zip(REPORT_STATE_STATUS_A_OFFSETS[slot]) { + *word = status_a + offset as u64; + } + table[*index] = ChannelEntry { + states, + ring: status_a + REPORT_RING_STATUS_A_OFFSETS[slot] as u64, + }; + } + table[PARTIAL_CHANNEL_INDEX] = ChannelEntry { + states: [status_a + PARTIAL_STATE_STATUS_A_OFFSET as u64, 0, 0], + ring: 0, + }; + table +} + +/// [`derive_channel_table`] with the profile's work-ring placement. For J700 +/// and J813 the result is identical to [`derive_channel_table`]. G18G keeps +/// the same channel/ring permutation and status-A-relative report layout +/// (FW_PROVEN consumers b000 0x84f0/0x27048/0x7f14/0x19984 match G17); only +/// the ring bundle offsets move (+0xf00, from the 26A428 size table). +pub(crate) fn derive_channel_table_for_profile( + profile: RootProfile, + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = derive_channel_table(role, state_grid, status_a, main_config, bundle); + if role == InstanceRole::Primary { + let rings = profile.work_ring_bundle_offsets(); + for (index, entry) in table.iter_mut().take(WORK_CHANNEL_COUNT).enumerate() { + entry.ring = bundle + rings[index] as u64; + } + } + table +} + +fn validate_populated_entry(entry: &ChannelEntry) -> Result<(), InitdataError> { + // A populated channel needs all three state-counter addresses — a null + // producer (state 2) on a populated entry makes firmware fault on address + // zero — and a real ring. + for state in entry.states { + if state == 0 { + return Err(InitdataError::ChannelTableShape); + } + if !address_representable(state) { + return Err(InitdataError::AddressOutOfRange); + } + } + required_address(entry.ring) +} + +fn validate_channel_table( + role: InstanceRole, + main_config: u64, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], +) -> Result<(), InitdataError> { + for (index, entry) in channels.iter().enumerate() { + match index { + 0..=11 => match role { + InstanceRole::Primary => validate_populated_entry(entry)?, + // The secondary has no work channels at all. + InstanceRole::Secondary => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + }, + 12 => { + validate_populated_entry(entry)?; + // Device control is embedded in the main object, at exactly + // main + 0x4c0 on both instances. + if entry.ring != main_config + MAIN_CONTROL_RING as u64 { + return Err(InitdataError::ChannelTableShape); + } + } + 13 | 14 => validate_populated_entry(entry)?, + 15 => { + // Partial: a first state address and nothing else. + required_address(entry.states[0])?; + if entry.states[1] != 0 || entry.states[2] != 0 || entry.ring != 0 { + return Err(InitdataError::ChannelTableShape); + } + } + _ => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + } + } + Ok(()) +} + +/// Encode one instance's 0x600-byte main configuration object. +/// +/// `region_views` must be `Some` for the primary and `None` for the +/// secondary: the secondary's region views carry the two low context-0 +/// values with no addresses at all. +pub(crate) fn encode_main_config( + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], + region_views: Option<&PrimaryRegionViews>, + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_main_config_for_profile(RootProfile::J700G17P, role, shared, inst, + channels, region_views, out) +} + +/// The two complete M5/25G76 pre-handoff main configurations match the common +/// layout except primary +0x32c/+0x344 and secondary +0x300. Keep these +/// measured scalars build-specific; their meanings are not inferred here. +pub(crate) fn encode_main_config_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], + region_views: Option<&PrimaryRegionViews>, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != MAIN_CONFIG_SIZE { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + validate_channel_table(role, inst.main_config, channels)?; + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + required_address(views.sentinel_high)?; + for high in views.alias_high { + required_address(high)?; + } + } + (InstanceRole::Secondary, None) => {} + _ => return Err(InitdataError::RoleSpecificAddress), + } + // Every view must fit inside the one bundle allocation. (The last two end + // exactly at 0xf000, inside the 0x10000 minimum.) The view table is the + // profile's: J700/J813 keep BUNDLE_VIEW_OFFSETS/EXTENTS unchanged. + let view_offsets = profile.bundle_view_offsets(); + for (offset, extent) in view_offsets.iter().zip(profile.bundle_view_extents()) { + if offset + extent > shared.hw_data_bundle_alloc { + return Err(InitdataError::BundleAllocationTooSmall); + } + } + + out.fill(0); + // Firmware log enable. The emitter at A000 0xfffffc0000021fa0 reads this + // word and returns immediately when it is zero; every buffer the facility + // needs is already allocated and pointed at by the addresses below. The + // records are plain ASCII. Shipping macOS provisions the whole facility + // and then throws the messages away (drainFirmwareLogRing is `bti c; ret`), + // which is why the captured value is zero. + if crate::g18::params::g17p_fw_log.value() != 0 { + put_u32(out, MAIN_FIRMWARE_LOG_ENABLE, 1); + } + let bundle = shared.hw_data_bundle; + put_u64(out, MAIN_HW_DATA, bundle); + let repeated = bundle + profile.repeated_region_bundle_offset() as u64; + put_u64(out, MAIN_REPEATED_0, repeated); + put_u64(out, MAIN_REPEATED_1, repeated); + + for (index, entry) in channels.iter().enumerate() { + let base = MAIN_CHANNEL_TABLE + index * CHANNEL_ENTRY_SIZE; + for (word, state) in entry.states.iter().enumerate() { + put_u64(out, base + word * 8, *state); + } + put_u64(out, base + CHANNEL_ENTRY_STATE_COUNT * 8, entry.ring); + } + + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + // The five bare pointers are views into the one bundle + // allocation, at fixed offsets. Unaligned stores at +0x254. + for (index, offset) in view_offsets.iter().enumerate() { + put_u64(out, MAIN_BUNDLE_VIEWS + index * 8, bundle + *offset as u64); + } + // High-only sentinel followed by two low/high region-view pairs. + put_u64(out, MAIN_REGION_VIEWS, views.sentinel_high); + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x10, views.alias_high[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u64(out, MAIN_REGION_VIEWS + 0x20, views.alias_high[1]); + if profile == RootProfile::J700G17P { + put_u32(out, MAIN_PRIMARY_SCALAR_32C, 1); + put_u32(out, MAIN_PRIMARY_SCALAR_344, 0xabcd_abcd); + } + // KEXT_PROVEN for G18G too (0xfffffe0008c31780: main+0x3e0=0xff). + put_u32(out, MAIN_PRIMARY_MASK, MAIN_PRIMARY_MASK_VALUE); + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + (InstanceRole::Secondary, _) => { + // The secondary keeps the two low context-0 values but carries no + // view or region addresses; the values are therefore not + // properties of any region the host allocates. + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u32(out, MAIN_SECONDARY_KIND, match profile { + RootProfile::J700G17P => MAIN_SECONDARY_KIND_VALUE, + RootProfile::J813G17G25G76 => 0xc, + // INFERRED: 26A428 stores accel+0x10a74 here (25G76: + // accel+0xed30, measured 0xc on J813). Value not read on J873. + RootProfile::J873G18G26A428 => G18G_26A428_MAIN_SECONDARY_KIND, + }); + if profile == RootProfile::J873G18G26A428 { + // KEXT_PROVEN (26A428 initFirmwareSharedData, g18g_key.c:4464): + // role 1 main+0x481 = FW VA of the perf/policy allocation. + // 25G76 stored it at +0x471 and also wrote an accelerator + // pair at +0x479/+0x481, which 26A428 no longer writes. + put_u64(out, G18G_26A428_MAIN_SECONDARY_POLICY_PTR, + bundle + G18G_26A428_SHARED_LAYOUT.policy as u64); + } else { + // One additional unaligned pointer *into* primary view 2, landing + // five bytes before that view's first populated run. + let extra = bundle + + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] as u64 + + SECONDARY_EXTRA_VIEW_OFFSET as u64; + put_u64(out, MAIN_SECONDARY_EXTRA_PTR, extra); + } + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + _ => unreachable!(), + } + Ok(()) +} + +/// Encode the shared 0x3db4-byte hardware-data object (the base of the +/// bundle). Both instances reference this one object; handing the secondary a +/// private copy makes it fault. +/// +/// `completion_aliases` are the low/high GPU mappings of the two retained +/// completion-queue backings named by descriptor ordinals 0 and 2. +pub(crate) fn encode_hw_data( + windows: &[RegisterWindow], + flag_only_slots: &[(usize, u32)], + completion_aliases: [G17PKsmCompletionAliases; 2], + qos_resource_high: u64, + sksm_qid_resource_high: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + let mut used = [false; HW_REGISTER_MAP_SLOTS]; + let mut claim = |slot: usize| -> Result<(), InitdataError> { + if slot >= HW_REGISTER_MAP_SLOTS || used[slot] { + return Err(InitdataError::RegisterWindow); + } + used[slot] = true; + Ok(()) + }; + for window in windows { + claim(window.slot)?; + if window.phys == 0 || window.size == 0 { + return Err(InitdataError::RegisterWindow); + } + // Firmware reaches its registers only through this table; the device + // VAs live in one fixed aperture. + if !HW_REGISTER_APERTURE.contains(&window.device_va) { + return Err(InitdataError::RegisterWindow); + } + } + for (slot, _flag) in flag_only_slots { + claim(*slot)?; + } + for aliases in completion_aliases { + required_address(aliases.low)?; + required_address(aliases.high)?; + } + required_address(qos_resource_high)?; + required_address(sksm_qid_resource_high)?; + + out.fill(0); + // Native's `AGXFirmware::initFirmwareData` opens this object with five + // qwords that Linux has always left zero: the range-3 base and size, the + // architectural GART window base twice (Asahi's HwDataB calls the pair + // "USC start"), and the reserved page at the top of range 5. They are the + // only place the host tells the firmware side about 0x10_0000_0000, which + // is the base hardware folds into every compact TA/VDM pointer, so a zero + // here leaves USC-class units resolving bare offsets. + for (offset, value) in HW_FIRMWARE_RANGE_HEAD { + put_u64(out, *offset, *value); + } + for window in windows { + let base = HW_REGISTER_MAP_OFFSET + window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, window.phys); + put_u64(out, base + 0x08, window.device_va); + put_u32(out, base + 0x10, window.size); + put_u32(out, base + 0x14, window.size); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + for (slot, flag) in flag_only_slots { + let base = HW_REGISTER_MAP_OFFSET + slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u32(out, base + 0x20, *flag); + } + + put_u32(out, HW_CHIP_ID_OFFSET, HW_CHIP_ID_T8140); + + for index in 0..HW_PERF_LADDER_ENTRIES { + put_u32(out, HW_PERF_GROUP_BASES[0] + index * 4, HW_PERF_FREQ_A[index]); + put_u32(out, HW_PERF_FREQ_LADDER_B + index * 4, HW_PERF_FREQ_B[index]); + put_u32(out, HW_PERF_SCALE_LADDER_B + index * 4, HW_PERF_SCALE_B[index]); + put_u32( + out, + HW_PERF_RELATIVE_LADDER_A + index * 4, + HW_PERF_RELATIVE_A_VALUES[index], + ); + put_u32( + out, + HW_PERF_RELATIVE_LADDER_B + index * 4, + HW_PERF_RELATIVE_B_VALUES[index], + ); + put_u32(out, HW_PERF_INDEX_MAP_A + index * 4, HW_PERF_INDEX_A_VALUES[index]); + put_u32(out, HW_PERF_INDEX_MAP_B + index * 4, HW_PERF_INDEX_B_VALUES[index]); + put_u32(out, HW_PERF_GROUP_BASES[1] + index * 4, HW_PERF_FREQ_B[index]); + + for repeat in 0..HW_PERF_GROUP_REPEATS[0] { + put_u32( + out, + HW_PERF_GROUP_BASES[0] + + HW_PERF_VOLTAGE_DELTA + + index * HW_PERF_STATE_BLOCK_STRIDE + + repeat * 4, + HW_PERF_CORE_VOLTAGE[index], + ); + put_u32( + out, + HW_PERF_GROUP_BASES[0] + + HW_PERF_MEMORY_VOLTAGE_DELTA + + index * HW_PERF_STATE_BLOCK_STRIDE + + repeat * 4, + HW_PERF_MEMORY_VOLTAGE[index], + ); + } + put_u32( + out, + HW_PERF_GROUP_BASES[1] + + HW_PERF_VOLTAGE_DELTA + + index * HW_PERF_STATE_BLOCK_STRIDE, + HW_PERF_CORE_VOLTAGE[index], + ); + put_u32( + out, + HW_PERF_GROUP_BASES[1] + + HW_PERF_MEMORY_VOLTAGE_DELTA + + index * HW_PERF_STATE_BLOCK_STRIDE, + HW_PERF_MEMORY_VOLTAGE[index], + ); + } + + // The acceptance block: derived from the work-channel count rather than + // copied, so a constructed descriptor needs nothing from a capture here. + put_u16(out, HW_ACCEPT_CHANNEL_COUNT, WORK_CHANNEL_COUNT as u16); + for group in HW_ACCEPT_FLAG_GROUPS { + for (index, value) in HW_ACCEPT_FLAG_PATTERN.iter().enumerate() { + put_u32(out, group + index * 4, *value); + } + } + for channel in 0..WORK_CHANNEL_COUNT { + put_u32(out, HW_ACCEPT_UNASSIGNED_TABLE + channel * 4, 0xffff_ffff); + } + for offset in HW_ACCEPT_TRAILING_ONES { + // NB +0x2600 is also DAT_178648, a firmware GATE: FUN_e004's per-DM + // KSM completion-block arm runs only when (a == 0 || DAT_178648 == 0). + // Writing 0 here to let that arm run was TESTED and the driver failed + // probe with -22. I first wrote that our own initdata validation + // rejected it; that was an invention -- nothing in the runtime path + // validates this field, and the cause of the -22 was never + // established. What settles it instead is the live 25G83 capture: + // native's hardware-data block reads 1 at +0x2600 and 1 at +0x2608, + // the same as ours, so this is not a divergence. + put_u32(out, offset, 1); + } + + put_u32( + out, + G17P_KSM_COMPLETION_TABLE_PREFIX_OFFSET, + G17P_KSM_COMPLETION_TABLE_PREFIX, + ); + for (aliases, ordinal) in completion_aliases + .iter() + .zip(G17P_KSM_COMPLETION_PRESENT_ORDINALS) + { + let base = G17P_KSM_COMPLETION_TABLE_OFFSET + + ordinal as usize * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE; + put_u64(out, base, aliases.low); + put_u64(out, base + 0x08, aliases.high); + put_u32(out, base + 0x10, G17P_KSM_COMPLETION_CAPACITY); + put_u32(out, base + 0x14, G17P_KSM_COMPLETION_ENTRY_SIZE); + put_u32(out, base + 0x18, ordinal); + // +0x1c is the firmware-owned processed cursor and starts at zero. + } + + put_u64(out, G17P_QOS_RESOURCE_LIVE_OFFSET, qos_resource_high); + put_u64( + out, + G17P_SKSM_QID_CONFIG_2700_OFFSET, + G17P_SKSM_QID_CONFIG_2700, + ); + // The valid live G17P HWData object leaves +0x2708, +0x27a4 and +0x27ac + // zero. The kext writes those only in a staging/version tail and B1 has no + // consumer; the live QoS pointer is the independent +0x26ac field above. + + // Recorded platform constants. Descriptor *acceptance* survives without + // them, but the device-control phase that follows does not. +0x2630 is + // deliberately absent from the runs: it is firmware state and must start + // zero (the fill above guarantees it). + write_runs(out, HW_DATA_OPAQUE_RUNS); + // The host->firmware transport selector. + // + // `AGXFirmware::initFirmwareData` @0xfffffe0008a20940 exports + // `initdata[0x26f8] = accel[0x4f8] & 1`. That same bit decides, in + // `AGX{TA,3D,CL}ChannelSKSM::submitBuffer`, whether the host publishes the + // command record and the tag-14 AddKicks as CHANNEL COMMANDS through + // `writeChannelCommandPointer` (bit set, `accel[0x10769] & 1 == 0`) or + // instead pushes onto the host kick-trace ring and drives + // `configureHardware`/`addKicks` over MMIO (bit clear). + // + // A live SKSM capture on this target shows tag-15 and tag-14 published as + // channel commands for both render queues, so native runs with the bit + // SET and this byte at 1. This driver uses the same channel-command + // transport but left the byte at 0 -- it is outside every + // `HW_DATA_OPAQUE_RUNS` extent and nothing else wrote it -- so the + // firmware was told the host drives the MMIO bridge while the host in + // fact speaks channel commands. Declare the transport we actually use. + // MEASURED 2026-09-02, and it contradicts the reasoning above. Native's + // complete hardware-data block (hwdata-gfx.txt, captured from + // accel+0x5b0 -> +0x2b8 at submit3DChannel ENTRY, i.e. long after init) + // carries ZERO at 0x26f8. The "native runs with the bit SET" conclusion was + // inferred from seeing channel commands on the wire, not read off the + // block, and the block disagrees. Publish what native publishes. + if let Some(byte) = out.get_mut(HW_DATA_TRANSPORT_SELECTOR_OFFSET) { + *byte = 0; + } + // The measured performance tables were encoded above. Their values are + // target-scoped observations rather than a complete ADT derivation; see + // [`PERF_TABLE_VALUES_ENCODED`]. + Ok(()) +} + +/// Encode the shared 0x1000-byte region C. Both roots point at the same +/// object; giving the secondary a private copy is a difference from what a +/// working host does. +pub(crate) fn encode_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + encode_region_c_for_profile(RootProfile::J700G17P, out) +} + +/// Encode target-specific pre-handoff platform policy. M5's native policy is +/// kept separate from the experimental T8140 watchdog and idle overrides. +pub(crate) fn encode_region_c_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + if profile == RootProfile::J873G18G26A428 { + // Derived from the J813 bytes below through the proven +4 shift. + return encode_g18g_26a428_region_c(out); + } + out.fill(0); + write_runs(out, REGION_C_RUNS); + if profile == RootProfile::J813G17G25G76 { + // Scalars measured before each ASC received its root on two cold + // J813 / 25G76 boots. These are a scoped reference configuration, + // not generic M5 calibration or a power-management qualification. + // The final pair moved by 0x20; leaving the T8140 pair in place would + // initialize unrelated M5 fields. + for (offset, value) in [ + (0x084, 0x0000_5eb0), + (0x088, 0x0000_04b0), + (0x08c, 0x0000_04b0), + (0x098, 0x0000_04b0), + (0x7c4, 0x0000_04e2), + (0x7e0, 0x3f33_3333), + (0x9a8, 0x0000_0333), + (0xe38, 0), + (0xe3c, 0), + (0xe58, 0x0000_0300), + (0xe5c, 0x0000_0100), + ] { + put_u32(out, offset, value); + } + return Ok(()); + } + // regionC+0x9cc is the KSM progress-watchdog threshold for data master 2 + // (compute); the firmware copies it to 0x1778d0 and its handler starts + // `cbz w20 -> ret`, so 0 disables the watchdog. We ship 0x32 (50) here and + // leave the other data master's slot at 0 -- i.e. we arm the watchdog on + // exactly the queue we submit to. When work cannot progress for any other + // reason this turns a stall into a self-sustaining recovery loop. + if crate::g18::fw::config::PROGRESS_WATCHDOG == 0 { + put_u32(out, 0x9cc, 0); + } + // Native initializes the idle-descent selector to zero. Keep the old + // alternate predicate only as an explicit diagnostic override. + if crate::g18::fw::config::IDLE_SELECTOR != 0 { + out[0xe21] = 1; + } + for (offset, value) in [ + (0x7c, 1), + (0x80, 1), + (0x9c, 0x3000_001d), + (0xa0, 0x3100_0002), + ] { + put_u32(out, offset, value); + } + Ok(()) +} + +/// Overlay the target-scoped static third-page bytes and its one live state +/// pointer without disturbing hardware-data or overlapping view content. +pub(crate) fn encode_bundle_static( + state_ptr: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() < HW_DATA_BUNDLE_MIN_ALLOC { + return Err(InitdataError::BufferSize); + } + required_address(state_ptr)?; + write_runs(out, BUNDLE_STATIC_RUNS); + for (index, runs) in BUNDLE_VIEW_RUNS.iter().enumerate() { + let base = BUNDLE_VIEW_OFFSETS[index]; + let valid = BUNDLE_VIEW_EXTENTS[index]; + for (offset, data) in *runs { + if offset + data.len() > valid { + continue; + } + let start = base + offset; + out[start..start + data.len()].copy_from_slice(data); + } + } + for (view, offset, count, trailing_ff) in BUNDLE_VIEW_U16_REPEATS { + let base = BUNDLE_VIEW_OFFSETS[*view] + offset; + let end = base + *count * 2 + usize::from(*trailing_ff); + if end > BUNDLE_VIEW_OFFSETS[*view] + BUNDLE_VIEW_EXTENTS[*view] { + return Err(InitdataError::BundleAllocationTooSmall); + } + for index in 0..*count { + out[base + index * 2..base + index * 2 + 2].copy_from_slice(&[0xff, 0x7f]); + } + if *trailing_ff { + out[base + *count * 2] = 0xff; + } + } + put_u64(out, 0x8ed8, state_ptr); + Ok(()) +} + +/// Encode one 0x80-byte status block in its pre-init state: u32 1 at `+0x04`, +/// everything else zero. `+0x10`/`+0x14` are post-acknowledgement firmware +/// state and must be zero at handoff. +pub(crate) fn encode_status_block(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, STATUS_BLOCK_ONE_OFFSET, 1); + Ok(()) +} + +/// Encode the complete pre-init primary status/config object. +pub(crate) fn encode_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != PRIMARY_STATUS_B_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, PRIMARY_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, PRIMARY_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, PRIMARY_STATUS_B_CONFIG_HEADER, 1); + for (offset, data) in PRIMARY_STATUS_B_CONFIG_RUNS { + let start = PRIMARY_STATUS_B_CONFIG_OFFSET + offset; + out[start..start + data.len()].copy_from_slice(data); + } + Ok(()) +} + +/// Encode the secondary root's independent `+0xc0` target. +pub(crate) fn encode_secondary_root_extra_1(out: &mut [u8]) -> Result<(), InitdataError> { + encode_secondary_root_extra_1_for_profile(RootProfile::J700G17P, out) +} + +pub(crate) fn encode_secondary_root_extra_1_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + write_runs(out, SECONDARY_ROOT_EXTRA_1_RUNS); + // G18G: KEXT_PROVEN identical store geometry for this 0x60 object + // (26A428 vs 25G76 R_60 stores, all at the same offsets); the J813 values + // are carried unchanged (INFERRED, listed in G18G_26A428_UNPROVEN). + if matches!(profile, RootProfile::J813G17G25G76 | RootProfile::J873G18G26A428) { + put_u32(out, 0x10, 2); + } + Ok(()) +} + +// --- GPU firmware fast-die controller (GFX1) -------------------------------- +// +// KEXT_PROVEN (25G83 AGXG17P): +// - AGXAcceleratorG17P::populateFastDieDeviceConfigData (0x88c7af0) sets target +// 110.0 C and sensor mask 0xaa0 when accel[0x4a0] == 0x1f, else 0x82a. It +// zeroes the per-sensor array for each mask bit. It panics ("Not all MTR +// Sensors required for FastDie are enabled") only when the host MTR-enable word +// accel[0xe978] is non-zero and lacks a required bit. +// - The base populate (0x883d41c) takes optional ADT gpu-fast-die0-sensor-mask64/ +// -sensor-mask/-target (default 101.0 C, overridden above). setupConfig reads +// the optional gpu-fast-die0-release-temp/-prop-tgt-delta/-proportional-gain/ +// -integral-gain and gpu-fastdie-mrc-period. The J700 ADT carries none of +// them. +// - initPowerAndPerformanceData copies the staged values into +// - the GFX1 power object: `[[fw+0x2d8]+0x6678]`; GFX1 reaches it through +// secondary main +0x471 = bundle view 2 + 0xe40; +// - the 0x6a8-byte secondary-root +0xb8 object (`fw+0xcd8`), which Linux +// places at primary status B + 0xe440. +// FW_PROVEN (fw1 25G83 b100, 0x174d0): +// - When the +0xb8 object's +0xc4 flag is set, GFX1 copies its +0xc8..+0xe4 +// fields into the power object. Native updateLifetimeServoTempTarget and +// updateFastDieController set that flag after writing the fields. +// - GFX1 takes the lower of that target and an external target, then programs +// each masked MTR sensor's threshold registers (stride 0x200, +0x1000/+0x1004/ +// +0x1014) at (target - delta) / 100 C. +// The recorded Linux runs already carry every non-zero value native publishes +// here (see the test `fast_die_config_matches_native`). + +/// Offset of the GFX1 power object inside the hardware-data bundle. +pub(crate) const FASTDIE_OBJECT_BUNDLE_OFFSET: usize = + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] + SECONDARY_EXTRA_VIEW_OFFSET; +pub(crate) const FASTDIE_OBJ_MASK: usize = 0x2310; +pub(crate) const FASTDIE_OBJ_RELEASE_CC: usize = 0x2318; +pub(crate) const FASTDIE_OBJ_KI_DT: usize = 0x233c; +pub(crate) const FASTDIE_OBJ_KP: usize = 0x234c; +pub(crate) const FASTDIE_OBJ_DELTA_CC: usize = 0x2364; +pub(crate) const FASTDIE_OBJ_TARGET_CC: usize = 0x236c; +pub(crate) const FASTDIE_OBJ_STATE: usize = 0x25bc; +/// Offsets inside the secondary-root +0xb8 object (status B + 0xe440 here). +pub(crate) const FASTDIE_LIVE_UPDATE: usize = 0xc4; +pub(crate) const FASTDIE_LIVE_TARGET_CC: usize = 0xc8; +pub(crate) const FASTDIE_LIVE_RELEASE_CC: usize = 0xcc; +pub(crate) const FASTDIE_LIVE_KP: usize = 0xd8; +pub(crate) const FASTDIE_LIVE_KI_DT: usize = 0xdc; +/// Native G17P target, 0.01 C. +pub(crate) const FASTDIE_NATIVE_TARGET_CC: u32 = 11_000; +/// Diagnostic overrides may only lower the target, never below this, so the +/// recorded 80.00 C release stays below it. +pub(crate) const FASTDIE_MIN_TARGET_CC: u32 = 8_500; + +/// Effective fast-die configuration as encoded (raw words; floats as bits). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct FastDieConfig { + pub(crate) mask: u64, + pub(crate) target_cc: u32, + pub(crate) release_cc: u32, + pub(crate) delta_cc: u32, + pub(crate) kp_bits: u32, + pub(crate) ki_dt_bits: u32, + pub(crate) state: u32, + pub(crate) live_target_cc: u32, + pub(crate) live_release_cc: u32, + pub(crate) live_kp_bits: u32, + pub(crate) live_ki_dt_bits: u32, + pub(crate) live_update: u32, +} + +/// Read the fast-die fields from an encoded bundle and status-B object. +pub(crate) fn fast_die_config(bundle: &[u8], status_b: &[u8]) -> Result { + let o = FASTDIE_OBJECT_BUNDLE_OFFSET; + let l = PRIMARY_STATUS_B_CONFIG_OFFSET; + if bundle.len() < o + FASTDIE_OBJ_STATE + 4 || status_b.len() < l + FASTDIE_LIVE_KI_DT + 4 { + return Err(InitdataError::BufferSize); + } + Ok(FastDieConfig { + mask: get_u64(bundle, o + FASTDIE_OBJ_MASK), + target_cc: get_u32(bundle, o + FASTDIE_OBJ_TARGET_CC), + release_cc: get_u32(bundle, o + FASTDIE_OBJ_RELEASE_CC), + delta_cc: get_u32(bundle, o + FASTDIE_OBJ_DELTA_CC), + kp_bits: get_u32(bundle, o + FASTDIE_OBJ_KP), + ki_dt_bits: get_u32(bundle, o + FASTDIE_OBJ_KI_DT), + state: get_u32(bundle, o + FASTDIE_OBJ_STATE), + live_target_cc: get_u32(status_b, l + FASTDIE_LIVE_TARGET_CC), + live_release_cc: get_u32(status_b, l + FASTDIE_LIVE_RELEASE_CC), + live_kp_bits: get_u32(status_b, l + FASTDIE_LIVE_KP), + live_ki_dt_bits: get_u32(status_b, l + FASTDIE_LIVE_KI_DT), + live_update: get_u32(status_b, l + FASTDIE_LIVE_UPDATE), + }) +} + +/// Validate a diagnostic target: 0 keeps native; otherwise lower-only within +/// [`FASTDIE_MIN_TARGET_CC`]..=[`FASTDIE_NATIVE_TARGET_CC`]. +pub(crate) fn fast_die_target_override(requested_cc: u32) -> Result, InitdataError> { + match requested_cc { + 0 => Ok(None), + t if (FASTDIE_MIN_TARGET_CC..=FASTDIE_NATIVE_TARGET_CC).contains(&t) => Ok(Some(t)), + _ => Err(InitdataError::BufferSize), + } +} + +/// Apply a validated target to the power object copy (pre-boot encode only). +pub(crate) fn apply_fast_die_target_bundle(bundle: &mut [u8], target_cc: u32) -> Result<(), InitdataError> { + let at = FASTDIE_OBJECT_BUNDLE_OFFSET + FASTDIE_OBJ_TARGET_CC; + if bundle.len() < at + 4 { + return Err(InitdataError::BufferSize); + } + put_u32(bundle, at, target_cc); + Ok(()) +} + +/// Apply a validated target to the +0xb8 live object copy (pre-boot encode +/// only; no update flag: firmware starts from these words). +pub(crate) fn apply_fast_die_target_status_b(status_b: &mut [u8], target_cc: u32) -> Result<(), InitdataError> { + let at = PRIMARY_STATUS_B_CONFIG_OFFSET + FASTDIE_LIVE_TARGET_CC; + if status_b.len() < at + 4 { + return Err(InitdataError::BufferSize); + } + put_u32(status_b, at, target_cc); + Ok(()) +} + +/// Validate the cross-instance invariants established before either ASC is +/// kicked. The two roots are not placed freely: the secondary's sits exactly +/// [`SECONDARY_ROOT_DELTA`] above the primary's. +pub(crate) fn validate_pair( + primary: &InstanceAddresses, + secondary: &InstanceAddresses, +) -> Result<(), InitdataError> { + if secondary.root != primary.root + SECONDARY_ROOT_DELTA { + return Err(InitdataError::PairRootSeparation); + } + if primary.main_config == secondary.main_config || primary.status_a == secondary.status_a { + return Err(InitdataError::PairAliasedObject); + } + Ok(()) +} + +/// Validate the per-instance halves of the ADT shared region: the secondary +/// maps it from the primary's base plus [`SECONDARY_SHARED_REGION_DELTA`]. +pub(crate) fn validate_shared_region_bases( + primary_base: u64, + secondary_base: u64, +) -> Result<(), InitdataError> { + if secondary_base != primary_base + SECONDARY_SHARED_REGION_DELTA { + return Err(InitdataError::SharedRegionSeparation); + } + Ok(()) +} + +/// Reproduce the firmware's root acceptance checks. +/// +/// The version-word gate is re-derived from our own J700 image +/// (`__TEXT+0x1e8a8`); a mismatch is answered with "Host and FW Interface is +/// mismatched" on the crashlog endpoint. The `+0x2c` nonzero gate was proven +/// on the M5 parser and the field is constant 1 in every T8140 capture; its +/// T8140 parse-gate status has not been independently re-derived, but a host +/// must write 1 there regardless. Passing this does not mean the rest of the +/// descriptor is complete. +pub(crate) fn check_firmware_root_gates(root: &[u8]) -> Result<(), InitdataError> { + check_firmware_root_gates_for_profile(RootProfile::J700G17P, root) +} + +pub(crate) fn check_firmware_root_gates_for_profile( + profile: RootProfile, + root: &[u8], +) -> Result<(), InitdataError> { + if root.len() != ROOT_SIZE_PRIMARY && root.len() != ROOT_SIZE_SECONDARY { + return Err(InitdataError::BufferSize); + } + // G18G: both roles are copied as 0xc8 bytes (FW_PROVEN b000 0x21664). + if profile == RootProfile::J873G18G26A428 && root.len() != G18G_26A428_ROOT_SIZE { + return Err(InitdataError::BufferSize); + } + if get_u64(root, ROOT_VERSION) != profile.version_word() { + return Err(InitdataError::InterfaceMismatch); + } + if get_u32(root, ROOT_CONSTANT_2C) == 0 { + return Err(InitdataError::HostMappedAllocationsDisabled); + } + Ok(()) +} + +/// Recorded T8140 hardware-data platform constants (values known, meanings +/// not; derived from the hardware-validated `g17p_initdata.py` +/// `HWDATA_CONSTANTS`). 29 sparse runs, 262 bytes total. `+0x2630` is +/// deliberately absent: it is firmware state, not host configuration. +/// +/// Native fixture pointer bytes are not platform constants. The original +/// +0x26ad run included seven bytes of the captured QoS allocation address; +/// replaying it after encode_hw_data's live pointer store silently replaced +/// the owned allocation with that unrelated address. Keep only its adjacent +/// state words here and encode the complete pointer from its retained owner. +/// +/// +0x0f08 is zero in native too. That field is the firmware's ISR mask +/// selector -- FUN_1e858 copies it to DAT_1785f4, and FUN_215cc picks the +/// interrupt-enable word from it as +/// DAT_1785f4 == 2 ? 0x0e00620038 : 0x0a00620038 +/// storing it to DAT_179328, which the ISR ANDs against MMIO[0x10a10]. Writing +/// 2 here was tested and does exactly that (0x10a08 measured 0x0e00620038, +/// bit 34 enabled) and changes nothing else, so leaving it unset is right and +/// this part is not the variant-2 configuration. +/// +/// A caution for anyone diffing this table again: it has TWENTY-NINE runs, +/// several spanning many lines. A naive `\(0x..,\s*&\[(.*?)\]\)` regex +/// silently matches only 19 of them and mis-associates offsets, which +/// manufactures differences that are not there and, if used to rewrite the +/// table, corrupts it into 17 runs / 292 bytes and makes probe fail with -22. +/// Parse by matching brackets, and check the 29/262 totals before trusting a +/// comparison. +pub(crate) const HW_DATA_OPAQUE_RUNS: &[(usize, &[u8])] = &[ + (0x0004, &[0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0xff]), + ( + 0x0014, + &[ + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x80, + 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, 0x81, 0x21, 0xfc, 0xff, 0xff, + ], + ), + (0x00e0, &[0x08, 0x20]), + (0x00ea, &[0x08, 0x20]), + ( + 0x00f4, + &[ + 0x08, 0x20, 0x00, 0x00, 0xcb, 0x24, 0x00, 0x00, 0xfa, 0x2c, 0x8a, 0xc9, 0xcb, 0x24, + 0xf6, 0xf4, 0x17, 0xe9, 0x77, 0x18, 0xcb, 0x24, 0xd9, 0x38, 0x00, 0x00, 0xab, 0xbd, + ], + ), + ( + 0x02d9, + &[ + 0x20, 0xf8, 0xff, 0xdb, 0x2c, 0x2d, 0xd3, 0x00, 0x20, 0x00, 0xf5, 0x26, 0xe9, 0xda, + 0x21, 0x00, 0x20, 0xb6, 0x38, 0x08, 0x00, 0x42, 0xc7, + ], + ), + (0x03e0, &[0xe0, 0x7f]), + (0x03ea, &[0xe0, 0x7f]), + (0x03f4, &[0xe0, 0x7f, 0x00, 0x00, 0x00, 0x80]), + (0x0403, &[0x80]), + (0x040d, &[0x80]), + ( + 0x05d8, + &[ + 0x45, 0x26, 0x23, 0x4b, 0x98, 0x0e, 0x00, 0x00, 0x5f, 0xea, 0xa2, 0xd5, 0xff, 0x3f, + 0x00, 0x40, 0x00, 0x40, 0x5e, 0xca, 0xa2, 0xf5, 0x00, 0x40, + ], + ), + ( + 0x0e94, + &[ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, + ], + ), + ( + 0x0eb8, + &[ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5d, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + ], + ), + (0x0f04, &[0x1f]), + // NOTE, do not "fix" this by adding 0x0f08 without evidence. That u32 is + // the firmware's ISR mask selector: FUN_1e858 copies hw-data +0xf08 to + // DAT_1785f4 (store at b1 0x1e9e8), and FUN_215cc picks the interrupt + // enable word from it with + // csel: DAT_1785f4 == 2 ? 0x0e00620038 : 0x0a00620038 + // storing the result to DAT_179328, which the ISR at 0x216f0 ANDs against + // MMIO[0x10a10] to decide which interrupts to service. The field is absent + // here, so it reads zero and 0x10a08 measures 0x0a00620038 -- the variant + // without bit 34. + // + // TESTED 2026-09-02: writing 2 here does exactly what the disassembly + // says -- 0x10a08 measured 0x0e00620038, bit 34 enabled -- and changes + // nothing else. The render still stalls with bit 4 clear, no KTrace 2, + // fragment status 4. Reverted because there is no oracle saying this part + // is variant 2 and it alters firmware interrupt behaviour on a guess. + // Both variants already enable bits 3, 4 and 5, so neither withholds + // new-work. + (0x0f24, &[0x04]), + (0x0f34, &[0x01, 0x00, 0x00, 0x00, 0x01]), + (0x0f4c, &[0x31]), + (0x0f6c, &[0x01]), + (0x0f88, &[0x06, 0x00, 0x00, 0x00, 0x01]), + (0x0fac, &[0x01]), + ( + 0x0fb8, + &[ + 0x1e, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0a, + ], + ), + (0x1cd8, &[0x0a]), + ( + 0x2544, + &[ + 0x04, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x07, 0x00, + 0x00, 0x00, 0x07, + ], + ), + ( + 0x2568, + &[0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01], + ), + // +0x26ac..+0x26b3 is the live QoS pointer, never captured allocation bytes. + (0x26b4, &[0x01, 0x00, 0x00, 0x00, 0x01]), + (0x26e8, &[0x01]), + (0x3d98, &[0xff, 0xff, 0xff, 0xff]), + (0x3db0, &[0xff, 0xff, 0xff, 0xff]), +]; + +/// Recorded T8140 region-C platform tuning constants (values known, meanings +/// not; generated from the hardware-validated `g17p_initdata.py` +/// `REGION_C_CONSTANTS`). 12 sparse runs, 180 bytes total. The final run +/// initializes the byte preceding the host power-assert counter and the +/// counter itself to their native cold value of zero. +pub(crate) const REGION_C_RUNS: &[(usize, &[u8])] = &[ + (0x0024, &[0xb8, 0x0b, 0x00, 0x00]), + ( + 0x0030, + &[ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x0054, + &[ + 0xff, 0xff, 0x28, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, + ], + ), + (0x0078, &[0x01, 0x00, 0x00, 0x00]), + ( + 0x0084, + &[ + 0x4e, 0x25, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, + ], + ), + (0x0098, &[0xe8, 0x03, 0x00, 0x00]), + ( + 0x07c4, + &[ + 0xd0, 0x07, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0xcd, 0xcc, 0x4c, 0x3f, 0xcd, 0xcc, + 0x4c, 0x3e, 0x66, 0x66, 0x66, 0x3f, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x80, 0x3e, + 0x9a, 0x99, 0x19, 0x3f, 0x66, 0x66, 0x66, 0x3f, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, + ], + ), + ( + 0x0998, + &[ + 0x28, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x09b8, + &[ + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + ], + ), + (0x0e1c, &[0x00, 0x01, 0x00, 0x00]), + (0x0e38, &[0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00]), + (0x0e50, &[0x00, 0x00, 0x00, 0x00]), +]; + +pub(crate) const BUNDLE_VIEW_RUNS: [&[(usize, &[u8])]; 5] = [ + &[ + (0x1658, &[0xff, 0xff, 0xff, 0xff]), + (0x1670, &[0xff, 0xff, 0xff, 0xff]), + ], + &[ + (0xa18, &[0xff, 0xff, 0xff, 0xff]), + (0xa30, &[0xff, 0xff, 0xff, 0xff]), + ], + &[ + (0xe45, &[0xdc, 0x05, 0x00, 0x00, 0xdc, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f]), + (0xe6c, &[0x01, 0x00, 0x00, 0x00, 0x01]), + (0xe80, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0xea0, &[0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01]), + (0xec0, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0xf00, &[0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f]), + (0x17f8, &[0x71, 0x02]), + (0x1804, &[0x9f, 0x2e, 0x7f, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x54, 0x61, 0x51, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x86, 0x95, 0xa5, 0x3c]), + (0x1821, &[0x38, 0x15, 0x46, 0xdb, 0x0f, 0xa9, 0x40, 0x00, 0x00, 0x00, 0x00, 0x44, 0x73, 0xd6, 0xbd, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03]), + (0x1844, &[0x4e, 0x25]), + (0x1878, &[0xe8, 0x03]), + (0x18b0, &[0x04]), + (0x18c6, &[0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0xcc, 0x40]), + (0x18da, &[0x80, 0x47, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03]), + (0x18fc, &[0x4e, 0x25]), + (0x1908, &[0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x05]), + (0x1968, &[0x5c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0x4c, 0x3f, 0xb6, 0xf3, 0x7d, 0x3f, 0xcd, 0xcc, 0x4c, 0x3e, 0x6f, 0x12, 0x03, 0x3c, 0xd8, 0xd4, 0x69, 0x3f, 0xd8, 0xd4, 0x69, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x42, 0xc3, 0xf5, 0x64, 0x40, 0xc3, 0xf5, 0x64, 0x40, 0x36, 0x94, 0x17, 0x41, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x64]), + (0x19cc, &[0x5c]), + (0x1a00, &[0x64]), + (0x1a20, &[0x01, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x1f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xa0, 0x0f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xd0, 0x07, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x01]), + (0x1aa4, &[0x01]), + (0x1ad0, &[0x01]), + (0x30da, &[0x80, 0x47, 0x00, 0x00, 0x20, 0x42, 0x00, 0x00, 0x7a, 0x44, 0xbe, 0x05]), + (0x30f8, &[0x28]), + (0x3102, &[0xc8, 0x42, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0x4c, 0x3f, 0xcd, 0xcc, 0x4c, 0x3e]), + (0x3150, &[0x2a, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04]), + (0x3176, &[0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x19, 0x04, 0x16, 0x41]), + (0x318a, &[0x80, 0x47, 0x52, 0xb8, 0xa2, 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03]), + (0x31ac, &[0xf8, 0x2a]), + (0x31e0, &[0xe8, 0x03]), + (0x33fc, &[0x06, 0x00, 0x00, 0x42]), + (0x3612, &[0x52, 0xb8, 0x7e, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x0a, 0xd7, 0xa3, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x43]), + (0x3630, &[0x80, 0x47, 0x00, 0x00, 0xc8, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0xba, 0x84, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3b, 0x45, 0xac, 0x26]), + (0x3686, &[0xe8, 0x03]), + (0x36a2, &[0xc8]), + (0x3758, &[0xe8, 0x03]), + (0x376c, &[0x01]), + (0x37a0, &[0x01, 0x01, 0x00, 0x00, 0x00, 0x04]), + (0x37bd, &[0x3f, 0x02, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01]), + (0x37dd, &[0x40, 0x02, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01]), + (0x37fd, &[0x41, 0x02]), + (0x3848, &[0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x25, 0x00, 0x00, 0x4e, 0x25, 0x00, 0x00, 0x4e, 0x25]), + (0x386a, &[0x80, 0x3f, 0x04, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x05]), + (0x3944, &[0x3c]), + (0x3950, &[0xef, 0xee, 0x6e, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x89, 0x88, 0x88, 0x3d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f]), + (0x396e, &[0x80, 0x47, 0x00, 0x00, 0x88, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x15, 0x46, 0x4e, 0x25]), + (0x399c, &[0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x57]), + (0x39c4, &[0xe8, 0x03]), + (0x3a04, &[0x01]), + (0x3ad4, &[0x32, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x8e, 0x63, 0x3f, 0xab, 0xaa, 0x2a, 0x3f, 0x39, 0x8e, 0xe3, 0x3d, 0xab, 0xaa, 0xaa, 0x3e, 0xcd, 0xcc, 0x4c, 0xbf, 0xcd, 0xcc, 0x4c, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x47, 0x00, 0x00, 0xa0, 0xc0, 0x00, 0x00, 0xa0, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x64]), + (0x3b1e, &[0x7a, 0x46, 0x78, 0x05]), + (0x3b2c, &[0x90, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x11]), + (0x3b4a, &[0x80, 0x47]), + (0x3b70, &[0x80, 0x3e, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0xc4, 0x09, 0x00, 0x00, 0x0d, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x32]), + (0x3bba, &[0x80, 0x47]), + (0x3bc8, &[0x28, 0x00, 0x00, 0x00, 0xe8, 0x03]), + ], + &[ + (0x15c5, &[0xdc, 0x05, 0x00, 0x00, 0xdc, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f]), + (0x15ec, &[0x01, 0x00, 0x00, 0x00, 0x01]), + (0x1600, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0x1620, &[0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01]), + (0x1640, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0x1680, &[0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f]), + (0x1f78, &[0x71, 0x02]), + (0x1f84, &[0x9f, 0x2e, 0x7f, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x54, 0x61, 0x51, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x86, 0x95, 0xa5, 0x3c]), + (0x1fa1, &[0x38, 0x15, 0x46, 0xdb, 0x0f, 0xa9, 0x40, 0x00, 0x00, 0x00, 0x00, 0x44, 0x73, 0xd6, 0xbd, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03]), + (0x1fc4, &[0x4e, 0x25]), + (0x1ff8, &[0xe8, 0x03]), + (0x2030, &[0x04]), + (0x2046, &[0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0xcc, 0x40]), + (0x205a, &[0x80, 0x47, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03]), + (0x207c, &[0x4e, 0x25]), + (0x2088, &[0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x05]), + (0x20e8, &[0x5c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0x4c, 0x3f, 0xb6, 0xf3, 0x7d, 0x3f, 0xcd, 0xcc, 0x4c, 0x3e, 0x6f, 0x12, 0x03, 0x3c, 0xd8, 0xd4, 0x69, 0x3f, 0xd8, 0xd4, 0x69, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x42, 0xc3, 0xf5, 0x64, 0x40, 0xc3, 0xf5, 0x64, 0x40, 0x36, 0x94, 0x17, 0x41, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x64]), + (0x214c, &[0x5c]), + (0x2180, &[0x64]), + (0x21a0, &[0x01, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x1f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xa0, 0x0f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xd0, 0x07, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x01]), + (0x2224, &[0x01]), + (0x2250, &[0x01]), + ], + &[ + (0x1545, &[0xdc, 0x05, 0x00, 0x00, 0xdc, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f]), + (0x156c, &[0x01, 0x00, 0x00, 0x00, 0x01]), + (0x1580, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0x15a0, &[0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01]), + (0x15c0, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0x1600, &[0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f]), + (0x1ef8, &[0x71, 0x02]), + (0x1f04, &[0x9f, 0x2e, 0x7f, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x54, 0x61, 0x51, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x86, 0x95, 0xa5, 0x3c]), + (0x1f21, &[0x38, 0x15, 0x46, 0xdb, 0x0f, 0xa9, 0x40, 0x00, 0x00, 0x00, 0x00, 0x44, 0x73, 0xd6, 0xbd, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03]), + (0x1f44, &[0x4e, 0x25]), + (0x1f78, &[0xe8, 0x03]), + (0x1fb0, &[0x04]), + (0x1fc6, &[0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0xcc, 0x40]), + (0x1fda, &[0x80, 0x47, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03]), + (0x1ffc, &[0x4e, 0x25]), + (0x2008, &[0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x05]), + (0x2068, &[0x5c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0x4c, 0x3f, 0xb6, 0xf3, 0x7d, 0x3f, 0xcd, 0xcc, 0x4c, 0x3e, 0x6f, 0x12, 0x03, 0x3c, 0xd8, 0xd4, 0x69, 0x3f, 0xd8, 0xd4, 0x69, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x42, 0xc3, 0xf5, 0x64, 0x40, 0xc3, 0xf5, 0x64, 0x40, 0x36, 0x94, 0x17, 0x41, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x64]), + (0x20cc, &[0x5c]), + (0x2100, &[0x64]), + (0x2120, &[0x01, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x1f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xa0, 0x0f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xd0, 0x07, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x01]), + (0x21a4, &[0x01]), + (0x21d0, &[0x01]), + ], +]; +pub(crate) const BUNDLE_VIEW_U16_REPEATS: &[(usize, usize, usize, bool)] = &[ + (2, 0x2ad7, 512, false), + (3, 0x3257, 148, true), + (4, 0x31d7, 148, true), +]; + +pub(crate) const BUNDLE_STATIC_RUNS: &[(usize, &[u8])] = &[ + (0x80ac, &[0x06]), + (0x80b8, &[0x01]), + (0x8177, &[0xac, 0x0d]), + (0x817b, &[0xe8, 0x03]), + (0x817f, &[0xb8, 0x0b]), + (0x8183, &[0xe8, 0x03]), + (0x8187, &[0xe8, 0x03]), + (0x818b, &[0x64]), + (0x818f, &[0x20]), + (0x81b4, &[0x04]), + (0x81c2, &[0x78, 0x3f]), + (0x81cb, &[0x3d]), + (0x81d2, &[0xa0, 0x40]), + (0x81de, &[0x80, 0x47]), + (0x81e2, &[0x70, 0x41]), + (0x81ec, &[0x84, 0x03]), + (0x81f0, &[0xe8, 0x03]), + (0x81f4, &[0xe8, 0x03]), + (0x8200, &[0xe8, 0x03]), + (0x8234, &[0xe8, 0x03]), + (0x8274, &[0x64]), + (0x8290, &[0x2a, 0x08]), + (0x8298, &[0x7d]), + (0x829c, &[0x01]), + (0x8ee0, &[0xae, 0x18]), + (0x8ee8, &[0x01]), + (0xaeec, &[0xcd, 0xcc, 0xc8, 0x41, 0xcd, 0xcc, 0xc8, 0x41]), + (0xb5e4, &[0x01]), + (0xb5e8, &[0xf4, 0x01]), + (0xb5ec, &[0x06]), + (0xb5f0, &[0xd4, 0x30]), + (0xb5f4, &[0xd4, 0x30]), + (0xb5f8, &[0xd4, 0x30]), + (0xb5fc, &[0xd4, 0x30]), + (0xb600, &[0xd4, 0x30]), + (0xb604, &[0xd4, 0x30]), + (0xb608, &[0xa0, 0x0f]), + (0xb61c, &[0xd4, 0x30]), + (0xb620, &[0xd4, 0x30]), + (0xb624, &[0xd4, 0x30]), + (0xb628, &[0xd4, 0x30]), + (0xb62c, &[0xd4, 0x30]), + (0xb630, &[0x01]), + (0xb7a8, &[0x01]), + (0xb7b4, &[0xcd, 0xcc, 0x4c, 0x42]), + (0xb7f5, &[0x80, 0x06, 0x44]), + (0xb834, &[0x9a, 0x99, 0xc9, 0x41]), + (0xb8e0, &[0x6a, 0x18]), +]; + +pub(crate) const PRIMARY_STATUS_B_CONFIG_RUNS: &[(usize, &[u8])] = &[ + (0x010, &[0x0f]), + (0x017, &[0x3f]), + (0x01a, &[0x88, 0x40, 0x28]), + (0x020, &[0x01]), + (0x0c8, &[0xf8, 0x2a]), + (0x0cc, &[0x40, 0x1f]), + (0x0d8, &[0x52, 0xb8, 0xa2, 0x41, 0x19, 0x04, 0x16, 0x41]), + (0x0f0, &[0xac, 0x26]), + (0x0f4, &[0xc8]), + (0x10a, &[0xc8, 0x42]), + (0x10e, &[0xc8, 0x43, 0xc8]), + (0x128, &[0x01]), + (0x12c, &[0x16, 0x26]), + (0x133, &[0x3f, 0xcd, 0xcc, 0xcc, 0x40]), + (0x14c, &[0x01]), + (0x1c0, &[0x01]), + (0x1c4, &[0x01]), + (0x1c8, &[0x04]), + (0x1cc, &[0x01]), + (0x1d0, &[0x01]), + (0x1d4, &[0x01]), + (0x1d8, &[0x01]), + (0x4e4, &[0x01]), + (0x4e8, &[0xf4, 0x01]), + (0x4ec, &[0x06]), + (0x4f0, &[0xd4, 0x30]), + (0x4f4, &[0xd4, 0x30]), + (0x4f8, &[0xd4, 0x30]), + (0x4fc, &[0xd4, 0x30]), + (0x500, &[0xd4, 0x30]), + (0x504, &[0xd4, 0x30]), + (0x51c, &[0xd4, 0x30]), + (0x520, &[0xd4, 0x30]), + (0x524, &[0xd4, 0x30]), + (0x528, &[0xd4, 0x30]), + (0x52c, &[0xd4, 0x30]), + (0x530, &[0x01]), + (0x58c, &[0x06]), + (0x590, &[0xac, 0x0d]), + (0x594, &[0xe8, 0x03]), + (0x598, &[0xb8, 0x0b]), + (0x59c, &[0x64]), + (0x5cc, &[0x01]), + (0x5d0, &[0x04]), + (0x5dc, &[0x40, 0x1f]), + (0x5e0, &[0xc8]), + (0x5e4, &[0xa0, 0x0f]), + (0x5e8, &[0xc8]), + (0x5ec, &[0xd0, 0x07]), + (0x5f0, &[0xc8]), + (0x5f4, &[0xe8, 0x03]), + (0x5f8, &[0xc8]), + (0x606, &[0x70, 0x41]), + (0x60a, &[0xa0, 0x40, 0x20]), + (0x610, &[0xe8, 0x03]), + (0x620, &[0x84, 0x03]), +]; + +pub(crate) const SECONDARY_ROOT_EXTRA_1_RUNS: &[(usize, &[u8])] = &[ + (0x14, &[0x01]), + (0x2c, &[0x01]), + (0x30, &[0x01]), + (0x40, &[0x01]), + (0x4c, &[0x01]), + (0x50, &[0x6a, 0x18]), +]; + +// ===================================================================================== +// J873 / T8152 / G18G / 26A428 (RTKit_release-3514.0.15) +// ===================================================================================== +// +// Plan M1 steps 1.1-1.7. Everything below is selected by +// `RootProfile::J873G18G26A428` or carries a `G18G_26A428`/`g18g_26a428` name. +// Nothing above changes its output for J700 or J813: the M5 constants stay +// literal, and `g18_initdata_test.rs` proves that the layout functions below, +// run over the 25G76 kext tables, reproduce those constants bit for bit, and +// that every J700/J813 encoder output hash is unchanged. +// +// Evidence tags: +// FW_PROVEN read off armfw_g18g-b000 / armfw1_g18g-b010 (J873 loads the B +// images; ADT iop nub UUIDs match them) +// KEXT_PROVEN read off the 26A428 AGXG18G kext in the kernelcache +// ADT_PROVEN read off the J873 ADT (full38-live-adt.bin) +// INFERRED carried from the measured J813/25G76 reference through a +// proven relocation, or matched by analogy; the value itself is +// not proven for 26A428 +// GUESSED no evidence for the value; a placeholder +// A relocation band's tag qualifies the offset mapping, not the carried value. +// Every INFERRED/GUESSED value is also a row of [`G18G_26A428_UNPROVEN`]. + +/// Root `+0x00`: u16 {0x04c0, 0x83a0, 0x4339, 0x0c8b}. FW_PROVEN: b000 +/// 0x2164c-0x21660 and b010 0x294f0-0x29504 compare root+0 with this word and +/// abort with "Host and FW Interface is mismatched" otherwise. KEXT_PROVEN: +/// AGXArmFirmware::initFirmwareData (0xfffffe0008c31f6c) writes it to both +/// roots. +pub(crate) const G18G_26A428_INITDATA_VERSION_WORD: u64 = 0x0c8b_4339_83a0_04c0; +/// Both roots are 0xc8 bytes. FW_PROVEN: each image copies 0xc8 bytes of its +/// root (b000 0x21664-0x216a8) and requires root+0x2c != 0 (b000 0x216ac). +/// Root field offsets are unchanged from 25G76 (KEXT_PROVEN g18g_key.c:3748). +pub(crate) const G18G_26A428_ROOT_SIZE: usize = 0xc8; + +/// Provenance of one G18G relocation band or unproven field. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum Evidence { + FirmwareProven, + KextProven, + AdtProven, + Inferred, + Guessed, +} + +// --- Kext allocation tables and the packing rule ----------------------------------- + +/// `AGXFirmware::allocFirmwareData` allocation-size table, 24 u64. +/// KEXT_PROVEN: 25G76 __const 0xfffffe0007139820. Index names are the +/// `KX_*` constants below. +pub(crate) const KEXT_ALLOCATION_SIZES_25G76: [usize; 24] = [ + 0x2708, 0x0c18, 0x1048, 0x0e10, 0x6a30, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe440, 0x06a8, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e78, 0x00e4, +]; +/// KEXT_PROVEN: 26A428 __const 0xfffffe000714f0d8. Only entries 0 (hw_data), +/// 4 (perf/policy), 12 (status B), 13 (status config) and 22 (region C) +/// differ from 25G76. +pub(crate) const KEXT_ALLOCATION_SIZES_26A428: [usize; 24] = [ + 0x2848, 0x0c18, 0x1048, 0x0e10, 0x7800, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe730, 0x06d0, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e80, 0x00e4, +]; +pub(crate) const KX_HW_DATA: usize = 0; +pub(crate) const KX_BLOCK_C18: usize = 1; +pub(crate) const KX_BLOCK_1048: usize = 2; +pub(crate) const KX_BLOCK_E10: usize = 3; +pub(crate) const KX_POLICY: usize = 4; +pub(crate) const KX_BLOCK_68: usize = 5; +pub(crate) const KX_BLOCK_800: usize = 6; +pub(crate) const KX_BLOCK_88: usize = 7; +pub(crate) const KX_DM_RING: usize = 8; +pub(crate) const KX_MAIN_CONFIG: usize = 9; +pub(crate) const KX_CONTROL_RING: usize = 10; +pub(crate) const KX_RING_STATE: usize = 11; +pub(crate) const KX_STATUS_B: usize = 12; +pub(crate) const KX_STATUS_CONFIG: usize = 13; +pub(crate) const KX_STATUS_A: usize = 14; +pub(crate) const KX_LOG_STATE: usize = 15; +pub(crate) const KX_RING_4800: usize = 16; +pub(crate) const KX_BUFFER_28800: usize = 17; +pub(crate) const KX_BUFFER_79800: usize = 18; +pub(crate) const KX_RING_9000: usize = 19; +pub(crate) const KX_SECONDARY_EXTRA_1: usize = 20; +pub(crate) const KX_POLICY_STATE: usize = 21; +pub(crate) const KX_REGION_C: usize = 22; +pub(crate) const KX_AON_PTD: usize = 23; +/// `getSizeOfFWBRNTable` returns 0 in both kexts (KEXT_PROVEN +/// 0xfffffe00089ee74c / 0xfffffe0008c507e4). +pub(crate) const KEXT_BRN_TABLE_SIZE: usize = 0; +/// Both shared allocations are rounded to one 16 KiB page (allocateSharedData). +pub(crate) const KEXT_ALLOCATION_PAGE: usize = 0x4000; +pub(crate) const SHARED_ALLOCATION_ENTRIES: usize = 25; +pub(crate) const PRIVATE_ALLOCATION_ENTRIES: usize = 38; +const ROLE_SET_ENTRIES: usize = 10; + +const fn round_up_usize(value: usize, alignment: usize) -> usize { + (value + alignment - 1) & !(alignment - 1) +} + +/// The allocateSharedData packing rule (KEXT_PROVEN 0xfffffe0008c038d8): +/// each next entry starts at `(previous offset + previous size + 0x3f) & !0x3f`. +pub(crate) const fn next_packed_allocation(offset: usize, size: usize) -> usize { + round_up_usize(offset + size, 0x40) +} + +/// Offsets of `sizes` packed by [`next_packed_allocation`], the first at 0. +pub(crate) const fn packed_allocation_offsets(sizes: &[usize; N]) -> [usize; N] { + let mut out = [0usize; N]; + let mut index = 1; + while index < N { + out[index] = next_packed_allocation(out[index - 1], sizes[index - 1]); + index += 1; + } + out +} + +/// The 25 entries of shared allocation #1 in allocation order: hw_data, the +/// 0xc18/0x1048/0xe10 blocks, perf/policy, the 0x68/0x800 blocks, the BRN +/// table, the 0x88 block, twelve data-master rings, then main config and +/// device-control ring for role 0 and for role 1 (KEXT_PROVEN descriptor +/// build, 25G76 g17g_key.c:1224-1310 / 26A428 g18g_key.c:1236-1320). +pub(crate) const fn shared_allocation_sizes(table: &[usize; 24]) -> [usize; SHARED_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; SHARED_ALLOCATION_ENTRIES]; + sizes[0] = table[KX_HW_DATA]; + sizes[1] = table[KX_BLOCK_C18]; + sizes[2] = table[KX_BLOCK_1048]; + sizes[3] = table[KX_BLOCK_E10]; + sizes[4] = table[KX_POLICY]; + sizes[5] = table[KX_BLOCK_68]; + sizes[6] = table[KX_BLOCK_800]; + sizes[7] = KEXT_BRN_TABLE_SIZE; + sizes[8] = table[KX_BLOCK_88]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + sizes[9 + ring] = table[KX_DM_RING]; + ring += 1; + } + sizes[21] = table[KX_MAIN_CONFIG]; + sizes[22] = table[KX_CONTROL_RING]; + sizes[23] = table[KX_MAIN_CONFIG]; + sizes[24] = table[KX_CONTROL_RING]; + sizes +} + +/// The 38 entries of shared allocation #2 (the private cluster): twelve +/// data-master ring states and the role-0 device-control state (0x30 each), +/// status B, status config, the ten-entry per-role set for role 0 and role 1, +/// the role-1 device-control state, the 0x60 secondary-root object and the +/// 8-byte policy state (KEXT_PROVEN 25G76 g17g_key.c:1318-1450 / 26A428 +/// g18g_key.c:1330-1460). +pub(crate) const fn private_allocation_sizes(table: &[usize; 24]) -> [usize; PRIVATE_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; PRIVATE_ALLOCATION_ENTRIES]; + let mut state = 0; + while state <= WORK_CHANNEL_COUNT { + sizes[state] = table[KX_RING_STATE]; + state += 1; + } + sizes[13] = table[KX_STATUS_B]; + sizes[14] = table[KX_STATUS_CONFIG]; + let role_set = [ + table[KX_STATUS_A], table[KX_RING_STATE], table[KX_LOG_STATE], table[KX_RING_STATE], + table[KX_RING_STATE], table[KX_RING_4800], table[KX_BUFFER_28800], + table[KX_BUFFER_79800], table[KX_RING_9000], table[KX_RING_4800], + ]; + let mut role = 0; + while role < 2 { + let mut entry = 0; + while entry < ROLE_SET_ENTRIES { + sizes[15 + role * ROLE_SET_ENTRIES + entry] = role_set[entry]; + entry += 1; + } + role += 1; + } + sizes[35] = table[KX_RING_STATE]; + sizes[36] = table[KX_SECONDARY_EXTRA_1]; + sizes[37] = table[KX_POLICY_STATE]; + sizes +} + +/// Offsets (from the start of the shared allocation #1 backing) of every +/// object in it. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedClusterLayout { + pub(crate) hw_data: usize, + pub(crate) block_c18: usize, + pub(crate) block_1048: usize, + pub(crate) block_e10: usize, + pub(crate) policy: usize, + pub(crate) block_68: usize, + pub(crate) block_800: usize, + pub(crate) brn_table: usize, + pub(crate) block_88: usize, + /// Data-master rings in allocation order (see [`WORK_CHANNEL_DM_INDEX`]). + pub(crate) dm_rings: [usize; WORK_CHANNEL_COUNT], + pub(crate) main_config: [usize; 2], + pub(crate) control_ring: [usize; 2], + pub(crate) end: usize, + pub(crate) total: usize, +} + +/// One role's ten-entry set in the private cluster. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RoleSetLayout { + /// Root `+0xa8` (0x20 bytes). + pub(crate) status_a: usize, + /// Event report state (report channel 13 state 0). + pub(crate) event_state: usize, + /// 0x1b0 firmware-log control block (report channel 13 state 2). + pub(crate) log_state: usize, + /// Trace report state (report channel 14 state 0). + pub(crate) trace_state: usize, + pub(crate) state_30: usize, + /// 256 x 0x48 event ring (report channel 13 state 1). + pub(crate) event_ring: usize, + pub(crate) buffer_28800: usize, + /// 0x79800 buffer (main +0x200 / channel 15 state 0). + pub(crate) buffer_79800: usize, + /// 512 x 0x48 trace ring (report channel 14 state 1). + pub(crate) trace_ring: usize, + pub(crate) buffer_4800: usize, +} + +/// Offsets (from the start of shared allocation #2) of every object in it. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrivateClusterLayout { + pub(crate) dm_ring_states: [usize; WORK_CHANNEL_COUNT], + /// Role-0 device-control state (0x30). + pub(crate) control_state: usize, + /// Primary root `+0xb0`. + pub(crate) status_b: usize, + /// Secondary root `+0xb8`. + pub(crate) status_config: usize, + pub(crate) roles: [RoleSetLayout; 2], + /// Role-1 device-control state (0x30). + pub(crate) secondary_control_state: usize, + /// Secondary root `+0xc0` (0x60 bytes). + pub(crate) secondary_extra_1: usize, + /// 8-byte state named by the policy block. + pub(crate) policy_state: usize, + pub(crate) end: usize, + pub(crate) total: usize, +} + +pub(crate) const fn shared_cluster_layout(table: &[usize; 24]) -> SharedClusterLayout { + let sizes = shared_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_rings = [0usize; WORK_CHANNEL_COUNT]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + dm_rings[ring] = offsets[9 + ring]; + ring += 1; + } + let end = offsets[24] + sizes[24]; + SharedClusterLayout { + hw_data: offsets[0], + block_c18: offsets[1], + block_1048: offsets[2], + block_e10: offsets[3], + policy: offsets[4], + block_68: offsets[5], + block_800: offsets[6], + brn_table: offsets[7], + block_88: offsets[8], + dm_rings, + main_config: [offsets[21], offsets[23]], + control_ring: [offsets[22], offsets[24]], + end, + total: round_up_usize(end, KEXT_ALLOCATION_PAGE), + } +} + +const fn role_set_layout(offsets: &[usize; PRIVATE_ALLOCATION_ENTRIES], first: usize) -> RoleSetLayout { + RoleSetLayout { + status_a: offsets[first], + event_state: offsets[first + 1], + log_state: offsets[first + 2], + trace_state: offsets[first + 3], + state_30: offsets[first + 4], + event_ring: offsets[first + 5], + buffer_28800: offsets[first + 6], + buffer_79800: offsets[first + 7], + trace_ring: offsets[first + 8], + buffer_4800: offsets[first + 9], + } +} + +pub(crate) const fn private_cluster_layout(table: &[usize; 24]) -> PrivateClusterLayout { + let sizes = private_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_ring_states = [0usize; WORK_CHANNEL_COUNT]; + let mut state = 0; + while state < WORK_CHANNEL_COUNT { + dm_ring_states[state] = offsets[state]; + state += 1; + } + let end = offsets[37] + sizes[37]; + PrivateClusterLayout { + dm_ring_states, + control_state: offsets[12], + status_b: offsets[13], + status_config: offsets[14], + roles: [role_set_layout(&offsets, 15), role_set_layout(&offsets, 25)], + secondary_control_state: offsets[35], + secondary_extra_1: offsets[36], + policy_state: offsets[37], + end, + total: round_up_usize(end, KEXT_ALLOCATION_PAGE), + } +} + +/// Data-master allocation index of each channel-table work entry (TA_0, 3D_0, +/// CL_0, TA_1, ...). A ring and its 0x30-byte state share the index, which is +/// why the native state grid [`WORK_STATE_GRID_OFFSETS`] is `index * 0x40`. +/// The host object order of both allocations is unchanged in 26A428 (the ring +/// descriptors are this+0x448.. in 25G76 and this+0x438.. in 26A428, same +/// sequence), so the permutation is INFERRED unchanged for G18G. +pub(crate) const WORK_CHANNEL_DM_INDEX: [usize; WORK_CHANNEL_COUNT] = + [3, 7, 11, 2, 6, 10, 1, 5, 9, 0, 4, 8]; + +/// Work-ring bundle offsets in channel-table order for a shared layout. +pub(crate) const fn work_ring_offsets(layout: &SharedClusterLayout) -> [usize; WORK_CHANNEL_COUNT] { + let mut rings = [0usize; WORK_CHANNEL_COUNT]; + let mut channel = 0; + while channel < WORK_CHANNEL_COUNT { + rings[channel] = layout.dm_rings[WORK_CHANNEL_DM_INDEX[channel]]; + channel += 1; + } + rings +} + +/// The 25G76 layouts. Not used by the M5 encoders above (which keep their +/// literal constants); the regression tests prove they agree. +pub(crate) const M5_25G76_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&KEXT_ALLOCATION_SIZES_25G76); +pub(crate) const M5_25G76_PRIVATE_LAYOUT: PrivateClusterLayout = + private_cluster_layout(&KEXT_ALLOCATION_SIZES_25G76); +/// The 26A428 layouts: hw_data 0x2848 at 0, blocks at 0x2880/0x34c0/0x4540, +/// perf/policy 0x7800 at 0x5380, 0x68/0x800 at 0xcb80/0xcc00, BRN and 0x88 at +/// 0xd400, rings from 0xd4c0, main r0 0x1f4c0, control r0 0x1f980, main r1 +/// 0x23980, control r1 0x23e40; status B 0x340, status config 0xea80, status +/// A 0xf180/0xc3440, tail 0x177700/0x177740/0x1777c0. Totals 0x28000 and +/// 0x178000, as in 25G76. +pub(crate) const G18G_26A428_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&KEXT_ALLOCATION_SIZES_26A428); +pub(crate) const G18G_26A428_PRIVATE_LAYOUT: PrivateClusterLayout = + private_cluster_layout(&KEXT_ALLOCATION_SIZES_26A428); + +pub(crate) const G18G_26A428_SHARED_CLUSTER_SIZE: usize = G18G_26A428_SHARED_LAYOUT.total; +pub(crate) const G18G_26A428_PRIVATE_CLUSTER_SIZE: usize = G18G_26A428_PRIVATE_LAYOUT.total; +pub(crate) const G18G_26A428_HW_DATA_SIZE: usize = KEXT_ALLOCATION_SIZES_26A428[KX_HW_DATA]; +pub(crate) const G18G_26A428_POLICY_SIZE: usize = KEXT_ALLOCATION_SIZES_26A428[KX_POLICY]; +pub(crate) const G18G_26A428_STATUS_B_SIZE: usize = KEXT_ALLOCATION_SIZES_26A428[KX_STATUS_B]; +pub(crate) const G18G_26A428_STATUS_CONFIG_SIZE: usize = KEXT_ALLOCATION_SIZES_26A428[KX_STATUS_CONFIG]; +/// The kext allocates 0xe80 bytes of region C (25G76: 0xe78). The host maps +/// a whole page ([`REGION_C_SIZE`]) for both. +pub(crate) const G18G_26A428_REGION_C_ALLOCATION: usize = KEXT_ALLOCATION_SIZES_26A428[KX_REGION_C]; +pub(crate) const G18G_26A428_AON_PTD_SIZE: usize = KEXT_ALLOCATION_SIZES_26A428[KX_AON_PTD]; +/// Bytes of shared allocation #1 before the BRN table: hw_data plus the six +/// blocks the policy encoder owns (25G76 analog: [`M5_25G76_SHARED_PREFIX_SIZE`]). +pub(crate) const G18G_26A428_SHARED_PREFIX_SIZE: usize = G18G_26A428_SHARED_LAYOUT.brn_table; +/// Main `+0x254` views (KEXT_PROVEN 26A428 initFirmwareSharedData: the FW VAs +/// of the 0xc18, 0x1048, 0xe10, 0x68 and 0x800 blocks, this+0x310..0x338). +pub(crate) const G18G_26A428_BUNDLE_VIEW_OFFSETS: [usize; 5] = [ + G18G_26A428_SHARED_LAYOUT.block_c18, + G18G_26A428_SHARED_LAYOUT.block_1048, + G18G_26A428_SHARED_LAYOUT.block_e10, + G18G_26A428_SHARED_LAYOUT.block_68, + G18G_26A428_SHARED_LAYOUT.block_800, +]; +pub(crate) const G18G_26A428_BUNDLE_VIEW_EXTENTS: [usize; 5] = [ + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_C18], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_1048], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_E10], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_68], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_800], +]; +pub(crate) const G18G_26A428_WORK_RING_BUNDLE_OFFSETS: [usize; WORK_CHANNEL_COUNT] = + work_ring_offsets(&G18G_26A428_SHARED_LAYOUT); +pub(crate) const G18G_26A428_MAIN_BUNDLE_OFFSETS: [usize; 2] = G18G_26A428_SHARED_LAYOUT.main_config; +pub(crate) const G18G_26A428_CONTROL_RING_BUNDLE_OFFSETS: [usize; 2] = + G18G_26A428_SHARED_LAYOUT.control_ring; +/// Role-1 main `+0x481`: FW VA of the perf/policy allocation (KEXT_PROVEN). +pub(crate) const G18G_26A428_MAIN_SECONDARY_POLICY_PTR: usize = 0x481; +/// Role-1 main `+0x300`. INFERRED: the J813 value (26A428 source accel+0x10a74). +pub(crate) const G18G_26A428_MAIN_SECONDARY_KIND: u32 = 0xc; + +/// Private-cluster offsets the resources module and live boot need. +pub(crate) const G18G_26A428_PRIVATE_STATUS_A: [usize; 2] = [ + G18G_26A428_PRIVATE_LAYOUT.roles[0].status_a, + G18G_26A428_PRIVATE_LAYOUT.roles[1].status_a, +]; +/// Compact channel-state grid of each role: role 0 starts at the first ring +/// state; role 1's device-control state is the tail 0x30 object, so its +/// (control-only) grid base is that object minus the grid's control offset. +pub(crate) const G18G_26A428_PRIVATE_STATE_GRIDS: [usize; 2] = [ + G18G_26A428_PRIVATE_LAYOUT.dm_ring_states[0], + G18G_26A428_PRIVATE_LAYOUT.secondary_control_state - WORK_STATE_GRID_OFFSETS[CONTROL_CHANNEL_INDEX], +]; +/// Status config inside the status-B span (primary status B .. status A). +pub(crate) const G18G_26A428_STATUS_CONFIG_IN_STATUS_B: usize = + G18G_26A428_PRIVATE_LAYOUT.status_config - G18G_26A428_PRIVATE_LAYOUT.status_b; +/// Bytes from primary status B to primary status A (the object M5 calls +/// [`PRIMARY_STATUS_B_OBJECT_SIZE`], 0xeb00 there). +pub(crate) const G18G_26A428_STATUS_B_SPAN: usize = + G18G_26A428_PRIVATE_LAYOUT.roles[0].status_a - G18G_26A428_PRIVATE_LAYOUT.status_b; +/// The fwctl ring follows its 0x20-byte state in the ArmFirmware fwctl +/// allocation (KEXT_PROVEN {0x20, 0x1800} entries, 64-byte packing). +pub(crate) const G18G_26A428_FWCTL_RING_OFFSET: usize = 0x40; + +// --- Status B and status config (plan step 1.3) ----------------------------------- + +/// KEXT_PROVEN (25G76 0x48e0/0x48e8 stores moved +0x1e0): FW-control state +/// and ring pointers. +pub(crate) const G18G_26A428_STATUS_B_FWCTL_STATE: usize = 0x4ac0; +pub(crate) const G18G_26A428_STATUS_B_FWCTL_RING: usize = 0x4ac8; +/// FW_PROVEN u64 recovery epoch (25G76 +0x48f0). The recovery routine +/// (b000 0x6ef0) increments it as it halts (0x7fb8-0x7fc4), and posts the +/// value it had before as the type-4 event's first payload word +/// (0x79ac/0x7df0). EP 0x21 type 6 (b000 0x255b0-0x255dc) requests a host +/// recovery only when its low 44 bits equal it. Starts zero. +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_EPOCH: usize = 0x4ad0; +/// FW_PROVEN u32 recovery handshake state (25G76 +0x4900). Firmware writes 1 +/// and spins until the host writes 2 (b000 0x7fb0-0x7fe0), recovers, then +/// writes 3 and spins until the host writes 0 (0x80fc-0x8114). Starts zero. +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_STATE: usize = 0x4ae0; +/// FW_PROVEN u32: nonzero when the host requested the recovery (b000 0x7fa8 +/// stores w25 = block 0x68 +4 != 0; 25G76 +0x4910). A host recovery posts +/// no type-4 event. +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_HOST: usize = 0x4af0; +/// FW_PROVEN: 256 x 8-byte recovery-info entries (bit 0 = valid) and the +/// cause words after them. The routine clears 0x828 bytes from +0x4298 before +/// each firmware-detected recovery (b000 0x7df8-0x7e08) and writes the cause +/// at +0x4a9c..+0x4abf (0x7eb0-0x7ee8); 25G76 +0x40b8..+0x48df. +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_INFO: usize = 0x4298; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_INFO_ENTRIES: usize = 0x100; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_EMITTED: usize = 0x4a98; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_SOURCE_COUNT: usize = 0x4aa0; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_COMMAND: usize = 0x4aa8; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_REASON: usize = 0x4aac; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_DETAIL: usize = 0x4ab0; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_SLOT_MASK: usize = 0x4ab8; +/// FW_PROVEN: the 0x68-byte shared block published at main +0x26c is the +/// recovery request (b000 0x6f18-0x6f44, 0x255c8-0x255dc): +0 dispatcher +/// state, +4 host request, +0xc firmware request, +0x20..+0x4b cause. +pub(crate) const G18G_26A428_RECOVERY_REQUEST_WORDS: usize = 0x68 / 4; +/// The 0x800-byte shared block published at main +0x274: one +/// `{u32 valid, u32 pad, u64 stamp}` last-submitted entry per QID. Native +/// `restartWorkQueue` writes it before resuming a halted firmware, which +/// rebuilds queue progress from it (G17 analysis, g17_resources.rs). +pub(crate) const G18G_26A428_QUEUE_PROGRESS_ENTRIES: usize = 128; +pub(crate) const G18G_26A428_QUEUE_PROGRESS_STRIDE: usize = 0x10; +/// FW_PROVEN: the region-C loader reads it (b000 0x36a0/0x377c); KEXT_PROVEN +/// value 1 (25G76 +0xe434). +pub(crate) const G18G_26A428_STATUS_B_CONFIG_HEADER: usize = 0xe714; +/// KEXT_PROVEN new cross-link pair (g18g_key.c:813-816): u32 0 at +0xe71c and +/// the FW VA of status config +0x6c4 at +0xe720. +pub(crate) const G18G_26A428_STATUS_B_LINK_WORD: usize = 0xe71c; +pub(crate) const G18G_26A428_STATUS_B_LINK_PTR: usize = 0xe720; +/// KEXT_PROVEN: status config +0x6c4 = 1 and +0x6c8 = FW VA of status B +0xe71c. +pub(crate) const G18G_26A428_STATUS_CONFIG_LINK_WORD: usize = 0x6c4; +pub(crate) const G18G_26A428_STATUS_CONFIG_LINK_PTR: usize = 0x6c8; +/// KEXT_PROVEN submit*Channel polls (25G76 +0x670/+0x678/+0x680). Written by +/// firmware at run time; they must start zero. +pub(crate) const G18G_26A428_STATUS_CONFIG_SUBMIT_POLLS: [usize; 3] = [0x68c, 0x694, 0x69c]; + +// --- Relocation bands ------------------------------------------------------------- + +/// Bytes `[start, end)` of a 25G76 object move to `[start + delta, end + delta)` +/// of the 26A428 object. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct LayoutBand { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) delta: isize, + pub(crate) evidence: Evidence, +} + +const fn band(start: usize, end: usize, delta: isize, evidence: Evidence) -> LayoutBand { + LayoutBand { start, end, delta, evidence } +} + +/// The 26A428 offset of 25G76 byte `offset`, or None when no band carries it +/// (the byte was removed or its new place is unknown). +pub(crate) const fn relocate_by_bands(bands: &[LayoutBand], offset: usize) -> Option { + let mut index = 0; + while index < bands.len() { + let band = bands[index]; + if band.start <= offset && offset < band.end { + return Some((offset as isize + band.delta) as usize); + } + index += 1; + } + None +} + +/// hw_data 25G76 -> 26A428. The offsets are FW_PROVEN by the init copier +/// (b000 0x21708-0x21d94 vs G17 0x1fb18-0x20124: every G17 read relocates onto +/// a b000 read, see `g18_initdata_test.rs`) and KEXT_PROVEN by the +/// initFirmwareData store alignment. The carried J813 values are INFERRED. +/// 0x640..0xe88 (the 53-slot 25G76 register map) is re-encoded per G18G slot, +/// not relocated. +pub(crate) const G18G_26A428_HW_DATA_BANDS: &[LayoutBand] = &[ + // Host prefix: range head (KEXT_PROVEN same five qwords), +0x28/+0x30/+0x638 + // qwords and the two 32 x 0x18 accelerator tables at +0x38/+0x338 (same + // geometry in both kexts). Not read by the copier; J813 values carried. + band(0x000, 0x640, 0, Evidence::Inferred), + // u64 FW VA of accel+0x1d40 (26A428 writes it at +0xfa0 = "4000"). + band(0xe88, 0xe90, 0x118, Evidence::KextProven), + // Chip block: the copier's 16-byte read moved 0xe90 -> 0xfb0. + band(0xe90, 0xebc, 0x120, Evidence::FirmwareProven), + // 0xebc: 25G76 u32 0 with no 26A428 counterpart (4 bytes removed). + band(0xec0, 0xf5c, 0x11c, Evidence::FirmwareProven), + band(0xf5c, 0xf60, 0x124, Evidence::KextProven), + // 0xf60..0xf68: 25G76 u64 0 removed; 26A428 adds its own u64 0 at +0x1078. + band(0xf68, 0xf98, 0x11c, Evidence::FirmwareProven), + band(0xf98, 0xfac, 0x120, Evidence::FirmwareProven), + // 0xfac: 25G76 flag (G17 copier read 0x1ff60, J813 value 1) has neither a + // 26A428 store nor a b000 read: dropped. + band(0xfb0, 0x25f8, 0x11c, Evidence::FirmwareProven), + band(0x25f8, 0x26b4, 0x120, Evidence::FirmwareProven), + band(0x26b4, 0x26f0, 0x128, Evidence::FirmwareProven), + band(0x26f0, M5_25G76_HW_DATA_SIZE, 0x12c, Evidence::FirmwareProven), +]; + +/// Perf/policy block (relative to the block) 25G76 -> 26A428, from the +/// AGXFirmware::initPowerAndPerformanceData store alignment (KEXT_PROVEN at +/// the band ends; interior bytes without a matched store are INFERRED). +/// 0x6468..0x651c is deliberately absent: 26A428 inserts 0x680 bytes +/// somewhere between the AON PTD pointer (+0x6460 -> +0x64d8) and the boot +/// timestamp (+0x651c -> +0x6c14), and the J813 words there are dropped. +pub(crate) const G18G_26A428_POLICY_BANDS: &[LayoutBand] = &[ + band(0x0000, 0x009c, 0, Evidence::KextProven), + // The 0x160-byte host-config memmove moved +0xa0 -> +0xb0. + band(0x00a0, 0x0200, 0x10, Evidence::KextProven), + band(0x09b8, 0x295c, 0x10, Evidence::KextProven), + band(0x2960, 0x29c1, 0x29, Evidence::KextProven), + band(0x2a01, 0x2a05, 0x8d, Evidence::KextProven), + band(0x2a08, 0x2e5c, 0x90, Evidence::KextProven), + band(0x2e6c, 0x2e70, 0x9c, Evidence::KextProven), + // this+0x216c -> this+0x211c (the host object moved by -0x50). + band(0x2e78, 0x2e7c, 0x74, Evidence::Inferred), + band(0x2f37, 0x2f43, -0x2b, Evidence::KextProven), + band(0x2f43, 0x2f53, 0x71, Evidence::KextProven), + band(0x2f58, 0x3260, 0x70, Evidence::KextProven), + // Policy state pointer and its neighbours (+0x3c98 -> +0x3d30). + band(0x3c98, 0x3cac, 0x98, Evidence::KextProven), + band(0x5cac, 0x5cb4, 0x98, Evidence::KextProven), + band(0x63a4, 0x63f4, 0xa0, Evidence::KextProven), + band(0x6458, 0x645c, 0x78, Evidence::KextProven), + // AON PTD FW VA (this+0xf78 in 26A428). + band(0x6460, 0x6468, 0x78, Evidence::KextProven), + // Boot mach_absolute_time (+0x651c -> +0x6c14) and what follows it. + band(0x651c, 0x66a8, 0x6f8, Evidence::KextProven), + band(0x6a18, 0x6a1c, 0x710, Evidence::KextProven), +]; + +/// Region C 25G76 -> 26A428. FW_PROVEN: every read of the G17 region-C loader +/// (0x3068-0x330c) relocates onto a b000 loader read (0x3508-0x37c0) under +/// this map, and the only extra b000 read is the new field at +0x38. The kext +/// agrees: 26A428 stores 0x78 at +0x40 (25G76 +0x3c) and a feature bit at +0x38. +pub(crate) const G18G_26A428_REGION_C_BANDS: &[LayoutBand] = &[ + band(0x000, 0x038, 0, Evidence::FirmwareProven), + band(0x038, 0xe78, 4, Evidence::FirmwareProven), +]; + +/// Status config 25G76 -> 26A428 (KEXT_PROVEN initFirmwareSharedData store +/// alignment). The J813 words live in the first, sixth and seventh bands. +pub(crate) const G18G_26A428_STATUS_CONFIG_BANDS: &[LayoutBand] = &[ + band(0x000, 0x1dc, 0, Evidence::KextProven), + band(0x4e0, 0x518, 0xc, Evidence::KextProven), + band(0x518, 0x528, 0x14, Evidence::KextProven), + band(0x530, 0x534, 0xc, Evidence::KextProven), + band(0x554, 0x584, 0xc, Evidence::KextProven), + band(0x584, 0x5a8, 0x1c, Evidence::KextProven), + band(0x5cc, 0x628, 0x18, Evidence::KextProven), +]; + +/// J873 secondary power-control inputs from eight native 26A428 capture views. +/// The original initPowerAndPerformanceData stores at 8c30740..8c30760 +/// copy these u32 fields from accelerator+10c10+92c..948/950. GFX1 then +/// consumes them at b010 1071c onward. Do not inherit the J813 controller +/// settings here. Values and store geometry are qualified; semantic names +/// and any relationship to the outstanding compute stall remain unqualified. +const G18G_26A428_STATUS_CONFIG_POWER_WORDS: &[(usize, u32)] = &[ + (0x1c0, 0), (0x1c4, 1), (0x1c8, 1), (0x1cc, 4), (0x1d0, 1), + (0x1d4, 1), (0x1d8, 1), (0x1dc, 1), (0x1e0, 29000), +]; + +/// Remaining fixed J873/26A428 status/config defaults. These typed scalar +/// corrections and the generated tables below make the complete 0x6d0-byte +/// allocation match all eight pre-publication native capture views. Preserve +/// the exact float bit patterns; individual calibration meanings are not +/// qualified. Never use post-run firmware counters as encoder defaults. +const G18G_26A428_STATUS_CONFIG_NATIVE_WORDS: &[(usize, u32)] = &[ + (0x0d8, 0x41a770a4), (0x0dc, 0x3d20045e), (0x12c, 0x8f16), + (0x4e8, 0x00010001), (0x59c, 1), (0x5a0, 700), (0x5a4, 12), + (0x5a8, 5000), (0x5ac, 2000), (0x5b0, 5000), (0x5b4, 0), + (0x5b8, 0), (0x618, 0), (0x628, 1300), (0x638, 1200), +]; + +fn encode_g18g_26a428_status_config_defaults(out: &mut [u8]) { + for words in [G18G_26A428_STATUS_CONFIG_POWER_WORDS, G18G_26A428_STATUS_CONFIG_NATIVE_WORDS] { + for &(offset, value) in words { + put_u32(out, offset, value); + } + } + // Original host stores at 8c30790..8c30804 copy two 16x16 tables + // from accelerator+111c4/+113c4, then two u16 enables at +4e8/+4ea. + // Native J873 tables have zero first rows/columns and linear remaining + // entries. Generate the values; do not carry the different J813 table. + for row in 0..16 { + for col in 0..16 { + let active = row != 0 && col != 0; + let wide = if active { (col * 0x1000 - 1) as u16 } else { 0 }; + let narrow = if active { (col * 0x10 - 1) as u8 } else { 0 }; + put_u16(out, 0x1e8 + 2 * (row * 16 + col), wide); + out[0x3e8 + row * 16 + col] = narrow; + } + } +} + +/// Copy every non-zero byte of `words` (u32 at `offset - source_base`) to its +/// relocated place in `out`. Bytes without a band are dropped. +fn write_relocated_words( + words: &[(usize, u32)], + source_base: usize, + bands: &[LayoutBand], + out: &mut [u8], +) -> Result<(), InitdataError> { + for &(offset, value) in words { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + if byte == 0 { + continue; + } + let source = offset + .checked_sub(source_base) + .ok_or(InitdataError::BufferSize)? + + index; + if let Some(target) = relocate_by_bands(bands, source) { + *out.get_mut(target).ok_or(InitdataError::BufferSize)? = byte; + } + } + } + Ok(()) +} + +// --- hw_data (plan step 1.4) -------------------------------------------------------- + +/// 60 register slots at `+0x640` (FW_PROVEN b000 0x236d8 walks 53 named slot +/// IDs from DATA 0xfffffc0000078bb8; KEXT_PROVEN the 26A428 copy loop runs +/// over 0x3c slots, 0x648..0xfa8). +pub(crate) const G18G_26A428_HW_REGISTER_MAP_SLOTS: usize = 60; +pub(crate) const G18G_26A428_HW_REGISTER_MAP_END: usize = + HW_REGISTER_MAP_OFFSET + G18G_26A428_HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE; +/// KEXT_PROVEN: initFirmwareData forces this slot's device VA (+0xcb0) to 0 +/// (25G76: slot 36, +0xbe8; FenderRegsDFenderPowerTransG16G in both). +pub(crate) const G18G_26A428_ZEROED_REGISTER_SLOT: usize = 41; +/// FW_PROVEN: the copier's 16-byte read of the chip block moved 0xe90 -> 0xfb0. +pub(crate) const G18G_26A428_HW_CHIP_ID_OFFSET: usize = 0xfb0; +pub(crate) const G18G_26A428_HW_CHIP_ID: u32 = 0x8152; +/// KEXT_PROVEN: initPowerAndPerformanceData stores this+0xf10 (the number of +/// performance states; J813 +0xfc4 = 12 = its gpu-num-perf-states) here. +pub(crate) const G18G_26A428_HW_PERF_STATE_COUNT: usize = 0x10e0; +/// KEXT_PROVEN: 16 frequencies in MHz (accel+0x1b8f8.. / 1e6). +pub(crate) const G18G_26A428_HW_PERF_FREQUENCY: usize = 0x10e4; +/// FW_PROVEN: 16 rows of 0x40 bytes; the b000 loop reads lanes 0 and 1 of +/// each row (0x1124/0x1128 core, 0x1524/0x1528 SRAM; b000 0x218d0-0x219f0). +pub(crate) const G18G_26A428_HW_PERF_CORE_VOLTAGE: usize = 0x1124; +pub(crate) const G18G_26A428_HW_PERF_SRAM_VOLTAGE: usize = 0x1524; +pub(crate) const G18G_26A428_HW_PERF_ROWS: usize = 16; +pub(crate) const G18G_26A428_HW_PERF_ROW_STRIDE: usize = 0x40; +pub(crate) const G18G_26A428_HW_PERF_LANES: usize = 16; +pub(crate) const G18G_26A428_HW_PERF_SCALE: usize = 0x1964; +pub(crate) const G18G_26A428_HW_PERF_RELATIVE_A: usize = 0x19e4; +pub(crate) const G18G_26A428_HW_PERF_RELATIVE_B: usize = 0x1a24; +pub(crate) const G18G_26A428_HW_PERF_INDEX_MAP: usize = 0x1ae4; +/// FW_PROVEN: four 0x20-byte KSM completion descriptors (b000 0xf57c `mov +/// w9,#0x2734`), firmware cursor at +0x1c of each (b000 0x1cbc0..0x1cc48). +pub(crate) const G18G_26A428_KSM_COMPLETION_TABLE_OFFSET: usize = 0x2734; +pub(crate) const G18G_26A428_KSM_COMPLETION_CURSOR: usize = 0x1c; +/// FW_PROVEN: QoS pointer (b000 0xa0f0; firmware dereferences QoS+0xe00). +pub(crate) const G18G_26A428_QOS_POINTER_OFFSET: usize = 0x27cc; +/// KEXT_PROVEN new constant (26A428 initFirmwareData `hw+0x2714 = 0x96`). +/// FW_PROVEN numeric use: b000 0x23220 compares it with -1 and scales it by 24; +/// 0xbb90 logs it. (It is not the 25G76 +0x25f4 flag, which moves to +0x2710.) +pub(crate) const G18G_26A428_HW_2714_OFFSET: usize = 0x2714; +pub(crate) const G18G_26A428_HW_2714_VALUE: u32 = 0x96; +/// Fields that exist only in 26A428 (KEXT_PROVEN stores), with their size. +/// +0x2714 is KEXT_PROVEN. Captured scalars below override the zero defaults; +/// the native-owned pointer +0x27d4 remains unresolved and is not copied. +pub(crate) const G18G_26A428_HW_NEW_FIELDS: [(usize, usize); 11] = [ + (0x0fa8, 8), (0x1078, 8), (0x10b4, 4), (0x2714, 4), (0x27d4, 8), (0x2818, 4), + (0x281e, 4), (0x2822, 8), (0x2832, 4), (0x283a, 4), (0x283e, 4), +]; + +/// Stable scalar values from both roles in macOS 26A428 captures 051/053/073/075. +/// Replace the J813 defaults and unknown zero enables only at these offsets. +/// Their values are observed; their complete semantics remain unqualified. +/// Failure boundaries: no pointers, timing values, voltage tables or register +/// writes are copied. audit-g18-hwdata-scalars.py checks all eight snapshots, +/// the float32 ladders, and the full opening records before a hardware trial. +pub(crate) const G18G_26A428_HW_CAPTURED_WORDS: [(usize, u32); 17] = [ + (0x0fb4, 1), (0x10d4, 0x1b), (0x10d8, 4), (0x10dc, 12), + (0x283a, 1), (0x283e, 1), + (0x0fa8, 0x8000), (0x0fac, 0), (0x0fb8, 1), + (0x1044, 12), (0x10a4, 12), (0x2660, 0), (0x2684, 12), + (0x2710, 0), (0x2720, 0), (0x2724, 1), (0x281e, 1), +]; +/// The adjacent +0x281e word is unaligned; +0x281c is only a halfword. +pub(crate) const G18G_26A428_HW_CAPTURED_HALFWORDS: [(usize, u16); 1] = [(0x281c, 1)]; + +/// J873 ADT /arm-io/sgx perf-states: (MHz, core mV), 14 entries including +/// state 0. ADT_PROVEN. +pub(crate) const J873_GPU_PERF_STATES: [(u32, u32); 14] = [ + (0, 125), (502, 675), (669, 700), (834, 720), (1002, 755), (1090, 770), (1178, 795), + (1266, 810), (1354, 840), (1442, 875), (1530, 915), (1620, 960), (1680, 975), (1735, 990), +]; +/// J873 ADT perf-states-sram voltages (mV). ADT_PROVEN. +pub(crate) const J873_GPU_PERF_STATES_SRAM_MV: [u32; 14] = + [775, 775, 775, 775, 775, 775, 795, 810, 840, 875, 915, 960, 975, 990]; +/// J873 ADT gpu-num-perf-states. ADT_PROVEN. +pub(crate) const J873_GPU_NUM_PERF_STATES: u32 = 13; +/// INFERRED: the J813 per-state scale word (1.02f), one per state. +const G18G_26A428_PERF_SCALE_VALUE: u32 = 0x3f82_8f5c; +/// Native 26A428 relative-performance ladders, identical in both roles of +/// captures 051/053/073/075. These are IEEE754 float32 bit patterns, not u32 +/// percentages. Store the bits without floating-point operations in kernel +/// context. All values are finite, monotonic, and bounded by 0..100; the +/// physical calibration formula is not yet derived. Voltage/frequency tables +/// above remain the independently verified J873 ADT values. +const G18G_26A428_PERF_RELATIVE_A: [u32; 14] = + [0, 1100735714, 1103989076, 1107351648, 1109719030, 1110992894, 1112721914, + 1114199739, 1116076971, 1117492892, 1119211535, 1120006268, 1120204862, 1120403456]; +const G18G_26A428_PERF_RELATIVE_B: [u32; 14] = + [0, 0, 1096332557, 1104636123, 1109537981, 1111408920, 1113279858, + 1115150797, 1116353300, 1117288769, 1118224238, 1119180968, 1119818789, 1120403456]; + +/// The 26A428 slot of each 25G76 register slot (FW_PROVEN name tables: G17 +/// DATA 0xfffffc0000073488, 47 rows; b000 DATA 0xfffffc0000078bb8, 53 rows). +/// None: 25G76 slot 8 (StreamingCodecRegisters) has no 26A428 slot. Unnamed +/// slots 17 and 21 keep their numbers; 25 -> 28 and 44/45/46 -> 50/51/52 are +/// INFERRED by position (44 -> 50 is confirmed by the identical die-1 PIO +/// record 0xd24000/0x100 in both kexts). +pub(crate) const G18G_26A428_SLOT_FROM_25G76: [Option; HW_REGISTER_MAP_SLOTS] = [ + Some(0), Some(1), Some(2), Some(3), Some(4), Some(5), Some(6), Some(7), None, Some(9), + Some(10), Some(11), Some(12), Some(13), Some(14), Some(15), Some(16), Some(17), Some(18), + Some(19), Some(20), Some(21), Some(22), Some(23), Some(24), Some(28), Some(31), Some(32), + Some(33), Some(34), Some(35), Some(36), Some(37), Some(38), Some(39), Some(40), Some(41), + Some(42), Some(43), Some(44), Some(45), Some(46), Some(47), Some(48), Some(50), Some(51), + Some(52), Some(53), Some(54), Some(55), Some(56), Some(57), Some(58), +]; + +/// J873 moved the always-on blocks the 25G76 map names (AIC, MCC telemetry, +/// PMGR event log, AON PTD) down by this much (ADT_PROVEN); the SGX and GPU +/// PMGR banks did not move. +pub(crate) const J873_AON_REBASE: u64 = 0x8000_0000; + +/// A register window before its device VA is assigned. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWindowSpec { + pub(crate) slot: usize, + pub(crate) phys: u64, + /// Instance count x instance size (hw_data slot +0x10). + pub(crate) size_10: u32, + /// Instance size (slot +0x14). + pub(crate) size_14: u32, + /// Offset from the GPU base for PIO-table windows (slot +0x18), else 0. + pub(crate) unk_18: u64, + /// accelerator entry flags & 2 (slot +0x20). + pub(crate) flag: u32, +} + +const fn spec(slot: usize, phys: u64, size: u32, unk_18: u64, flag: u32) -> RegisterWindowSpec { + RegisterWindowSpec { slot, phys, size_10: size, size_14: size, unk_18, flag } +} + +/// Assign firmware device VAs in slot order, from the register aperture base: +/// each window starts at the next 32 KiB boundary at least one 16 KiB guard +/// page past the previous window, plus its physical page offset. This rule +/// reproduces all 22 native J813/25G76 device VAs exactly (regression test); +/// for G18G the VAs are host-chosen (INFERRED: same rule), since firmware +/// reaches its registers only through this table. +pub(crate) const fn allocate_register_window_vas( + specs: &[RegisterWindowSpec; N], +) -> [M5RegisterWindow; N] { + let mask = UAT_PAGE_SIZE as u64 - 1; + let mut out = [M5RegisterWindow { + slot: 0, phys: 0, device_va: 0, size_10: 0, size_14: 0, unk_18: 0, flag: 0, + }; N]; + let mut next: u64 = 0; + let mut index = 0; + while index < N { + let window = specs[index]; + let va = next + (window.phys & mask); + out[index] = M5RegisterWindow { + slot: window.slot, + phys: window.phys, + device_va: HW_REGISTER_APERTURE.start + va, + size_10: window.size_10, + size_14: window.size_14, + unk_18: window.unk_18, + flag: window.flag, + }; + let end = (va + window.size_10 as u64 + mask) & !mask; + next = (end + UAT_PAGE_SIZE as u64 + 0x7fff) & !0x7fff; + index += 1; + } + out +} + +/// The 22 mapped G18G windows, in slot order. Sources (26A428 kext): +/// - AcceleratorG::configureDevice (0xfffffe0008c490ac): slot 3 MCache +/// 0x220104000 (0x140fb rounded, count = popcount(active-mcc-bit-vector), +/// absent on J873 -> 1), 9/10 telemetry at AON+0x3d0000/+0x3c0000, 14 PMS at +/// AON+0x280000 size 0x14f54 (25G76: 0x8000), 15 AONPTDSpace AON+0xc840000 +/// 0x24000, 29 DPEP 0x580001800/0x27f8 and 30 ADCLK 0x580ec2200/0xfc (both +/// new), 46 ThrtlStatCntrs GPU+0xe33800. Slots 10/14/29/30 get flags 0. +/// - PI_400 Accelerator::configureDevice (0xfffffe0008c45428): slot 22 +/// GPU+0x1000000/0x8000 and slot 44 MTR GPU+0xde0000, one 0x15000 instance +/// (25G76: GPU+0xe08000, two 0x8000 instances). +/// - AGFPIOGFXHALRelativeOffsetTableG18G (0xfffffe0007152228, 12 records): +/// slot 17 GPU+0 size 0x36000 (25G76 0x21400), Fender slots 31..37/40 at the +/// 25G76 offsets, new 35 (KSMFE, 0xd28000) and 49 (FenderRegsHWL, 0xe18000); +/// +0x18 is the table offset. 25G76 slot 50 (DpwrMXU, 0x14400) is gone. +/// - AGXAccelerator::configureDevice (base): flags 2 for the named slots, +/// 0 for 49; slot 0 {flags 2, count 1} and the AIC SW-interrupt page (the +/// ADT meta-sw-interrupt, 0x301014048 on J873: ADT_PROVEN). +/// - Slot 24 (DVFMMgpu) is absent. Both roles in native 26A428 initialization +/// captures 051/053/073 leave all 0x28 bytes zero. Do not carry the J813 +/// 0x580ec0000 window into this profile or allocate firmware VA space for it. +/// The GPU base is the SGX reg 0x580000000 (unchanged, ADT_PROVEN); the AON +/// base moved 0x380000000 -> 0x300000000 ([`J873_AON_REBASE`]). +pub(crate) const G18G_26A428_REGISTER_WINDOW_SPECS: [RegisterWindowSpec; 21] = [ + spec(0, 0x3_0101_4000, 0x4000, 0, 2), + spec(3, 0x2_2010_4000, 0x18000, 0, 2), + spec(9, 0x3_003d_0000, 0x4000, 0, 2), + spec(10, 0x3_003c_0000, 0x4000, 0, 0), + spec(14, 0x3_0028_0000, 0x14f54, 0, 0), + spec(15, 0x3_0c84_0000, 0x24000, 0, 2), + spec(17, 0x5_8000_0000, 0x36000, 0, 2), + spec(22, 0x5_8100_0000, 0x8000, 0, 2), + spec(29, 0x5_8000_1800, 0x27f8, 0, 0), + spec(30, 0x5_80ec_2200, 0xfc, 0, 0), + spec(31, 0x5_80d0_4000, 0x8000, 0xd0_4000, 2), + spec(32, 0x5_80d0_d000, 0x1000, 0xd0_d000, 2), + spec(33, 0x5_80d5_0000, 0x10000, 0xd5_0000, 2), + spec(34, 0x5_80d1_0000, 0x4000, 0xd1_0000, 2), + spec(35, 0x5_80d2_8000, 0x4000, 0xd2_8000, 2), + spec(36, 0x5_80d4_0000, 0x4000, 0xd4_0000, 2), + spec(37, 0x5_80d6_0000, 0x8000, 0xd6_0000, 2), + spec(40, 0x5_80e0_0000, 0x4000, 0xe0_0000, 2), + spec(44, 0x5_80de_0000, 0x15000, 0, 2), + spec(46, 0x5_80e3_3800, 0x4000, 0, 2), + spec(49, 0x5_80e1_8000, 0x4000, 0xe1_8000, 0), +]; +pub(crate) const G18G_26A428_REGISTER_WINDOWS: [M5RegisterWindow; 21] = + allocate_register_window_vas(&G18G_26A428_REGISTER_WINDOW_SPECS); +/// Slots whose base-configured flags are 2 but which receive no physical +/// window (KEXT_PROVEN by decoding AGXAccelerator::configureDevice; the same +/// decode of the 25G76 function reproduces the J813 list exactly). Slot 47 +/// (AVMR) lost its G-variant window in 26A428; slot 8 keeps a flag although +/// firmware no longer walks it. +pub(crate) const G18G_26A428_REGISTER_FLAG_ONLY_SLOTS: [usize; 20] = + [2, 5, 6, 7, 8, 25, 26, 27, 38, 39, 42, 43, 45, 47, 48, 52, 53, 54, 55, 58]; + +/// Encode the 60-slot, 0x960-byte G18G register map. Construction only: +/// page tables for these windows are the caller's (see g18_resources.rs). +pub(crate) fn encode_g18g_26a428_register_map(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != G18G_26A428_HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for window in G18G_26A428_REGISTER_WINDOWS { + if window.slot >= G18G_26A428_HW_REGISTER_MAP_SLOTS + || window.slot == G18G_26A428_ZEROED_REGISTER_SLOT + || !HW_REGISTER_APERTURE.contains(&window.device_va) + { + return Err(InitdataError::RegisterWindow); + } + let base = window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, window.phys); + put_u64(out, base + 0x08, window.device_va); + put_u32(out, base + 0x10, window.size_10); + put_u32(out, base + 0x14, window.size_14); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + for slot in G18G_26A428_REGISTER_FLAG_ONLY_SLOTS { + put_u32(out, slot * HW_REGISTER_MAP_ENTRY_SIZE + 0x20, 2); + } + Ok(()) +} + +/// Reconstruct the G18G hw_data object (0x2848 bytes) with caller-owned +/// completion-queue and QoS bindings. The J813 platform words are relocated by +/// [`G18G_26A428_HW_DATA_BANDS`]; the register map, chip id, perf tables, +/// completion table and QoS pointer are G18G encodings. No MMIO access. +pub(crate) fn encode_g18g_26a428_hw_data( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + qos_high: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != G18G_26A428_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(qos_high)?; + let qos_end = qos_high + .checked_add(G17P_QOS_RESOURCE_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(qos_end - 1)?; + let mut completion = [0; M5_25G76_COMPLETION_QUEUE_COUNT * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE]; + // INFERRED descriptor format: the J813 one (26A428 fills it through an + // accelerator virtual that was not decoded; the location is FW_PROVEN). + encode_m5_25g76_completion_table(aliases, &mut completion)?; + for alias in aliases { + if qos_high < alias.high + M5_25G76_COMPLETION_BACKING_SIZE as u64 && alias.high < qos_end { + return Err(InitdataError::PairAliasedObject); + } + } + + out.fill(0); + write_relocated_words(M5_25G76_HW_PLATFORM_WORDS, 0, G18G_26A428_HW_DATA_BANDS, out)?; + encode_g18g_26a428_register_map(&mut out[HW_REGISTER_MAP_OFFSET..G18G_26A428_HW_REGISTER_MAP_END])?; + put_u32(out, G18G_26A428_HW_CHIP_ID_OFFSET, G18G_26A428_HW_CHIP_ID); + // INFERRED: this+0xf10 is the number of performance states (J813 12 = + // its gpu-num-perf-states); J873 ADT gpu-num-perf-states is 13. + put_u32(out, G18G_26A428_HW_PERF_STATE_COUNT, J873_GPU_NUM_PERF_STATES); + put_u32(out, G18G_26A428_HW_2714_OFFSET, G18G_26A428_HW_2714_VALUE); + for (offset, value) in G18G_26A428_HW_CAPTURED_WORDS { + put_u32(out, offset, value); + } + for (offset, value) in G18G_26A428_HW_CAPTURED_HALFWORDS { + put_u16(out, offset, value); + } + + // Performance ladder from the J873 ADT (values ADT_PROVEN, locations + // FW_PROVEN/KEXT_PROVEN). Each voltage fills all 16 lanes of its row as on + // J813 (INFERRED); b000 reads lanes 0 and 1. Rows 14 and 15 stay zero. + for (state, &(frequency, core)) in J873_GPU_PERF_STATES.iter().enumerate() { + put_u32(out, G18G_26A428_HW_PERF_FREQUENCY + state * 4, frequency); + put_u32(out, G18G_26A428_HW_PERF_SCALE + state * 4, G18G_26A428_PERF_SCALE_VALUE); + put_u32(out, G18G_26A428_HW_PERF_RELATIVE_A + state * 4, G18G_26A428_PERF_RELATIVE_A[state]); + put_u32(out, G18G_26A428_HW_PERF_RELATIVE_B + state * 4, G18G_26A428_PERF_RELATIVE_B[state]); + for lane in 0..G18G_26A428_HW_PERF_LANES { + let row = state * G18G_26A428_HW_PERF_ROW_STRIDE + lane * 4; + put_u32(out, G18G_26A428_HW_PERF_CORE_VOLTAGE + row, core); + put_u32(out, G18G_26A428_HW_PERF_SRAM_VOLTAGE + row, J873_GPU_PERF_STATES_SRAM_MV[state]); + } + } + // INFERRED: the J813 identity index map over all 16 rows. + for index in 0..G18G_26A428_HW_PERF_ROWS { + put_u32(out, G18G_26A428_HW_PERF_INDEX_MAP + index * 4, index as u32); + } + let table = G18G_26A428_KSM_COMPLETION_TABLE_OFFSET; + out[table..table + completion.len()].copy_from_slice(&completion); + put_u64(out, G18G_26A428_QOS_POINTER_OFFSET, qos_high); + Ok(()) +} + +// --- Perf/policy block and the blocks around it (plan step 1.6) ------------------- + +/// KEXT_PROVEN (26A428 initPowerAndPerformanceData 0xfffffe0008c0807c): the +/// host bzeros 0x7800 bytes, memmoves a 0x160-byte host config to +0xb0 and +/// stores mach_absolute_time() at +0x6c14 (25G76: +0x651c). +pub(crate) const G18G_26A428_POLICY_BOOT_TIMESTAMP: usize = 0x6c14; +/// KEXT_PROVEN: FW VA of the 8-byte private-cluster tail (this+0x3a0) at +/// +0x3d30 (25G76 +0x3c98). +pub(crate) const G18G_26A428_POLICY_STATE_PTR: usize = 0x3d30; +/// KEXT_PROVEN: FW VA of the AON PTD (this+0xf78) at +0x64d8 (25G76 +0x6460). +pub(crate) const G18G_26A428_POLICY_AON_PTD_PTR: usize = 0x64d8; +/// Policy-relative place of the J813 512-entry u16 0x7fff table that +/// [`encode_m5_25g76_bundle_policy`] writes at bundle +0x6ed7. +const M5_25G76_POLICY_U16_TABLE: usize = 0x1c97; +const M5_25G76_POLICY_U16_TABLE_ENTRIES: usize = 512; +/// ADT tunables at the places the 26A428 kext stores them. KEXT_PROVEN +/// locations: +0xb38 = accel+0x11080 (gpu-perf-tgt-utilization), +0xb6c/+0xb70 +/// = getPerformanceControllerIntegralGain(0/1) = accel+0x110a8/0x110ac +/// (gpu-perf-integral-gain/2), +0xb7c/+0xb80 = ProportionalGain(0/1) = +/// accel+0x110b8/0x110bc; +0x22d4/+0x2928/+0x2e1c = this+0xf10 * 100. +/// Values ADT_PROVEN (J873 sgx). +pub(crate) const G18G_26A428_POLICY_ADT_WORDS: [(usize, u32); 8] = [ + (0x0b38, 75), + (0x0b6c, 0x3f92_e148), + (0x0b70, 0x3f92_e148), + (0x0b7c, 0x4092_e148), + (0x0b80, 0x4092_e148), + (0x22d4, J873_GPU_NUM_PERF_STATES * 100), + (0x2928, J873_GPU_NUM_PERF_STATES * 100), + (0x2e1c, J873_GPU_NUM_PERF_STATES * 100), +]; +/// Region-C ADT tunables. KEXT_PROVEN: +0x7c8 = accel+0x106b0 +/// (gpu-idleoff-standby-timer; J813 +0x7c4 carried its ADT 1250) and +0x7e4 = +/// accel+0x106cc (gpu-idleoff-min_conf); ADT_PROVEN values 1500 and 0.8f. +/// +0x9c = this+0xf14 * 100 (25G76 +0x98 = this+0xf40 * 100 = 1200 on J813): +/// INFERRED to be the perf-state count again. +pub(crate) const G18G_26A428_REGION_C_ADT_WORDS: [(usize, u32); 3] = [ + (0x7c8, 1500), + (0x7e4, 0x3f4c_cccd), + (0x09c, J873_GPU_NUM_PERF_STATES * 100), +]; + +/// J873/26A428 pre-handoff policy corrections relative to the relocated +/// J813 baseline. Both roles in four cold captures (051/053/073/075) agree +/// byte-for-byte except the separately encoded boot timestamp. Values are +/// captured calibration/configuration, not inferred field types or MHz. +/// No owned address or timestamp is part of this table. The complete +/// production encoder is compared by a host startup +/// comparison; allocation and execution require live proof. +pub(crate) const G18G_26A428_POLICY_CAPTURED_WORDS: [(usize, u32); 65] = [ + (0x0048, 0x00000514), (0x0088, 0x00000514), + (0x009c, 1), (0x00a0, 1), (0x00a4, 1), (0x00a8, 1), + (0x0104, 0x3f828f5c), (0x09f0, 0x470f1600), + (0x09fc, 0xbd115e00), (0x0a04, 0x00000514), + (0x0a08, 0x00000514), (0x0a14, 0x00008f16), + (0x0a48, 0x00000514), (0x0abc, 0x00000514), + (0x0acc, 0x00008f16), (0x0b4c, 1), + (0x0b84, 0x414a1af3), (0x0b8c, 0x00000514), (0x0ca0, 0), + (0x22ac, 0x42c80000), (0x22b0, 0x44a28000), + (0x22b4, 0x000006c7), (0x22c8, 0x00000064), + (0x2320, 0xff003f80), (0x234c, 0x3d20045e), + (0x235c, 0x41a770a4), (0x236c, 0x00000514), + (0x2370, 0x00000514), (0x23b0, 0x00000514), + (0x25c8, 0x00000020), (0x2810, 0x05140000), + (0x2854, 0x05140000), (0x298c, 0x00040000), + (0x2a88, 0x71480000), (0x2a90, 0x00640000), + (0x2aa0, 0x00008f16), (0x2aa4, 0x00008f16), + (0x2aa8, 0x00008f16), (0x2bd0, 0x00000514), + (0x2bd4, 0x00000514), (0x2bdc, 0x470f1600), + (0x2be0, 0x00008f16), (0x2c14, 0x00000514), + (0x2d60, 0x00000514), (0x2f08, 0x0000000c), + (0x2f0c, 0x00001388), (0x2f10, 0x000007d0), + (0x2f14, 0x00001388), (0x2fb4, 0x00000514), + (0x2fb8, 0x00000514), (0x2fc8, 0), + (0x301c, 0x000004b0), (0x3020, 0x00000514), + (0x3024, 0x00000514), (0x3030, 0x00000514), + (0x3064, 0x00000514), (0x30ac, 0), + (0x30b0, 0x42c80000), (0x30c0, 0xff003f80), + (0x3d38, 0xff003f80), (0x64e0, 0x67305452), + (0x6520, 1), (0x6c6c, 0x43366667), + (0x6cac, 0x44e5c000), (0x6cec, 0x42f00000), +]; + +/// Stable region-C inputs from the same eight initialization snapshots. +/// +0x38 is the new 26A428 feature bit, no longer a guessed zero. +pub(crate) const G18G_26A428_REGION_C_CAPTURED_WORDS: [(usize, u32); 7] = [ + (0x0038, 1), (0x0088, 0x00008f16), (0x008c, 0x00000514), + (0x0090, 0x00000514), (0x09b0, 0x00000200), + (0x09dc, 0x00000032), (0x0e0c, 0x000000fa), +]; + +/// Initialize the six allocations following G18G hw_data (0x2880..0xd400) +/// from the J813 policy words relocated by [`G18G_26A428_POLICY_BANDS`], plus +/// the owned pointers, the boot timestamp and the J873 ADT tunables. Preserves +/// hw_data and everything from the BRN table on. No firmware is started. +pub(crate) fn encode_g18g_26a428_bundle_policy( + bindings: M5PolicyBindings, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = G18G_26A428_SHARED_LAYOUT; + if out.len() < G18G_26A428_SHARED_PREFIX_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(bindings.state)?; + required_address(bindings.aon_ptd)?; + let state_end = bindings.state.checked_add(STATUS_BLOCK_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + let aon_end = bindings.aon_ptd.checked_add(G18G_26A428_AON_PTD_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(state_end - 1)?; + required_address(aon_end - 1)?; + if bindings.state < aon_end && bindings.aon_ptd < state_end { + return Err(InitdataError::PairAliasedObject); + } + out[layout.block_c18..G18G_26A428_SHARED_PREFIX_SIZE].fill(0); + let m5 = M5_25G76_SHARED_LAYOUT; + let m5_policy_end = m5.policy + KEXT_ALLOCATION_SIZES_25G76[KX_POLICY]; + for &(offset, value) in M5_25G76_POLICY_WORDS { + if (m5.block_1048..m5.block_e10).contains(&offset) { + // KEXT_PROVEN (R_1048 stores identical in both kexts): the two + // 0xffffffff words at +0xa18/+0xa30 of the 0x1048 block. + put_u32(out, layout.block_1048 + (offset - m5.block_1048), value); + } else if !(m5.policy..m5_policy_end).contains(&offset) { + return Err(InitdataError::BufferSize); + } + } + let policy = &mut out[layout.policy..layout.policy + G18G_26A428_POLICY_SIZE]; + for &(offset, value) in M5_25G76_POLICY_WORDS { + if !(m5.policy..m5_policy_end).contains(&offset) { + continue; + } + write_relocated_words(&[(offset, value)], m5.policy, G18G_26A428_POLICY_BANDS, policy)?; + } + // The 512 x 0x7fff u16 table (J813 bundle +0x6ed7 = policy +0x1c97) + // moves with its band (INFERRED: no store is matched inside it). + for index in 0..M5_25G76_POLICY_U16_TABLE_ENTRIES { + for (byte_index, byte) in 0x7fffu16.to_le_bytes().into_iter().enumerate() { + let source = M5_25G76_POLICY_U16_TABLE + index * 2 + byte_index; + let target = relocate_by_bands(G18G_26A428_POLICY_BANDS, source) + .ok_or(InitdataError::BufferSize)?; + *policy.get_mut(target).ok_or(InitdataError::BufferSize)? = byte; + } + } + put_u64(policy, G18G_26A428_POLICY_STATE_PTR, bindings.state); + put_u64(policy, G18G_26A428_POLICY_AON_PTD_PTR, bindings.aon_ptd); + put_u64(policy, G18G_26A428_POLICY_BOOT_TIMESTAMP, bindings.boot_timestamp); + for (offset, value) in G18G_26A428_POLICY_ADT_WORDS { + put_u32(policy, offset, value); + } + for (offset, value) in G18G_26A428_POLICY_CAPTURED_WORDS { + put_u32(policy, offset, value); + } + Ok(()) +} + +// --- Region C (plan step 1.5) ------------------------------------------------------- + +/// Encode the G18G region C: the J813 bytes, the 25G76 fields from +0x38 on +/// moved up by 4 (FW_PROVEN), the J873 ADT tunables and stable native J873 +/// inputs. The new +0x38 feature bit (accel+0x6b9 bit 2) is captured as one. +pub(crate) fn encode_g18g_26a428_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + encode_region_c_for_profile(RootProfile::J813G17G25G76, out)?; + // The J813 object never reaches its 0xe78 end; shifting in place keeps the + // tail zero and needs no second 4 KiB buffer on the kernel stack. + let shifted = KEXT_ALLOCATION_SIZES_25G76[KX_REGION_C]; + if out[shifted..].iter().any(|byte| *byte != 0) { + return Err(InitdataError::BufferSize); + } + out.copy_within(0x38..shifted, 0x3c); + out[0x38..0x3c].fill(0); + for (offset, value) in G18G_26A428_REGION_C_ADT_WORDS { + put_u32(out, offset, value); + } + for (offset, value) in G18G_26A428_REGION_C_CAPTURED_WORDS { + put_u32(out, offset, value); + } + // Idle power-off policy word (Region C +0x30). The firmware loads + // it on every cold wake, which undoes the IdlePowerOff(0) sent once at boot. + // 0 keeps the scheduler idle event from powering the cores down between + // jobs. Measured on the M6: 8.6 percent less time per op. + put_u32(out, 0x30, 0); + Ok(()) +} + +// --- Private cluster: status B, status config, status A (plan step 1.3) ---------- + +/// Encode the G18G primary status-B span (status B, status config and the gap +/// to status A; [`G18G_26A428_STATUS_B_SPAN`] bytes) before either role is +/// started. `status_b` is the FW VA of the span, for the new cross-links. +pub(crate) fn encode_g18g_26a428_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + status_b: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != G18G_26A428_STATUS_B_SPAN { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + required_address(status_b)?; + let status_b_end = status_b + .checked_add(G18G_26A428_STATUS_B_SPAN as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(status_b_end - 1)?; + let config = G18G_26A428_STATUS_CONFIG_IN_STATUS_B; + out.fill(0); + // +0x04 = 1: INFERRED from J813 (26A428 stores bit 2 of accel+0x6b0 there). + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, G18G_26A428_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, G18G_26A428_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, G18G_26A428_STATUS_B_CONFIG_HEADER, 1); + put_u32(out, G18G_26A428_STATUS_B_LINK_WORD, 0); + put_u64(out, G18G_26A428_STATUS_B_LINK_PTR, + status_b + (config + G18G_26A428_STATUS_CONFIG_LINK_WORD) as u64); + write_relocated_words(M5_25G76_STATUS_CONFIG_WORDS, 0, G18G_26A428_STATUS_CONFIG_BANDS, + &mut out[config..config + G18G_26A428_STATUS_CONFIG_SIZE])?; + encode_g18g_26a428_status_config_defaults( + &mut out[config..config + G18G_26A428_STATUS_CONFIG_SIZE]); + put_u32(out, config + G18G_26A428_STATUS_CONFIG_LINK_WORD, 1); + put_u64(out, config + G18G_26A428_STATUS_CONFIG_LINK_PTR, + status_b + G18G_26A428_STATUS_B_LINK_WORD as u64); + Ok(()) +} + +/// Encode the whole 0x178000-byte G18G private cluster mapped at +/// `private_base`: status-B span, both status A blocks, the secondary root +/// `+0xc0` object and exactly one opening-producer entry per control channel +/// (plan 1.3: role 0 +0x320, role 1 +0x177720 = the J813 +0x1773e0 moved with +/// the tail). Work and report counters stay zero. +pub(crate) fn encode_g18g_26a428_private_cluster( + private_base: u64, + fwctl_state: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = G18G_26A428_PRIVATE_LAYOUT; + if out.len() != layout.total { + return Err(InitdataError::BufferSize); + } + required_address(private_base)?; + let private_end = private_base + .checked_add(layout.total as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(private_end - 1)?; + required_address(fwctl_state)?; + let fwctl_ring = fwctl_state + .checked_add(G18G_26A428_FWCTL_RING_OFFSET as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(fwctl_ring)?; + if fwctl_state < private_end && private_base < fwctl_ring + CONTROL_RECORD_SIZE as u64 { + return Err(InitdataError::PairAliasedObject); + } + out.fill(0); + let status_b = layout.status_b; + let status_a = layout.roles[0].status_a; + encode_g18g_26a428_primary_status_b(fwctl_state, fwctl_ring, private_base + status_b as u64, + &mut out[status_b..status_a])?; + for role in layout.roles { + encode_status_block(&mut out[role.status_a..role.status_a + STATUS_BLOCK_SIZE])?; + } + let extra = layout.secondary_extra_1; + encode_secondary_root_extra_1_for_profile(RootProfile::J873G18G26A428, + &mut out[extra..extra + STATUS_BLOCK_SIZE])?; + put_u32(out, layout.control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_PRIMARY_PRODUCER); + put_u32(out, layout.secondary_control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_SECONDARY_PRODUCER); + Ok(()) +} + +/// AON PTD (0xe4, KEXT_PROVEN size). INFERRED: the J813 header words. +pub(crate) fn encode_g18g_26a428_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { + encode_m5_25g76_aon_ptd(out) +} + +// --- Device-control and endpoint vocabulary (plan step 1.7) ----------------------- + +/// FW_PROVEN (jump table 0xfffffc0000006c60, 56 entries from opcode 4): the +/// opening/region-C reload handler (b000 0x3ed0) is reached by 0x15, 0x2a +/// and 0x30. +pub(crate) const G18G_26A428_CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x15; +pub(crate) const G18G_26A428_CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; +/// FW_PROVEN: IdlePowerOff keeps its opcode and behaviour (b000 0x4ef0). +pub(crate) const G18G_26A428_DEVICE_CONTROL_OPCODE_IDLE_POWER_OFF: u32 = 0x0a; +pub(crate) const G18G_26A428_DEVICE_CONTROL_OPCODE_MIN: u32 = 4; +pub(crate) const G18G_26A428_DEVICE_CONTROL_OPCODE_MAX: u32 = 0x3b; +/// FW_PROVEN: jump-table entries routed to the default (invalid) handler. +pub(crate) const G18G_26A428_DEVICE_CONTROL_INVALID_OPCODES: [u32; 9] = + [0x19, 0x1c, 0x20, 0x23, 0x26, 0x28, 0x31, 0x39, 0x3a]; + +pub(crate) const fn g18g_26a428_device_control_opcode_valid(opcode: u32) -> bool { + if opcode < G18G_26A428_DEVICE_CONTROL_OPCODE_MIN || opcode > G18G_26A428_DEVICE_CONTROL_OPCODE_MAX { + return false; + } + let mut index = 0; + while index < G18G_26A428_DEVICE_CONTROL_INVALID_OPCODES.len() { + if G18G_26A428_DEVICE_CONTROL_INVALID_OPCODES[index] == opcode { + return false; + } + index += 1; + } + true +} + +// --- Unproven values ------------------------------------------------------------------ + +/// One G18G value that is not proven for 26A428. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UnprovenField { + /// "hw_data", "register_map" (offset = slot), "policy", "region_c", + /// "status_b", "status_config", "main_config", "aon_ptd", + /// "secondary_extra_1". + pub(crate) object: &'static str, + pub(crate) offset: usize, + pub(crate) len: usize, + pub(crate) evidence: Evidence, + pub(crate) note: &'static str, +} + +const fn unproven(object: &'static str, offset: usize, len: usize, evidence: Evidence, + note: &'static str) -> UnprovenField { + UnprovenField { object, offset, len, evidence, note } +} + +/// Every INFERRED or GUESSED G18G value, for review and for the hardware +/// bring-up log. A crashlog assertion should be matched against this list +/// first. +pub(crate) const G18G_26A428_UNPROVEN: &[UnprovenField] = &[ + unproven("hw_data", 0x0000, 0x640, Evidence::Inferred, + "host prefix (+0x28 FW VA, two 32x0x18 accelerator tables) carried from J813"), + unproven("hw_data", 0x0fa0, 8, Evidence::Inferred, "u64 FW VA of accel+0x1d40; J813 0"), + unproven("hw_data", 0x0fa8, 8, Evidence::Inferred, "captured 0x8000; complete semantics unqualified"), + unproven("hw_data", 0x0fb4, 4, Evidence::Inferred, "captured J873 value 1; meaning unknown"), + unproven("hw_data", 0x0fc0, 0x110, Evidence::Inferred, + "J813 relocated words with captured J873 scalar corrections; complete semantics unqualified"), + unproven("hw_data", 0x10b4, 4, Evidence::Guessed, "new u32 (accel byte & 1); left 0"), + unproven("hw_data", 0x10d4, 0xc, Evidence::Inferred, + "captured J873 0x1b/4/12; complete field semantics unqualified"), + unproven("hw_data", 0x10e0, 4, Evidence::Inferred, "13 = gpu-num-perf-states (J813 12 matched by value)"), + unproven("hw_data", 0x1124, 0x800, Evidence::Inferred, + "ADT voltages filled into all 16 lanes of each row; b000 reads lanes 0 and 1"), + unproven("hw_data", 0x1964, 0x38, Evidence::Inferred, "1.02f scale word per state (J813 value)"), + unproven("hw_data", 0x19e4, 0x38, Evidence::Inferred, "native float32 ladder A; calibration formula unqualified"), + unproven("hw_data", 0x1a24, 0x38, Evidence::Inferred, "native float32 ladder B; calibration formula unqualified"), + unproven("hw_data", 0x1ae4, 0x40, Evidence::Inferred, "identity index map 0..15 (J813)"), + unproven("hw_data", 0x2660, 0xd4, Evidence::Inferred, + "J813 relocated words with captured J873 scalar corrections; complete semantics unqualified"), + unproven("hw_data", 0x2734, 0x80, Evidence::Inferred, + "KSM completion descriptor format (capacity 0x800, entry 0x40) carried from J813"), + unproven("hw_data", 0x27d4, 8, Evidence::Guessed, "new u64 (accel+0x12338); left 0"), + unproven("hw_data", 0x27dc, 0x3c, Evidence::Inferred, "J813 +0x26b4..+0x26ec words relocated"), + unproven("hw_data", 0x2818, 4, Evidence::Guessed, "new u32 (accel+0x6c8); read as part of a u64 by b000"), + unproven("hw_data", 0x281e, 0xc, Evidence::Inferred, "captured u32=1/u64=0; complete semantics unqualified"), + unproven("hw_data", 0x2832, 4, Evidence::Guessed, "new mode word, compared with 3 at ~10 b000 sites; left 0"), + unproven("hw_data", 0x283a, 8, Evidence::Inferred, "new enables (accel+0x1165c/+0x1166c bit 0); captured as 1/1"), + unproven("register_map", 0, 1, Evidence::Inferred, "AIC SW-int page from ADT meta-sw-interrupt"), + unproven("register_map", 12, 1, Evidence::Inferred, "ANE0 doorbell left unmapped: 26A428 G variant no longer sets it"), + unproven("register_map", 29, 2, Evidence::Inferred, "DPEP/ADCLK +0x18 left 0; flags 0 mapped read-only like J813 slots 10/14"), + unproven("register_map", 49, 1, Evidence::Inferred, "FenderRegsHWL flags 0 (base sets none) mapped read-only"), + unproven("register_map", 0, 60, Evidence::Inferred, "device VAs assigned by the J813 allocation rule"), + unproven("policy", 0x0000, 0x7800, Evidence::Inferred, + "matches eight native J873 snapshots with owned pointers/timestamp normalized; calibration semantics unqualified"), + unproven("policy", 0x00b0, 0x160, Evidence::Inferred, "captured host config matches; individual field semantics unqualified"), + unproven("policy", 0x1ca7, 0x400, Evidence::Inferred, "512 x 0x7fff u16 table moved with its +0x10 band"), + unproven("policy", 0x0b5c, 0x10, Evidence::Inferred, + "filter-time-constant terms: J813 defaults (J873 ADT has none)"), + unproven("policy", 0x0b9c, 0xc, Evidence::Inferred, + "tgt - dz and dz with dz = 1 carried from J813 (J873 ADT has no gpu-perf-dz)"), + unproven("policy", 0x64e0, 0x9c, Evidence::Inferred, + "native RT0g at +0x64e0 and one at +0x6520; other words zero, complete semantics unqualified"), + unproven("region_c", 0x0000, 0xe80, Evidence::Inferred, + "matches eight native J873 snapshots; individual field semantics unqualified"), + unproven("region_c", 0x0038, 4, Evidence::Inferred, "new feature bit captured as one; runtime effect unqualified"), + unproven("region_c", 0x008c, 8, Evidence::Inferred, "native 1300/1300; field units unqualified"), + unproven("region_c", 0x009c, 4, Evidence::Inferred, "1300 = this+0xf14 x 100 assumed equal to the state count"), + unproven("status_b", 0x0004, 4, Evidence::Inferred, "1 carried (26A428 stores accel+0x6b0 bit 2)"), + unproven("status_config", 0x0000, 0x640, Evidence::Inferred, + "complete 0x6d0 allocation matches eight native J873 pre-boot views; individual calibration semantics unqualified"), + unproven("status_config", 0x01dc, 8, Evidence::Inferred, + "native J873 1/29000 and original host stores verified; semantic names unqualified"), + unproven("status_config", 0x03e8, 0x104, Evidence::Inferred, + "native generated 16x16 u8 table and two u16 enables; original host copy geometry verified"), + unproven("main_config", 0x0300, 4, Evidence::Inferred, "secondary kind 0xc carried (accel+0x10a74)"), + unproven("main_config", 0x02f8, 8, Evidence::Guessed, "third descriptor table (accel[0x235e]) absent: 0"), + unproven("aon_ptd", 0x0000, 0xc, Evidence::Inferred, "J813 PTD header words"), + unproven("secondary_extra_1", 0x0000, 0x60, Evidence::Inferred, "T8140/J813 content (same store geometry)"), +]; + +#[cfg(test)] +#[path = "g18_initdata_test.rs"] +mod g18_tests; + +#[cfg(test)] +mod tests { + #[test] + fn m5_register_extent_uses_size_10_and_rejects_invalid_pages() { + let native = super::M5_25G76_REGISTER_WINDOWS.iter().find(|w| w.slot == 39).unwrap(); + assert_eq!(native.mapped_pages(), Ok((0x580e08000, 0xfffffc2180128000, 0x10000))); + let mut invalid = *native; + invalid.phys += 1; + assert_eq!(invalid.mapped_pages(), Err(super::InitdataError::RegisterWindow)); + invalid = *native; + invalid.size_10 = 0; + assert_eq!(invalid.mapped_pages(), Err(super::InitdataError::RegisterWindow)); + invalid = *native; + invalid.device_va = u64::MAX & !0x3fff; + assert_eq!(invalid.mapped_pages(), Err(super::InitdataError::AddressOutOfRange)); + invalid = *native; + invalid.phys = u64::MAX & !0x3fff; + assert_eq!(invalid.mapped_pages(), Err(super::InitdataError::AddressOutOfRange)); + } + + #[test] + fn m5_private_arena_rejects_invalid_bindings_before_writing() { + let mut arena = vec![0xa5; super::M5_25G76_PRIVATE_CLUSTER_SIZE]; + for pointer in [0, u64::MAX - 8, 1 << 48] { + assert!(super::encode_m5_25g76_private_cluster(pointer, &mut arena).is_err()); + assert!(arena.iter().all(|&byte| byte == 0xa5)); + } + } + + use super::*; + + const KERN: u64 = 0xffff_fc20_0000_0000; + + fn encoded_fast_die() -> FastDieConfig { + let mut bundle = vec![0u8; HW_DATA_BUNDLE_MIN_ALLOC]; + encode_bundle_static(KERN + 0x1000, &mut bundle).unwrap(); + let mut status_b = vec![0u8; PRIMARY_STATUS_B_OBJECT_SIZE]; + encode_primary_status_b(KERN + 0x2000, KERN + 0x2040, &mut status_b).unwrap(); + fast_die_config(&bundle, &status_b).unwrap() + } + + #[test] + fn fast_die_config_matches_native() { + let c = encoded_fast_die(); + assert_eq!(FASTDIE_OBJECT_BUNDLE_OFFSET, 0x5240); + assert_eq!(c.mask, 0x82a); // G17P mask for accel[0x4a0] != 0x1f + assert_eq!(c.target_cc, FASTDIE_NATIVE_TARGET_CC); // 110.00 C + assert_eq!(c.release_cc, 8000); + assert_eq!(c.delta_cc, 0); + assert_eq!(c.kp_bits, 0x41a2_b852); + assert_eq!(c.ki_dt_bits, 0x4116_0419); + assert_eq!(c.state, 0x4200_0006); // fw1 0x174fc constant + assert_eq!(c.live_target_cc, c.target_cc); + assert_eq!(c.live_release_cc, c.release_cc); + assert_eq!(c.live_kp_bits, c.kp_bits); + assert_eq!(c.live_ki_dt_bits, c.ki_dt_bits); + assert_eq!(c.live_update, 0); + // Same object g17_dvfs_policy::GFX1_STATS_OFFSET (primary state + 0xe780) reads. + assert_eq!(PRIMARY_STATUS_B_STATE_GRID_OFFSET + PRIMARY_STATUS_B_CONFIG_OFFSET, 0xe780); + } + + #[test] + fn fast_die_target_override_is_lower_only() { + assert_eq!(fast_die_target_override(0).unwrap(), None); + assert_eq!(fast_die_target_override(10_000).unwrap(), Some(10_000)); + assert_eq!(fast_die_target_override(8_500).unwrap(), Some(8_500)); + assert!(fast_die_target_override(11_001).is_err()); + assert!(fast_die_target_override(8_499).is_err()); + let mut bundle = vec![0u8; HW_DATA_BUNDLE_MIN_ALLOC]; + encode_bundle_static(KERN + 0x1000, &mut bundle).unwrap(); + let mut status_b = vec![0u8; PRIMARY_STATUS_B_OBJECT_SIZE]; + encode_primary_status_b(KERN + 0x2000, KERN + 0x2040, &mut status_b).unwrap(); + apply_fast_die_target_bundle(&mut bundle, 10_000).unwrap(); + apply_fast_die_target_status_b(&mut status_b, 10_000).unwrap(); + let c = fast_die_config(&bundle, &status_b).unwrap(); + assert_eq!((c.target_cc, c.live_target_cc, c.release_cc, c.mask), (10_000, 10_000, 8000, 0x82a)); + } + + + fn native_compute_bytes(bytes: &[u8]) -> Vec { + let mut native = bytes.to_vec(); + for offset in 0..bytes.len().saturating_sub(7) { + let value = get_u64(bytes, offset); + let address = value & !CONTROL_OPERAND_ENTRY_FLAG; + if (0x70_0200_0000..0x70_03ef_8000).contains(&address) { + put_u64(&mut native, offset, value - COMPUTE_FLIST_RELOCATION); + } + } + native + } + #[test] + fn compute_aperture_ranges_do_not_overlap_each_other_or_render() { + let mut ranges = vec![ + (COMPUTE_FLIST_PAGE_LIST_ADDRESS, 0x20_0000), + (COMPUTE_FLIST_RUN_TABLE_ADDRESS, 0x1_0000), + (COMPUTE_RUNTIME_PAGE_LIST_ADDRESS, 0xc000), + (COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS, 0x4000), + ]; + for i in 0..CONTROL_OPERAND_ENTRY_COUNT { + ranges.push((COMPUTE_FLIST_BUFFER_BASE + i as u64 * CONTROL_OPERAND_BUFFER_STRIDE, + CONTROL_OPERAND_BUFFER_SIZE)); + } + for base in G17P_KSM_COMPLETION_LOW_VAS { + ranges.push((base, G17P_KSM_COMPLETION_BACKING_SIZE as u64)); + } + // Native mature kernel-low ZERO0/ZERO1 intentionally alias that + // mode's main backing, whereas ordinary client mappings retain its + // separate page-list. They are not additional overlapping mappings. + assert_eq!(COMPUTE_RUNTIME_ZERO_BUFFER_0_ADDRESS, COMPUTE_FLIST_PAGE_LIST_ADDRESS); + assert_eq!(COMPUTE_RUNTIME_ZERO_BUFFER_1_ADDRESS, COMPUTE_FLIST_PAGE_LIST_ADDRESS + 0x10_8000); + for (i, &(base, size)) in ranges.iter().enumerate() { + assert_eq!(base & 0x3fff, 0); + assert!(base >= 0x70_0200_0000 && base + size <= 0x70_0400_0000); + for &(other, other_size) in ranges.iter().skip(i + 1) { + assert!(base + size <= other || other + other_size <= base, + "overlap: {base:#x}+{size:#x} with {other:#x}+{other_size:#x}"); + } + } + assert_eq!(CONTROL_OPERAND_TABLE_ADDRESS, 0x70_0020_8000); + assert_eq!(CONTROL_OPERAND_BUFFER_BASE, 0x70_0022_0000); + } + + #[test] + fn compute_support_uses_full_unaligned_page_pointer() { + for encode in [encode_compute_readiness_class1_support, + encode_compute_readiness_class3_support, + encode_compute_runtime_class2_support, + encode_compute_partial_opening_control_shared] { + let mut bytes = vec![0xa5; 0x4000]; + encode(&mut bytes).unwrap(); + assert_eq!(get_u64(&bytes, 0x14), 0x70_0200_0000); + assert_eq!(get_u32(&bytes, 0x1c), 0x4_0000); + assert_eq!(get_u64(&bytes, 0x30), 0x70_0220_8000); + } + } + + fn shared() -> SharedAddresses { + SharedAddresses { + region_a: KERN + 0x1b_8000, + region_c: KERN + 0x1a_0000, + hw_data_bundle: KERN + 0xc078_8000, + hw_data_bundle_alloc: NATIVE_SHARED_CLUSTER_SIZE, + } + } + + fn primary() -> InstanceAddresses { + InstanceAddresses { + root: KERN + 0x1a_8000, + main_config: KERN + 0xc078_8000 + NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET as u64, + status_a: KERN + 0x2_ee40, + status_b: KERN + 0x2_0000 + PRIMARY_STATUS_B_STATE_GRID_OFFSET as u64, + secondary_extras: [0, 0], + } + } + + fn secondary() -> InstanceAddresses { + InstanceAddresses { + root: KERN + 0x1a_8000 + SECONDARY_ROOT_DELTA, + main_config: KERN + 0xc078_8000 + NATIVE_SECONDARY_MAIN_BUNDLE_OFFSET as u64, + status_a: KERN + 0xe_3100, + status_b: 0, + secondary_extras: [KERN + 0x2_e780, KERN + 0x19_7400], + } + } + + fn state_grid(role: InstanceRole) -> u64 { + match role { + InstanceRole::Primary => KERN + 0x2_0000, + InstanceRole::Secondary => KERN + 0x19_70c0, + } + } + + fn channels(role: InstanceRole) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let inst = match role { + InstanceRole::Primary => primary(), + InstanceRole::Secondary => secondary(), + }; + derive_channel_table( + role, + state_grid(role), + inst.status_a, + inst.main_config, + shared().hw_data_bundle, + ) + } + + fn region_views() -> PrimaryRegionViews { + PrimaryRegionViews { + sentinel_high: KERN + 0xc07c_8000, + alias_high: [KERN + 0x15e_0000, KERN + 0x15e_8000], + } + } + + fn encode_root_bytes(role: InstanceRole) -> Vec { + let inst = match role { + InstanceRole::Primary => primary(), + InstanceRole::Secondary => secondary(), + }; + let mut out = vec![0xa5; root_size(role)]; + encode_root(role, &shared(), &inst, &mut out).unwrap(); + out + } + + fn encode_main_bytes(role: InstanceRole) -> Vec { + let inst = match role { + InstanceRole::Primary => primary(), + InstanceRole::Secondary => secondary(), + }; + let views = region_views(); + let views = match role { + InstanceRole::Primary => Some(&views), + InstanceRole::Secondary => None, + }; + let mut out = vec![0xa5; MAIN_CONFIG_SIZE]; + encode_main_config(role, &shared(), &inst, &channels(role), views, &mut out).unwrap(); + out + } + + #[test] + fn descriptor_table_aliases_are_published_in_both_main_configs() { + let views = region_views(); + let primary = encode_main_bytes(InstanceRole::Primary); + assert_eq!(get_u64(&primary, 0x2d8), REGION_VIEW_LOW_ADDRS[0]); + assert_eq!(get_u64(&primary, 0x2e0), views.alias_high[0]); + assert_eq!(get_u64(&primary, 0x2e8), REGION_VIEW_LOW_ADDRS[1]); + assert_eq!(get_u64(&primary, 0x2f0), views.alias_high[1]); + + let secondary = encode_main_bytes(InstanceRole::Secondary); + assert_eq!(get_u64(&secondary, 0x2d8), REGION_VIEW_LOW_ADDRS[0]); + assert_eq!(get_u64(&secondary, 0x2e0), 0); + assert_eq!(get_u64(&secondary, 0x2e8), REGION_VIEW_LOW_ADDRS[1]); + assert_eq!(get_u64(&secondary, 0x2f0), 0); + } + + #[test] + fn per_role_root_sizes_version_tuple_and_handoff_zeroes() { + assert_eq!(ROOT_SIZE_PRIMARY, 0xb8); + assert_eq!(ROOT_SIZE_SECONDARY, 0xc8); + assert_eq!(root_size(InstanceRole::Primary), 0xb8); + assert_eq!(root_size(InstanceRole::Secondary), 0xc8); + // The exact version tuple, both as words and as the u64 the firmware + // compares. The M5 value 0x0c8d_e339_0728_04c0 must never reappear. + assert_eq!(INITDATA_VERSION, [0x04c0, 0x0396, 0xa322, 0x0c8a]); + assert_eq!(INITDATA_VERSION_WORD, 0x0c8a_a322_0396_04c0); + + let root = encode_root_bytes(InstanceRole::Primary); + assert_eq!(root.len(), 0xb8); + assert_eq!(get_u64(&root, 0x00), INITDATA_VERSION_WORD); + assert_eq!(get_u64(&root, 0x08), shared().region_a); + // +0x10 is zero at handoff even though the buffer started dirty. + assert_eq!(get_u64(&root, 0x10), 0); + assert_eq!(get_u64(&root, 0x18), primary().main_config); + assert_eq!(get_u64(&root, 0x20), shared().region_c); + assert_eq!(get_u32(&root, 0x28), 0); + assert_eq!(get_u32(&root, 0x2c), 1); + assert_eq!(get_u64(&root, 0xa8), primary().status_a); + assert_eq!(get_u64(&root, 0xb0), primary().status_b); + assert!(check_firmware_root_gates(&root).is_ok()); + + let root = encode_root_bytes(InstanceRole::Secondary); + assert_eq!(root.len(), 0xc8); + assert_eq!(get_u32(&root, 0x28), 1); + assert_eq!(get_u64(&root, 0xa8), secondary().status_a); + // Status B is real only on the primary. + assert_eq!(get_u64(&root, 0xb0), 0); + assert_eq!(get_u64(&root, 0xb8), secondary().secondary_extras[0]); + assert_eq!(get_u64(&root, 0xc0), secondary().secondary_extras[1]); + assert!(check_firmware_root_gates(&root).is_ok()); + } + + #[test] + fn uat_geometry_block_and_level_descriptors() { + let root = encode_root_bytes(InstanceRole::Primary); + // Page size 0x4000 (u16), page bits 14, level count 3. + assert_eq!( + u16::from_le_bytes(root[0x30..0x32].try_into().unwrap()), + 0x4000 + ); + assert_eq!(root[0x32], 14); + assert_eq!(root[0x33], 3); + // Three 0x20-byte descriptors: shifts 36/25/14, entries 64/2048/2048. + let expected = [(36u8, 64u16), (25, 2048), (14, 2048)]; + for (index, (shift, entries)) in expected.iter().enumerate() { + let base = 0x34 + index * 0x20; + assert_eq!(root[base], 8); + assert_eq!(root[base + 1], 14); + assert_eq!(root[base + 2], 14); + assert_eq!(root[base + 3], *shift); + assert_eq!( + u16::from_le_bytes(root[base + 4..base + 6].try_into().unwrap()), + *entries + ); + assert_eq!( + u16::from_le_bytes(root[base + 6..base + 8].try_into().unwrap()), + 0x4000 + ); + assert_eq!(get_u64(&root, base + 0x08), 1); + assert_eq!(get_u64(&root, base + 0x10), 0x0000_03ff_ffff_c000); + assert_eq!(get_u64(&root, base + 0x18), (*entries as u64 - 1) << *shift); + } + // Spot-check the derived index masks against their absolute values. + assert_eq!(get_u64(&root, 0x34 + 0x18), 0x3f_0000_0000_0 >> 4 << 4); // 63 << 36 + assert_eq!(get_u64(&root, 0x34 + 0x18), 63u64 << 36); + assert_eq!(get_u64(&root, 0x54 + 0x18), 2047u64 << 25); + assert_eq!(get_u64(&root, 0x74 + 0x18), 2047u64 << 14); + // The descriptor block ends at 0x94; the gap to status A stays zero. + assert!(root[0x94..0xa8].iter().all(|byte| *byte == 0)); + } + + #[test] + fn main_config_bundle_views_are_views_into_one_allocation() { + assert_eq!(MAIN_CONFIG_SIZE, 0x600); + let main = encode_main_bytes(InstanceRole::Primary); + assert_eq!(main.len(), 0x600); + let bundle = shared().hw_data_bundle; + assert_eq!(get_u64(&main, 0x00), bundle); + // The repeated writable region is bundle + 0xc500, at +0x08 and + // +0x10 — past the 0xc000 principal bundle, which is why the + // allocation must reach 0x10000. + assert_eq!(get_u64(&main, 0x08), bundle + 0xc500); + assert_eq!(get_u64(&main, 0x10), bundle + 0xc500); + assert!(HW_DATA_BUNDLE_MIN_ALLOC >= 0xc500); + // Five unaligned qwords at +0x254: views into the ONE bundle + // allocation at the fixed offsets, not independent objects. + let offsets = [0x2740u64, 0x3380, 0x4400, 0xbc80, 0xbd00]; + let extents = [0x18c0usize, 0x0c80, 0x3c00, 0x3380, 0x3300]; + for (index, (offset, extent)) in offsets.iter().zip(extents).enumerate() { + let view = get_u64(&main, 0x254 + index * 8); + assert_eq!(view, bundle + offset); + // Each view lies wholly inside the single allocation. + assert!((*offset as usize) + extent <= shared().hw_data_bundle_alloc); + } + assert_eq!(BUNDLE_VIEW_OFFSETS.map(|o| o as u64), offsets); + assert_eq!(BUNDLE_VIEW_EXTENTS, extents); + // Views 3 and 4 overlap by all but 0x80 bytes, and view 0 overlaps + // the tail of the hardware-data object — the relations independent + // allocations cannot reproduce. + assert_eq!(0xbc80 + 0x3380, 0xbd00 + 0x3300); + assert!(BUNDLE_VIEW_OFFSETS[0] < HW_DATA_SIZE); + } + + #[test] + fn bundle_allocation_below_minimum_fails_closed() { + let mut small = shared(); + small.hw_data_bundle_alloc = 0xc000; + let mut out = vec![0; ROOT_SIZE_PRIMARY]; + assert_eq!( + encode_root(InstanceRole::Primary, &small, &primary(), &mut out), + Err(InitdataError::BundleAllocationTooSmall) + ); + } + + #[test] + fn channel_table_has_seventeen_entries_with_the_control_grid_at_12() { + assert_eq!(CHANNEL_TABLE_ENTRIES, 17); + assert_eq!(channel_table_entry_range(0), Some(0x20..0x40)); + assert_eq!(channel_table_entry_range(11), Some(0x180..0x1a0)); + // Entry 12 exists. The M5-derived model refused it, misreading its + // state-counter addresses at +0x1a0/+0x1a8 as an unrelated cursor. + assert_eq!(channel_table_entry_range(12), Some(0x1a0..0x1c0)); + assert_eq!(channel_table_entry_range(16), Some(0x220..0x240)); + assert_eq!(channel_table_entry_range(17), None); + + let main = encode_main_bytes(InstanceRole::Primary); + let grid = state_grid(InstanceRole::Primary); + let bundle = shared().hw_data_bundle; + // Work entry 0 (TA_0): three separate state-counter addresses at 0x10 + // spacing; consumer = state 0, producer = state 2. Its ring is a work + // ring in the bundle. + assert_eq!(get_u64(&main, 0x20), grid + 0x0c0); // consumer + assert_eq!(get_u64(&main, 0x28), grid + 0x0d0); + assert_eq!(get_u64(&main, 0x30), grid + 0x0e0); // producer + assert_eq!( + get_u64(&main, 0x38), + bundle + WORK_RING_BUNDLE_OFFSETS[0] as u64 + ); + + // Entry 12 (device control): three counters in the compact grid, and + // the ring embedded at main + 0x4c0, where the opening opcode is staged. + let e12 = channel_table_entry_range(12).unwrap().start; + assert_eq!( + get_u64(&main, e12 + CHANNEL_STATE_CONSUMER * 8), + grid + 0x300 + ); + assert_eq!( + get_u64(&main, e12 + CHANNEL_STATE_PRODUCER * 8), + grid + 0x320 + ); + assert_eq!(get_u64(&main, e12 + 0x18), primary().main_config + 0x4c0); + assert_eq!(get_u32(&main, 0x4c0), CONTROL_OPENING_OPCODE_PRIMARY); + + // Entry 15 is partial: a first state address, no producer, no ring. + let e15 = channel_table_entry_range(15).unwrap().start; + assert_ne!(get_u64(&main, e15), 0); + assert_eq!(get_u64(&main, e15 + 0x08), 0); + assert_eq!(get_u64(&main, e15 + 0x10), 0); + assert_eq!(get_u64(&main, e15 + 0x18), 0); + // Entry 16 is entirely empty. + let e16 = channel_table_entry_range(16).unwrap().start; + assert!(main[e16..e16 + CHANNEL_ENTRY_SIZE] + .iter() + .all(|byte| *byte == 0)); + // The primary carries the +0x3e0 mask; +0x300 (secondary kind) stays 0. + assert_eq!(get_u32(&main, 0x3e0), MAIN_PRIMARY_MASK_VALUE); + assert_eq!(get_u32(&main, 0x32c), 1); + assert_eq!(get_u32(&main, 0x344), 0xabcd_abcd); + assert_eq!(get_u32(&main, 0x300), 0); + } + + #[test] + fn secondary_leaves_work_channels_empty_but_keeps_control() { + let main = encode_main_bytes(InstanceRole::Secondary); + // Entries 0..11 are empty on the control-only peer. + for index in 0..WORK_CHANNEL_COUNT { + let range = channel_table_entry_range(index).unwrap(); + assert!(main[range].iter().all(|byte| *byte == 0)); + } + // Entry 12 control ring is still main + 0x4c0, with the secondary opcode. + let e12 = channel_table_entry_range(12).unwrap().start; + assert_eq!(get_u64(&main, e12 + 0x18), secondary().main_config + 0x4c0); + assert_eq!(get_u32(&main, 0x4c0), CONTROL_OPENING_OPCODE_SECONDARY); + // Secondary-only fields: +0x300 = 4, and the unaligned +0x471 pointer + // into primary view 2. The primary's +0x3e0 mask is absent here. + assert_eq!(get_u32(&main, 0x300), MAIN_SECONDARY_KIND_VALUE); + let extra = shared().hw_data_bundle + + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] as u64 + + SECONDARY_EXTRA_VIEW_OFFSET as u64; + assert_eq!(get_u64(&main, 0x471), extra); + assert_eq!(get_u32(&main, 0x3e0), 0); + } + + #[test] + fn primary_control_opening_is_the_single_native_record() { + let mut opening = vec![0; control_opening_size(InstanceRole::Primary)]; + encode_control_opening(InstanceRole::Primary, &mut opening).unwrap(); + + assert_eq!(control_opening_producer(InstanceRole::Primary), 1); + for slot in 0..1 { + let base = slot * CONTROL_RECORD_SIZE; + assert_eq!(get_u32(&opening, base), CONTROL_OPENING_OPCODE_PRIMARY); + assert!(opening[base + 4..base + CONTROL_RECORD_SIZE] + .iter() + .all(|byte| *byte == 0)); + } + assert_eq!(opening.len(), CONTROL_RECORD_SIZE); + } + + #[test] + fn live_compute_control_tail_matches_the_output_positive_sequence() { + fn fnv1a64(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) + } + + let mut tail = vec![0u8; COMPUTE_RUNTIME_CONTROL_RECORDS * CONTROL_RECORD_SIZE]; + for index in 0..COMPUTE_RUNTIME_CONTROL_RECORDS { + let start = index * CONTROL_RECORD_SIZE; + encode_compute_runtime_control_record( + index, + &mut tail[start..start + CONTROL_RECORD_SIZE], + ) + .unwrap(); + } + assert_eq!(tail.len(), 65 * 0x40); + assert_eq!(fnv1a64(&native_compute_bytes(&tail)), 0x544f_a4b7_bc23_e65a); + assert_eq!(COMPUTE_RUNTIME_CONTROL_FINAL_COUNTER, 66); + + for (index, sequence) in [(0, 0), (53, 53), (55, 54), (56, 55), (58, 56), (64, 62)] { + let base = index * CONTROL_RECORD_SIZE; + assert_eq!(get_u32(&tail, base), 0x2e); + assert_eq!(get_u32(&tail, base + 0x04), sequence); + assert_eq!(get_u32(&tail, base + 0x0c), 0); + } + for (index, class, sequence, operand, slot_offset, count) in [ + ( + COMPUTE_RUNTIME_CLASS1_RECORD_INDEX, + 1, + 54, + COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS, + 0x480, + 0x18, + ), + ( + COMPUTE_RUNTIME_CLASS2_RECORD_INDEX, + 2, + 56, + COMPUTE_FLIST_RUN_TABLE_ADDRESS, + 0x5c0, + 0x28, + ), + ] { + let base = index * CONTROL_RECORD_SIZE; + assert_eq!(get_u32(&tail, base), 0x20); + assert_eq!(get_u32(&tail, base + 0x04), class); + assert_eq!(get_u32(&tail, base + 0x08), 0x3f); + assert_eq!(get_u32(&tail, base + 0x0c), sequence); + assert_eq!(get_u64(&tail, base + 0x14), COMPUTE_RUNTIME_SUPPORT_ADDRESS); + assert_eq!(get_u64(&tail, base + 0x1c), operand); + assert_eq!(get_u64(&tail, base + 0x24), operand + slot_offset); + assert_eq!(get_u32(&tail, base + 0x2c), count); + assert_eq!(get_u32(&tail, base + 0x30), 0); + assert_eq!(get_u32(&tail, base + 0x34), 1); + } + } + + #[test] + fn scheduler_state_release_record_is_exact() { + let mut record = [0xa5; CONTROL_RECORD_SIZE]; + encode_scheduler_state_release_record( + 0xffff_fc20_0123_4000, + [0x11, 0x22, 0x33, 0x44], + &mut record, + ) + .unwrap(); + assert_eq!(get_u32(&record, 0), 0x14); + assert_eq!(get_u32(&record, 4), 0); + assert_eq!(record[0x08], 0x44); + assert_eq!(record[0x09], 0x11); + assert_eq!(record[0x0a], 0x22); + assert_eq!(record[0x0b], 0x33); + assert_eq!(get_u64(&record, 0x0c), 0xffff_fc20_0123_4000); + assert!(record[0x14..].iter().all(|byte| *byte == 0)); + assert!(device_control_asserts_gpu_power(0x14) == Some(true)); + } + + #[test] + fn bootstrap_uma_release_record_is_exact() { + let mut record = [0xa5; CONTROL_RECORD_SIZE]; + encode_bootstrap_uma_release_record(0, 0, &mut record).unwrap(); + assert_eq!(get_u32(&record, 0), CONTROL_BOOTSTRAP_UMA_RELEASE_OPCODE); + assert!(record[4..].iter().all(|byte| *byte == 0)); + + encode_bootstrap_uma_release_record(0x1122_3344_5566_7788, 0xaabb_ccdd, &mut record) + .unwrap(); + assert_eq!(get_u64(&record, 0x04), 0x1122_3344_5566_7788); + assert_eq!(get_u32(&record, 0x0c), 0xaabb_ccdd); + assert!(record[0x10..].iter().all(|byte| *byte == 0)); + } + + #[test] + fn first_cl_readiness_suffix_has_exact_sequences_contexts_and_bytes() { + assert_eq!( + COMPUTE_READINESS_PUBLICATION_ORDER, + [ + ComputeReadinessPublicationStep::ControlRecord(0), + ComputeReadinessPublicationStep::ControlRecord(1), + ComputeReadinessPublicationStep::ControlRecord(2), + ComputeReadinessPublicationStep::InitializeContext2Activation, + ComputeReadinessPublicationStep::ControlRecord(3), + ComputeReadinessPublicationStep::ControlRecord(4), + ComputeReadinessPublicationStep::ControlRecord(5), + ] + ); + let mut records = [[0u8; CONTROL_RECORD_SIZE]; COMPUTE_READINESS_RECORDS]; + for (index, record) in records.iter_mut().enumerate() { + encode_compute_readiness_record(index, record).unwrap(); + } + + for (index, boundary) in COMPUTE_READINESS_BOUNDARIES.iter().enumerate() { + assert_eq!(boundary.index as usize, index); + assert_eq!( + boundary.producer_before, + CONTROL_OPENING_PRIMARY_PRODUCER + index as u32 + ); + assert_eq!( + boundary.producer_after, + CONTROL_OPENING_PRIMARY_PRODUCER + 1 + index as u32 + ); + assert_eq!(get_u32(&records[index], 0x00), boundary.opcode); + let (sequence_offset, context_offset) = if boundary.opcode == 0x20 { + (0x0c, 0x30) + } else { + (0x04, 0x0c) + }; + assert_eq!(get_u32(&records[index], sequence_offset), boundary.sequence); + assert_eq!(get_u32(&records[index], context_offset), boundary.context); + assert_eq!(compute_readiness_boundary(index), Some(*boundary)); + } + assert_eq!(compute_readiness_boundary(COMPUTE_READINESS_RECORDS), None); + + let mut progress = ComputeReadinessProgress::new(); + assert_eq!(progress.retire(5), None); + for boundary in COMPUTE_READINESS_BOUNDARIES { + assert_eq!(progress.next_boundary(), Some(boundary)); + let pending = progress.begin_publication(boundary).unwrap(); + assert_eq!(pending.pending_target(), Some(boundary.producer_after)); + assert_eq!(pending.next_boundary(), None); + assert_eq!(pending.begin_publication(boundary), None); + assert_eq!(pending.retire(boundary.producer_before), None); + progress = pending.retire(boundary.producer_after).unwrap(); + assert_eq!(progress.retired_records(), boundary.index as usize + 1); + } + assert!(progress.complete()); + assert_eq!(progress.next_boundary(), None); + + for (index, class, sequence, support, slot, count, context) in [ + ( + 0, + 1, + 35, + COMPUTE_READINESS_CLASS1_SUPPORT_ADDRESS, + 0x440, + 0x20, + 1, + ), + ( + 3, + 3, + 37, + COMPUTE_READINESS_CLASS3_SUPPORT_ADDRESS, + 0x5c0, + 0x28, + 2, + ), + ] { + let record = &records[index]; + assert_eq!(get_u32(record, 0x00), 0x20); + assert_eq!(get_u32(record, 0x04), class); + assert_eq!(get_u32(record, 0x08), 0x3f); + assert_eq!(get_u32(record, 0x0c), sequence); + assert_eq!(get_u64(record, 0x14), support); + assert_eq!(get_u64(record, 0x1c), COMPUTE_FLIST_RUN_TABLE_ADDRESS); + assert_eq!(get_u64(record, 0x24), COMPUTE_FLIST_RUN_TABLE_ADDRESS + slot); + assert_eq!(get_u32(record, 0x2c), count); + assert_eq!(get_u32(record, 0x30), context); + assert_eq!(get_u32(record, 0x34), 1); + } + + for (index, sequence, context) in [(1, 35, 1), (2, 36, 1), (4, 37, 2), (5, 38, 2)] { + let record = &records[index]; + assert_eq!(get_u32(record, 0x00), 0x2e); + assert_eq!(get_u32(record, 0x04), sequence); + assert_eq!(get_u32(record, 0x0c), context); + assert!(record[0x10..].iter().all(|byte| *byte == 0)); + } + assert_eq!( + COMPUTE_READINESS_FINAL_COUNTER, + CONTROL_OPENING_PRIMARY_PRODUCER + COMPUTE_READINESS_RECORDS as u32 + ); + assert!(encode_compute_readiness_record(COMPUTE_READINESS_RECORDS, &mut records[0]) + .is_err()); + } + + #[test] + fn first_cl_readiness_support_state_and_activation_pages_are_exact() { + fn expected_support( + class: u32, + active: u32, + resource_class: u64, + cursor: u32, + state: u64, + final_kind: u32, + ) -> [u8; COMPUTE_READINESS_PAGE_SIZE] { + let mut expected = [0u8; COMPUTE_READINESS_PAGE_SIZE]; + put_u32(&mut expected, 0x00, class); + put_u32(&mut expected, 0x08, active); + put_u32(&mut expected, 0x10, class); + put_u64(&mut expected, 0x14, COMPUTE_FLIST_PAGE_LIST_ADDRESS); + put_u32(&mut expected, 0x1c, 0x4_0000); + put_u64(&mut expected, 0x20, resource_class << 40); + put_u64(&mut expected, 0x28, resource_class << 40); + put_u64(&mut expected, 0x30, COMPUTE_FLIST_RUN_TABLE_ADDRESS); + put_u32(&mut expected, 0x40, 4); + put_u32(&mut expected, 0x48, cursor); + put_u64(&mut expected, 0x4c, state); + put_u32(&mut expected, 0x60, final_kind); + expected + } + + let mut class1 = [0u8; COMPUTE_READINESS_PAGE_SIZE]; + let mut class3 = [0u8; COMPUTE_READINESS_PAGE_SIZE]; + let mut state = [0xffu8; COMPUTE_READINESS_PAGE_SIZE]; + encode_compute_readiness_class1_support(&mut class1).unwrap(); + encode_compute_readiness_class3_support(&mut class3).unwrap(); + encode_compute_readiness_state(&mut state).unwrap(); + assert_eq!( + class1, + expected_support( + 1, + 1, + 0x11, + 0x88, + COMPUTE_READINESS_CLASS1_STATE_ADDRESS, + 2, + ) + ); + assert_eq!( + class3, + expected_support( + 3, + 2, + 0x17, + 0xb8, + COMPUTE_READINESS_CLASS3_STATE_ADDRESS, + 3, + ) + ); + assert_eq!(get_u32(&state, 0), 1); + assert!(state[4..].iter().all(|byte| *byte == 0)); + + let mut activation = [0xffu8; CONTROL_RECORD_SIZE]; + encode_compute_readiness_activation_record(&mut activation).unwrap(); + let mut expected = [0u8; CONTROL_RECORD_SIZE]; + put_u64(&mut expected, 0x00, 0x0000_0100_0000_ffff); + put_u64(&mut expected, 0x20, 0x0002_0000_0000_0000); + put_u64(&mut expected, 0x30, 0x0000_0000_ff00_0000); + assert_eq!(activation, expected); + assert_eq!(COMPUTE_READINESS_ACTIVATION_RECORD_OFFSET, 0x80); + + let mut operand_table = [0xffu8; CONTROL_OPERAND_TABLE_SIZE]; + encode_compute_readiness_operand_table(&mut operand_table).unwrap(); + for index in 0..COMPUTE_READINESS_OPERAND_ENTRY_COUNT { + assert_eq!( + get_u64(&operand_table, index * CONTROL_OPERAND_ENTRY_STRIDE), + (COMPUTE_FLIST_BUFFER_BASE + + index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + | CONTROL_OPERAND_ENTRY_FLAG, + ); + } + assert!(operand_table + [COMPUTE_READINESS_OPERAND_ENTRY_COUNT * CONTROL_OPERAND_ENTRY_STRIDE..] + .iter() + .all(|byte| *byte == 0)); + } + + #[test] + fn live_compute_control_closure_matches_source_builders() { + fn fnv1a64(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) + } + + let mut class1 = vec![0; COMPUTE_RUNTIME_SUPPORT_SIZE]; + encode_compute_runtime_class1_support(&mut class1).unwrap(); + assert_eq!(fnv1a64(&native_compute_bytes(&class1)), 0xb40b_a63a_16c1_937d); + assert_eq!(get_u32(&class1, 0x00), 1); + assert_eq!(get_u64(&class1, 0x14), COMPUTE_RUNTIME_PAGE_LIST_ADDRESS); + assert_eq!(get_u64(&class1, 0x30), COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS); + assert_eq!(get_u64(&class1, 0x4c), COMPUTE_RUNTIME_STATE_ADDRESS); + + let mut class2 = vec![0; COMPUTE_RUNTIME_SUPPORT_SIZE]; + encode_compute_runtime_class2_support(&mut class2).unwrap(); + assert_eq!(fnv1a64(&native_compute_bytes(&class2)), 0xdd12_cbe2_2d7c_3d50); + assert_eq!(get_u32(&class2, 0x00), 2); + assert_eq!(get_u64(&class2, 0x30), COMPUTE_FLIST_RUN_TABLE_ADDRESS); + + let mut table = vec![0; COMPUTE_RUNTIME_CLASS1_TABLE_SIZE]; + encode_compute_runtime_class1_table(&mut table).unwrap(); + assert_eq!(fnv1a64(&native_compute_bytes(&table)), 0xc820_66ed_9e50_6c41); + for (index, address) in COMPUTE_RUNTIME_BUFFER_ADDRESSES.into_iter().enumerate() { + assert_eq!(get_u64(&table, index * 0x40), address | (1 << 60)); + } + + let mut inventory = vec![0; COMPUTE_RUNTIME_PAGE_LIST_SIZE]; + encode_compute_runtime_page_inventory(&mut inventory).unwrap(); + assert_eq!(fnv1a64(&native_compute_bytes(&inventory)), 0x9f78_e915_163a_cea5); + assert_eq!(get_u64(&inventory, 0), COMPUTE_RUNTIME_BUFFER_ADDRESSES[0]); + assert_eq!( + get_u64(&inventory, inventory.len() - 8), + COMPUTE_RUNTIME_BUFFER_ADDRESSES[20] + 0xff000 + ); + + let mut dispatch = [0u8; COMPUTE_DISPATCH_RECORD_SIZE]; + encode_compute_dispatch_record(&mut dispatch).unwrap(); + let packed = get_u64(&dispatch, 0); + assert_eq!((packed & ((1 << 41) - 1)) << 7, COMPUTE_FLIST_PAGE_LIST_ADDRESS); + // Compare the capture only after undoing the packed VA relocation. + put_u64(&mut dispatch, 0, packed - (COMPUTE_FLIST_RELOCATION >> 7)); + assert_eq!(fnv1a64(&dispatch), 0x12a0_cf17_a88d_a788); + } + + #[test] + fn secondary_control_opening_is_the_single_native_record() { + let mut opening = vec![0; control_opening_size(InstanceRole::Secondary)]; + encode_control_opening(InstanceRole::Secondary, &mut opening).unwrap(); + + assert_eq!(control_opening_producer(InstanceRole::Secondary), 1); + for slot in 0..CONTROL_OPENING_SECONDARY_RECORDS { + let base = slot * CONTROL_RECORD_SIZE; + let expected = if slot < core::cmp::min(3, CONTROL_OPENING_SECONDARY_RECORDS) { + CONTROL_OPENING_OPCODE_SECONDARY + } else { + CONTROL_OPENING_SECONDARY_CONTINUE_OPCODE + }; + assert_eq!(get_u32(&opening, base), expected); + assert!(opening[base + 4..base + CONTROL_RECORD_SIZE] + .iter() + .all(|byte| *byte == 0)); + } + } + + #[test] + fn generic_control_support_keeps_descriptor_and_runtime_identity() { + let mut shared = vec![0; CONTROL_SHARED_OBJECT_SIZE]; + encode_control_shared(&mut shared).unwrap(); + + assert_eq!(get_u64(&shared, 0x00), 1); + assert_eq!(get_u64(&shared, 0x10), 1); + assert_eq!(get_u64(&shared, 0x18), 0x0004_0000_0000_0070); + assert_eq!(get_u64(&shared, 0x20), 0x0000_1100_0000_0000); + assert_eq!(get_u64(&shared, 0x28), 0x0000_1100_0000_0000); + assert_eq!(get_u64(&shared, 0x30), CONTROL_OPERAND_TABLE_ADDRESS); + assert_eq!( + get_u32(&shared, CONTROL_SHARED_CURSOR_OFFSET), + CONTROL_SHARED_CURSOR_BEFORE + ); + assert_eq!(get_u64(&shared, 0x4c), CONTROL_SHARED_INNER_ADDRESS); + assert_eq!(get_u64(&shared, 0x60), 2); + assert_eq!(CONTROL_SHARED_INNER_PRESENTED, 2); + } + + #[test] + fn compute_partial_support_preserves_hardware_proven_compact_state() { + let mut shared = vec![0; CONTROL_SHARED_OBJECT_SIZE]; + encode_compute_partial_opening_control_shared(&mut shared).unwrap(); + + assert_eq!(get_u64(&shared, 0x00), 1); + assert_eq!(get_u32(&shared, 0x08), u32::MAX); + assert_eq!(get_u32(&shared, 0x10), 2); + assert_eq!(get_u64(&shared, 0x20), 0x0000_1900_0000_0000); + assert_eq!(get_u64(&shared, 0x28), 0x0000_1900_0000_0000); + assert_eq!(get_u64(&shared, 0x30), COMPUTE_FLIST_RUN_TABLE_ADDRESS); + assert_eq!( + get_u32(&shared, CONTROL_SHARED_CURSOR_OFFSET), + COMPUTE_PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE + ); + assert_eq!(COMPUTE_PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE, 0x18); + assert_eq!( + get_u64(&shared, 0x4c), + PARTIAL_OPENING_CONTROL_SHARED_INNER_ADDRESS + ); + assert_eq!(get_u64(&shared, 0x60), 3); + } + + #[test] + fn first_partial_support_has_distinct_address_cursor_and_inner_page() { + let mut shared = vec![0; CONTROL_SHARED_OBJECT_SIZE]; + encode_partial_opening_control_shared(&mut shared).unwrap(); + + assert_eq!(PARTIAL_OPENING_CONTROL_SHARED_ADDRESS, 0xffff_fc20_c082_8000); + assert_eq!(get_u64(&shared, 0x00), 1); + assert_eq!(get_u32(&shared, 0x08), 0); + assert_eq!(get_u32(&shared, 0x0c), 0); + assert_eq!(get_u64(&shared, 0x10), 2); + assert_eq!(get_u64(&shared, 0x20), 0x0000_1100_0000_0000); + assert_eq!(get_u64(&shared, 0x28), 0x0000_1100_0000_0000); + assert_eq!(get_u64(&shared, 0x30), CONTROL_OPERAND_TABLE_ADDRESS); + assert_eq!( + get_u32(&shared, CONTROL_SHARED_CURSOR_OFFSET), + PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE + ); + assert_eq!( + get_u64(&shared, 0x4c), + PARTIAL_OPENING_CONTROL_SHARED_INNER_ADDRESS + ); + assert_eq!(get_u64(&shared, 0x60), 3); + assert_eq!(PARTIAL_OPENING_CONTROL_SHARED_INNER_PRESENTED, 0); + + encode_render_control_shared(&mut shared).unwrap(); + assert_eq!(get_u64(&shared, 0x00), CONTROL_SHARED_INNER_ADDRESS); + assert_eq!(get_u32(&shared, 0x08), 0); + assert_eq!(get_u64(&shared, 0x10), 2); + assert_eq!(get_u64(&shared, 0x14), 0x70_0000_0000); + assert_eq!(get_u64(&shared, 0x30), CONTROL_OPERAND_TABLE_ADDRESS); + assert_eq!(get_u64(&shared, 0x4c), CONTROL_SHARED_INNER_ADDRESS); + assert_eq!(get_u64(&shared, 0x60), 3); + } + + #[test] + fn presized_render_control_shared_matches_apple_initial_3d_pool() { + let mut shared = [0u8; CONTROL_SHARED_OBJECT_SIZE]; + encode_presized_render_control_shared(&mut shared, 22).unwrap(); + // 25G83 populateInitialFirmwareState for the 3D pool: 0x1600 pages + // at +0x24 and +0x2c, run cursor 0xb0. + assert_eq!(get_u64(&shared, 0x20), 0x0000_1600_0000_0000); + assert_eq!(get_u64(&shared, 0x28), 0x0000_1600_0000_0000); + assert_eq!(get_u32(&shared, CONTROL_SHARED_CURSOR_OFFSET), 0xb0); + assert_eq!(get_u64(&shared, 0x00), CONTROL_SHARED_INNER_ADDRESS); + assert_eq!(get_u64(&shared, 0x4c), CONTROL_SHARED_INNER_ADDRESS); + assert_eq!(get_u64(&shared, 0x60), 3); + } + + #[test] + fn first_partial_operand_table_is_published_between_control_and_work() { + let mut table = vec![0xff; CONTROL_OPERAND_TABLE_SIZE]; + encode_partial_opening_operand_table_pre_control(&mut table).unwrap(); + assert!(table.iter().all(|byte| *byte == 0)); + + encode_partial_opening_operand_table_post_control(&mut table).unwrap(); + for index in 0..PARTIAL_OPENING_CONTROL_OPERAND_ENTRY_COUNT { + assert_eq!( + get_u64(&table, index * CONTROL_OPERAND_ENTRY_STRIDE), + CONTROL_OPERAND_ENTRY_FLAG + | (CONTROL_OPERAND_BUFFER_BASE + + index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + ); + } + assert!(table[PARTIAL_OPENING_CONTROL_OPERAND_ENTRY_COUNT + * CONTROL_OPERAND_ENTRY_STRIDE..] + .iter() + .all(|byte| *byte == 0)); + + encode_control_operand_table(&mut table).unwrap(); + + for index in 0..CONTROL_OPERAND_ENTRY_COUNT { + assert_eq!( + get_u64(&table, index * CONTROL_OPERAND_ENTRY_STRIDE), + CONTROL_OPERAND_ENTRY_FLAG + | (COMPUTE_FLIST_BUFFER_BASE + + index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + ); + } + assert!(table[CONTROL_OPERAND_ENTRY_COUNT * CONTROL_OPERAND_ENTRY_STRIDE..] + .iter() + .all(|byte| *byte == 0)); + } + + #[test] + fn a_null_producer_on_a_populated_channel_fails_closed() { + // A populated work channel with a null producer (state 2) makes the + // firmware fault on address zero; encode must refuse it. + let mut table = channels(InstanceRole::Primary); + table[0].states[CHANNEL_STATE_PRODUCER] = 0; + let views = region_views(); + let mut out = vec![0; MAIN_CONFIG_SIZE]; + assert_eq!( + encode_main_config( + InstanceRole::Primary, + &shared(), + &primary(), + &table, + Some(&views), + &mut out, + ), + Err(InitdataError::ChannelTableShape) + ); + } + + #[test] + fn role_specific_root_fields_fail_closed() { + // A primary must not carry the secondary-only extras (its root is only + // 0xb8 bytes and cannot even hold them). + let mut inst = primary(); + inst.secondary_extras = [KERN + 0x1000, 0]; + let mut out = vec![0; ROOT_SIZE_PRIMARY]; + assert_eq!( + encode_root(InstanceRole::Primary, &shared(), &inst, &mut out), + Err(InitdataError::RoleSpecificAddress) + ); + // A secondary must not carry a status-B pointer. + let mut inst = secondary(); + inst.status_b = KERN + 0x1000; + let mut out = vec![0; ROOT_SIZE_SECONDARY]; + assert_eq!( + encode_root(InstanceRole::Secondary, &shared(), &inst, &mut out), + Err(InitdataError::RoleSpecificAddress) + ); + } + + #[test] + fn wrong_root_buffer_size_fails_closed() { + // The primary root is 0xb8, not 0xc8; a secondary-sized buffer is wrong. + let mut out = vec![0; ROOT_SIZE_SECONDARY]; + assert_eq!( + encode_root(InstanceRole::Primary, &shared(), &primary(), &mut out), + Err(InitdataError::BufferSize) + ); + } + + fn register_windows() -> Vec { + vec![ + RegisterWindow { + slot: 0, + phys: 0x2_0000_0000, + device_va: HW_REGISTER_APERTURE.start, + size: 0x4000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 5, + phys: 0x2_1000_0000, + device_va: HW_REGISTER_APERTURE.start + 0x1_0000, + size: 0x8000, + unk_18: 0, + flag: 2, + }, + ] + } + + fn completion_aliases() -> [G17PKsmCompletionAliases; 2] { + [ + G17PKsmCompletionAliases { + low: G17P_KSM_COMPLETION_LOW_VAS[0], + high: shared().hw_data_bundle + + G17P_KSM_COMPLETION_HIGH_BUNDLE_OFFSETS[0] as u64, + }, + G17PKsmCompletionAliases { + low: G17P_KSM_COMPLETION_LOW_VAS[1], + high: shared().hw_data_bundle + + G17P_KSM_COMPLETION_HIGH_BUNDLE_OFFSETS[1] as u64, + }, + ] + } + + fn sksm_qid_resource_high() -> u64 { + shared().hw_data_bundle + NATIVE_SHARED_CLUSTER_SIZE as u64 + 0x4000 + } + + fn qos_resource_high() -> u64 { + shared().hw_data_bundle + NATIVE_SHARED_CLUSTER_SIZE as u64 + 0x8000 + } + + #[test] + fn hw_data_live_pointers_survive_opaque_constants() { + for qos_resource in [ + 0xffff_fc20_001f_8000, + 0xffff_fc20_013a_c000, + 0xffff_fc20_cabc_4000, + ] { + let aliases = [ + G17PKsmCompletionAliases { + low: 0x70_0240_0000, + high: 0xffff_fc20_c240_0000, + }, + G17PKsmCompletionAliases { + low: 0x70_0280_0000, + high: 0xffff_fc20_c280_0000, + }, + ]; + let windows: Vec<_> = (0..HW_REGISTER_MAP_SLOTS) + .map(|slot| RegisterWindow { + slot, + phys: 0x4_0000_0000 + (slot as u64) * 0x4000, + device_va: HW_REGISTER_APERTURE.start + (slot as u64) * 0x4000, + size: 0x4000, + unk_18: 0, + flag: 2, + }) + .collect(); + let mut hw = vec![0xa5; HW_DATA_SIZE]; + encode_hw_data(&windows, &[], aliases, qos_resource, + sksm_qid_resource_high(), &mut hw).unwrap(); + assert_eq!(get_u64(&hw, G17P_QOS_RESOURCE_LIVE_OFFSET), qos_resource); + assert_eq!(get_u32(&hw, 0x26b4), 1); + assert_eq!(get_u32(&hw, 0x26b8), 1); + for (aliases, ordinal) in aliases.iter().zip(G17P_KSM_COMPLETION_PRESENT_ORDINALS) { + let base = G17P_KSM_COMPLETION_TABLE_OFFSET + + ordinal as usize * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE; + assert_eq!(get_u64(&hw, base), aliases.low); + assert_eq!(get_u64(&hw, base + 8), aliases.high); + } + for window in windows { + let base = HW_REGISTER_MAP_OFFSET + window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + assert_eq!(get_u64(&hw, base), window.phys); + assert_eq!(get_u64(&hw, base + 8), window.device_va); + } + } + } + + #[test] + fn hw_data_opaque_runs_do_not_overlap_live_fields() { + let live = [ + HW_REGISTER_MAP_OFFSET + ..HW_REGISTER_MAP_OFFSET + HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE, + G17P_KSM_COMPLETION_TABLE_OFFSET + ..G17P_KSM_COMPLETION_TABLE_OFFSET + 4 * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE, + G17P_QOS_RESOURCE_LIVE_OFFSET..G17P_QOS_RESOURCE_LIVE_OFFSET + 8, + ]; + for (offset, data) in HW_DATA_OPAQUE_RUNS { + for range in &live { + assert!(*offset + data.len() <= range.start || *offset >= range.end, + "opaque range {offset:#x}..{:#x} overlaps live {range:#x?}", + offset + data.len()); + } + } + } + + #[test] + fn hw_data_size_acceptance_completion_table_and_firmware_state() { + assert_eq!(HW_DATA_SIZE, 0x3db4); + let windows = register_windows(); + let flag_only = [(3usize, 2u32), (7, 2)]; + let aliases = completion_aliases(); + let mut hw = vec![0xa5; HW_DATA_SIZE]; + let qos_resource = qos_resource_high(); + let sksm_qid_resource = sksm_qid_resource_high(); + encode_hw_data( + &windows, + &flag_only, + aliases, + qos_resource, + sksm_qid_resource, + &mut hw, + ) + .unwrap(); + assert_eq!(hw.len(), 0x3db4); + + // Chip id at +0xe90. + assert_eq!(get_u32(&hw, 0xe90), 0x8140); + + // Register-mapping array: 0x28 stride at +0x640. + let base0 = HW_REGISTER_MAP_OFFSET; + assert_eq!(get_u64(&hw, base0), 0x2_0000_0000); + assert_eq!(get_u64(&hw, base0 + 0x08), HW_REGISTER_APERTURE.start); + assert_eq!(get_u32(&hw, base0 + 0x10), 0x4000); + assert_eq!(get_u32(&hw, base0 + 0x14), 0x4000); + assert_eq!(get_u32(&hw, base0 + 0x20), 2); + // A declared-but-unmapped slot carries only its flag. + let base3 = HW_REGISTER_MAP_OFFSET + 3 * HW_REGISTER_MAP_ENTRY_SIZE; + assert_eq!(get_u64(&hw, base3), 0); + assert_eq!(get_u32(&hw, base3 + 0x20), 2); + + // The +0x258e acceptance block — the one block firmware requires. + assert_eq!( + u16::from_le_bytes(hw[0x258e..0x2590].try_into().unwrap()), + 12 + ); + for group in [0x2590usize, 0x25a0] { + for (index, value) in [0u32, 1, 1, 1].iter().enumerate() { + assert_eq!(get_u32(&hw, group + index * 4), *value); + } + } + for channel in 0..12 { + assert_eq!(get_u32(&hw, 0x25b4 + channel * 4), 0xffff_ffff); + } + for offset in [0x25f4usize, 0x2600, 0x2608] { + assert_eq!(get_u32(&hw, offset), 1); + } + // +0x2630 is firmware state and must start zero even though the buffer + // began dirty (0xa5). + assert_eq!(hw[0x2630], 0); + + // Four inline descriptors begin at +0x2614; only ordinals 0/2 exist. + assert_eq!(get_u32(&hw, G17P_KSM_COMPLETION_TABLE_PREFIX_OFFSET), 0x100); + for ordinal in [1usize, 3] { + let base = G17P_KSM_COMPLETION_TABLE_OFFSET + + ordinal * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE; + assert_eq!(&hw[base..base + 0x20], &[0; 0x20]); + } + for (aliases, ordinal) in aliases.iter().zip(G17P_KSM_COMPLETION_PRESENT_ORDINALS) { + let base = G17P_KSM_COMPLETION_TABLE_OFFSET + + ordinal as usize * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE; + assert_eq!(get_u64(&hw, base), aliases.low); + assert_eq!(get_u64(&hw, base + 0x08), aliases.high); + assert_eq!(get_u32(&hw, base + 0x10), 0x800); + assert_eq!(get_u32(&hw, base + 0x14), 0x40); + assert_eq!(get_u32(&hw, base + 0x18), ordinal); + assert_eq!(get_u32(&hw, base + 0x1c), 0); + } + assert_eq!(get_u64(&hw, G17P_SKSM_QID_CONFIG_2700_OFFSET), 0); + assert_eq!(get_u64(&hw, G17P_QOS_RESOURCE_LIVE_OFFSET), qos_resource); + assert_eq!(get_u32(&hw, G17P_SKSM_QID_CONFIG_2708_OFFSET), 0); + assert_eq!(get_u64(&hw, G17P_QOS_RESOURCE_OFFSET), 0); + assert_eq!(get_u64(&hw, G17P_SKSM_QID_RESOURCE_OFFSET), 0); + + // The recorded opaque runs landed (spot check the two 0xffffffff tails). + assert_eq!(get_u32(&hw, 0x3d98), 0xffff_ffff); + assert_eq!(get_u32(&hw, 0x3db0), 0xffff_ffff); + + // Both measured ladders and their voltage columns are carried. + assert!(PERF_TABLE_VALUES_ENCODED); + assert_eq!(get_u32(&hw, HW_PERF_GROUP_BASES[0]), 0); + assert_eq!(get_u32(&hw, HW_PERF_GROUP_BASES[0] + 10 * 4), 1470); + assert_eq!(get_u32(&hw, HW_PERF_GROUP_BASES[1] + 10 * 4), 1326); + assert_eq!(get_u32(&hw, HW_PERF_GROUP_BASES[0] + 0x40), 125); + assert_eq!(get_u32(&hw, HW_PERF_GROUP_BASES[1] + 0x40), 125); + } + + #[test] + fn hw_data_rejects_out_of_aperture_and_duplicate_slots() { + let aliases = completion_aliases(); + let mut hw = vec![0; HW_DATA_SIZE]; + + // A device VA outside the fixed register aperture fails closed. + let mut window = register_windows(); + window[0].device_va = 0x1000; + assert_eq!( + encode_hw_data( + &window, + &[], + aliases, + qos_resource_high(), + sksm_qid_resource_high(), + &mut hw, + ), + Err(InitdataError::RegisterWindow) + ); + + // Two declarations for the same slot fail closed. + assert_eq!( + encode_hw_data( + ®ister_windows(), + &[(0usize, 2u32)], + aliases, + qos_resource_high(), + sksm_qid_resource_high(), + &mut hw, + ), + Err(InitdataError::RegisterWindow) + ); + } + + #[test] + fn region_c_size_and_required_word() { + assert_eq!(REGION_C_SIZE, 0x1000); + let mut region_c = vec![0xa5; REGION_C_SIZE]; + encode_region_c(&mut region_c).unwrap(); + assert_eq!(region_c.len(), 0x1000); + assert_eq!(REGION_C_REQUIRED_VALUE, 0); + assert_eq!(get_u32(®ion_c, 0xe50), REGION_C_REQUIRED_VALUE); + assert_eq!(®ion_c[0xe50..0xe55], &[0; 5]); + // Those same four bytes, framed the way b1 actually reads them: the + // byte at +0xe50 is zero and the GPU-core power-assert counter is the + // UNALIGNED u32 at +0xe51. Nothing in the image reads +0xe50 or +0xe55. + assert_eq!(region_c[REGION_C_REQUIRED_OFFSET], 0); + assert_eq!( + get_u32(®ion_c, REGION_C_GPU_CORE_PWR_ASSERT_COUNTER_OFFSET), + REGION_C_GPU_CORE_PWR_ASSERT_COUNT + ); + assert_eq!(REGION_C_GPU_CORE_PWR_ASSERT_COUNTER_OFFSET, 0xe51); + assert_eq!(get_u32(®ion_c, 0x9b8), 2); + assert_eq!(get_u32(®ion_c, 0x9bc), 40); + assert_eq!(region_c[0xe21], 0); + // A recorded run near the front landed; the tail past the last run is 0. + assert_eq!(get_u32(®ion_c, 0x24), 0x0000_0bb8); + assert_eq!(get_u32(®ion_c, 0x7c), 1); + assert_eq!(get_u32(®ion_c, 0x80), 1); + assert_eq!(get_u32(®ion_c, 0x9c), 0x3000_001d); + assert_eq!(get_u32(®ion_c, 0xa0), 0x3100_0002); + assert_eq!(region_c[REGION_C_SIZE - 1], 0); + } + + #[test] + fn device_control_power_assert_flags_match_b1() { + const CLEAR: [u32; 15] = [ + 0x00, 0x0e, 0x15, 0x16, 0x1b, 0x22, 0x2a, 0x2e, 0x3a, 0x3b, 0x3c, 0x3d, + 0x3e, 0x3f, 0x40, + ]; + for opcode in 0..=0x3f { + assert_eq!( + device_control_asserts_gpu_power(opcode), + Some(!CLEAR[..CLEAR.len() - 1].contains(&opcode)), + "opcode {opcode:#x}", + ); + } + assert_eq!(device_control_asserts_gpu_power(CLEAR[CLEAR.len() - 1]), None); + + // Native host-tally lifecycle for two updateSubmitInfo records, one + // HardwareBufferID release, and one IdlePowerOff record. + let mut tally = REGION_C_GPU_CORE_PWR_ASSERT_COUNT; + for (opcode, expected) in [(0x20, 1), (0x20, 2), (0x2e, 2), (0x0a, 3)] { + if device_control_asserts_gpu_power(opcode).unwrap() { + tally = tally.wrapping_add(1); + } + assert_eq!(tally, expected); + } + } + + #[test] + fn status_block_pre_init_shape() { + assert_eq!(STATUS_BLOCK_SIZE, 0x80); + let mut status = vec![0xa5; STATUS_BLOCK_SIZE]; + encode_status_block(&mut status).unwrap(); + assert_eq!(get_u32(&status, 0x00), 0); + assert_eq!(get_u32(&status, 0x04), 1); + // +0x10 / +0x14 are post-ack firmware state and must be zero at handoff. + assert_eq!(get_u32(&status, 0x10), 0); + assert_eq!(get_u32(&status, 0x14), 0); + } + + #[test] + fn full_status_objects_and_bundle_views_are_carried() { + let mut primary_status_b = vec![0xa5; PRIMARY_STATUS_B_OBJECT_SIZE]; + let fwctl = KERN + 0x5000_0000; + encode_primary_status_b(fwctl, fwctl + CONTROL_RECORD_SIZE as u64, &mut primary_status_b) + .unwrap(); + assert_eq!(get_u32(&primary_status_b, 0x04), 1); + assert_eq!(get_u64(&primary_status_b, PRIMARY_STATUS_B_FWCTL_STATE), fwctl); + assert_eq!( + get_u64(&primary_status_b, PRIMARY_STATUS_B_FWCTL_RING), + fwctl + CONTROL_RECORD_SIZE as u64 + ); + assert_eq!(get_u32(&primary_status_b, PRIMARY_STATUS_B_CONFIG_HEADER), 1); + assert_eq!(primary_status_b[PRIMARY_STATUS_B_CONFIG_OFFSET + 0x10], 0x0f); + + let mut secondary_extra = [0xa5; STATUS_BLOCK_SIZE]; + encode_secondary_root_extra_1(&mut secondary_extra).unwrap(); + assert_eq!(secondary_extra[0x14], 1); + assert_eq!(&secondary_extra[0x50..0x52], &[0x6a, 0x18]); + + let mut bundle = vec![0; NATIVE_SHARED_CLUSTER_SIZE]; + let state = KERN + 0x6000_0000; + encode_bundle_static(state, &mut bundle).unwrap(); + assert_eq!(get_u64(&bundle, 0x8ed8), state); + assert_eq!(bundle[0x80ac], 0x06); + assert_eq!(bundle[BUNDLE_VIEW_OFFSETS[2] + 0xe45], 0xdc); + assert_eq!( + &bundle[BUNDLE_VIEW_OFFSETS[2] + 0x2ad7..BUNDLE_VIEW_OFFSETS[2] + 0x2adb], + &[0xff, 0x7f, 0xff, 0x7f] + ); + } + + #[test] + fn instance_separation_invariants() { + assert!(validate_pair(&primary(), &secondary()).is_ok()); + // The two roots are exactly 0x8000 apart. + assert_eq!(SECONDARY_ROOT_DELTA, 0x8000); + let mut bad = secondary(); + bad.root += 0x4000; + assert_eq!( + validate_pair(&primary(), &bad), + Err(InitdataError::PairRootSeparation) + ); + // The two instances' shared-region halves are 0x40000 apart. + assert_eq!(SECONDARY_SHARED_REGION_DELTA, 0x40000); + assert!(validate_shared_region_bases(KERN, KERN + 0x40000).is_ok()); + assert_eq!( + validate_shared_region_bases(KERN, KERN + 0x8000), + Err(InitdataError::SharedRegionSeparation) + ); + } + + #[test] + fn root_gate_rejects_the_m5_version_word() { + let mut root = encode_root_bytes(InstanceRole::Primary); + assert!(check_firmware_root_gates(&root).is_ok()); + // The M5/G17S word is answered with "Host and FW Interface is mismatched". + put_u64(&mut root, 0x00, 0x0c8d_e339_0728_04c0); + assert_eq!( + check_firmware_root_gates(&root), + Err(InitdataError::InterfaceMismatch) + ); + put_u64(&mut root, 0x00, INITDATA_VERSION_WORD); + put_u32(&mut root, 0x2c, 0); + assert_eq!( + check_firmware_root_gates(&root), + Err(InitdataError::HostMappedAllocationsDisabled) + ); + } + + #[test] + fn evidence_flags_are_honest() { + // The graph the firmware accepts and boots with is grounded. + assert!(GROUNDED_LAYOUT_AVAILABLE); + assert!(PARSE_ACCEPTED_GRAPH_AVAILABLE); + // The measured performance values are carried, while ADT derivation, + // The target-scoped bundle and status-B bytes are carried too. + assert!(!ALL_HW_GLOBALS_VALUES_GROUNDED); + assert!(PERF_TABLE_VALUES_ENCODED); + assert!(BUNDLE_VIEW_CONTENT_CARRIED); + assert!(PRIMARY_STATUS_B_FULL_OBJECT_CARRIED); + } + + #[test] + fn recorded_run_tables_match_the_serializer_totals() { + // HWData excludes the seven captured QoS pointer bytes; the adjacent + // state words and the remaining platform runs are preserved. + let hw: usize = HW_DATA_OPAQUE_RUNS.iter().map(|(_, data)| data.len()).sum(); + assert_eq!(HW_DATA_OPAQUE_RUNS.len(), 29); + assert_eq!(hw, 262); + let region_c: usize = REGION_C_RUNS.iter().map(|(_, data)| data.len()).sum(); + assert_eq!(region_c, 180); + // Every run fits its object. + assert!(HW_DATA_OPAQUE_RUNS + .iter() + .all(|(off, data)| off + data.len() <= HW_DATA_SIZE)); + assert!(REGION_C_RUNS + .iter() + .all(|(off, data)| off + data.len() <= REGION_C_SIZE)); + // No opaque run touches firmware-state +0x2630. + assert!(HW_DATA_OPAQUE_RUNS.iter().all(|(off, data)| { + !(*off <= HW_FIRMWARE_STATE_OFFSET && HW_FIRMWARE_STATE_OFFSET < *off + data.len()) + })); + } + + /// Pins the remaining documented ABI/geometry constants to their grounded + /// values. They describe parts of the graph this offline module records but + /// does not yet encode (the control-record framing and the full primary + /// status-B object); asserting them keeps the + /// recorded facts under test even before the encoders reach them. + #[test] + fn documented_constants_have_their_grounded_values() { + assert_eq!(ROOT_ZERO_10, 0x10); + assert_eq!(REGION_A_SIZE, 0x4000); + assert_eq!(HW_DATA_BUNDLE_SIZE, 0xc000); + assert_eq!(REGION_C_REQUIRED_OFFSET, 0xe50); + assert_eq!(STATUS_BLOCK_POST_ACK_OFFSETS, [0x10, 0x14]); + assert_eq!(PRIMARY_STATUS_B_OBJECT_SIZE, 0xeb00); + // Device-control framing and the ownerless-UMA-free opening prefixes. + assert_eq!(CONTROL_RECORD_SIZE, 0x40); + assert_eq!(CONTROL_OPENING_PRIMARY_PRODUCER, 1); + assert_eq!(COMPUTE_RUNTIME_CONTROL_RECORDS, 65); + assert_eq!(COMPUTE_RUNTIME_CONTROL_FINAL_COUNTER, 66); + assert_eq!(CONTROL_OPENING_SECONDARY_PRODUCER, 1); + // Channel state-block stride within the compact grid. + assert_eq!(CHANNEL_STATE_STRIDE, 0x40); + // Performance-table geometry and measured values. + assert_eq!(HW_PERF_LADDER_ENTRIES, 11); + assert_eq!(HW_PERF_GROUP_REPEATS, [16, 1]); + assert_eq!(HW_PERF_VOLTAGE_DELTA, 0x40); + assert_eq!(HW_PERF_MEMORY_VOLTAGE_DELTA, 0x440); + assert_eq!(HW_PERF_STATE_BLOCK_STRIDE, 0x40); + assert_eq!(HW_PERF_FREQ_LADDER_B, 0x1808); + assert_eq!(HW_PERF_SCALE_LADDER_B, 0x1848); + assert_eq!(HW_PERF_RELATIVE_LADDER_A, 0x18c8); + assert_eq!(HW_PERF_RELATIVE_LADDER_B, 0x1908); + assert_eq!(HW_PERF_INDEX_MAP_A, 0x19c8); + assert_eq!(HW_PERF_INDEX_MAP_B, 0x1a08); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/m5_render.rs b/drivers/gpu/drm/asahi/g18/fw/m5_render.rs new file mode 100644 index 0000000000000..0bb6f2b149c52 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/m5_render.rs @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: MIT +//! M5/G17G graphics register construction from owned render resources. +//! +//! Common resource lowering is shared with G17P. Register order, tile packing +//! and per-stage controls follow the native 25G76 M5 reference. This codec is +//! not a claim that Linux TA/3D submission or presentation is qualified. + +use crate::g18::fw::render_params::{ + self as g17_render, G17pRenderParameters, RegisterWrite, RenderBuildError, RenderDescriptorMetadata, + RenderDescriptorObjects, +}; + +#[cfg(test)] +#[path = "m5_render_test.rs"] +mod tests; + +pub(crate) const TA_REGISTER_COUNT: usize = 82; +pub(crate) const FRAGMENT_REGISTER_COUNT: usize = 90; + +/// Encode the TA parameter-manager metrics slot in the compact GART window. +/// The native 25G76 slot at 0x1000080040 produces register 1c910 = 0x80041: +/// the low bit enables the record; the remaining bits address its 64 bytes. +pub(crate) fn parameter_buffer_metrics_record( + scene_metrics: u64, +) -> Result { + let offset = scene_metrics.checked_sub(0x10_0000_0000) + .ok_or(M5RenderBuildError::OwnershipMismatch)?; + if offset >= (1u64 << 32) || offset & 0x3f != 0 { + return Err(M5RenderBuildError::OwnershipMismatch); + } + Ok(offset | 1) +} + +/// Initial M5 PB slot with fresh ring cursors. The owning job chooses the +/// hardware slot separately and retains the page-list mapping. +pub(crate) fn initial_parameter_buffer_descriptor( + page_list: u64, + page_count: u32, +) -> Result<[u32; 4], M5RenderBuildError> { + if page_list == 0 || page_list >= (1u64 << 42) || page_list & 0x7f != 0 + || page_count == 0 || page_count >= (1 << 22) { + return Err(M5RenderBuildError::OwnershipMismatch); + } + // 25G76 halLoadPBDesc, 89de844..89de8dc. The descriptor has four + // dwords; +0's low flags and both cursors start at zero. Native's + // untagged user address is translated into its PB segment encoding. + let segment = ((page_list + 0x70_0000_0000) & 0x70_0000_0000) >> 8; + Ok([((page_list >> 4) as u32) & !7, + (segment as u32 & 0xf000_0000) | page_count, 0, 0]) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5RenderBuildError { + Common(RenderBuildError), + SharedRegisterLayout, + TileCountOutOfRange, + OverlappingTileFlags, + DispatchBitAlreadySet, + InvalidContext, + OwnershipMismatch, + InvalidUscState, +} + +impl From for M5RenderBuildError { + fn from(error: RenderBuildError) -> Self { + Self::Common(error) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderStageControls { + /// Native host descriptor +0x1120 (TA) / +0x7e8 (fragment). + pub(crate) resource_flags: u64, + /// Explicit per-stage d419/d429 input, independent of the work key. + pub(crate) resource_control: u64, + /// Native +0x1128 / +0x7f0; a zero source selects the native 0x60 fallback. + pub(crate) ptc_max: u32, + pub(crate) work_key: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderControls { + pub(crate) ta: M5RenderStageControls, + pub(crate) fragment: M5RenderStageControls, + pub(crate) ta_tile_control: u64, + /// Base value for the two 1c898 writes. Bit zero is supplied by the codec. + pub(crate) ta_dispatch_control: u64, + pub(crate) fragment_raster_control: u64, + /// Non-geometry bits for 16068. Never includes tile counts, utile or layers. + pub(crate) fragment_tile_flags: u64, + /// M5 adds this second pair of store-program bindings at 153b1/153b9. + pub(crate) store_pipeline_2_bind: u64, + pub(crate) store_pipeline_2: u64, + pub(crate) partial_store_pipeline_2_bind: u64, + pub(crate) partial_store_pipeline_2: u64, + pub(crate) partial_tile_flags: u64, +} + +/// Per-submission USC private-memory accounting, separate from user status +/// registers and queue completion stamps. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderUscState { + pub(crate) control: u64, + /// AGXUMADataRec +0x50. Native allocatePagesEnd derives this from pool + /// sizes and 4-KiB allocation units; it is a byte count, not a GPU VA. + pub(crate) required_bytes: u64, + /// Capacity actually backed by this job's retained USC allocation pages. + pub(crate) allocated_bytes: u64, + /// A 64-byte slot in the firmware-visible statistics ring. + pub(crate) statistics: u64, +} + +impl M5RenderUscState { + fn valid(self) -> bool { + let firmware_range = |address: u64, size: u64| { + address >= 0xffff_fc00_0000_0000 + && address & 0x3f == 0 + && address.checked_add(size).is_some() + }; + firmware_range(self.control, 0x80) + && firmware_range(self.statistics, 0x40) + && self.required_bytes != 0 + && self.required_bytes <= self.allocated_bytes + && self.allocated_bytes & 0xfff == 0 + } +} + +/// Fill the native USC fields only after the caller owns the pool and the +/// statistics slot. Other runtime fields still need queue/link publication. +/// Validate both stages before modifying either descriptor. +pub(crate) fn apply_usc_submission_state( + ta: &mut [u8], + fragment: &mut [u8], + ta_state: M5RenderUscState, + fragment_state: M5RenderUscState, +) -> Result<(), M5RenderBuildError> { + if ta.len() < g17_render::TA_DESCRIPTOR_SIZE + || fragment.len() < g17_render::FRAGMENT_DESCRIPTOR_SIZE + { + return Err(RenderBuildError::BufferTooSmall.into()); + } + if !ta_state.valid() || !fragment_state.valid() { + return Err(M5RenderBuildError::InvalidUscState); + } + // 25G76 AGXTAChannelSKSM::submitBuffer: 8a244c4..8a244ec. + // AGX3DChannelSKSM::submitBuffer takes the corresponding value from + // host +0xba0 = AGXUMADataRec +0x50 at 895b70c..895b718. + for (raw, state, control, required, statistics) in [ + (ta, ta_state, 0x934, 0x93d, 0x945), + (fragment, fragment_state, 0x21ce, 0x21d7, 0x21df), + ] { + put_u64(raw, control, state.control); + put_u64(raw, required, state.required_bytes); + put_u64(raw, statistics, state.statistics); + } + Ok(()) +} + +const EMPTY_REGISTER: RegisterWrite = RegisterWrite { + number: 0, + value: 0, +}; + +struct RegisterList { + values: [RegisterWrite; N], + count: usize, +} + +impl RegisterList { + fn new() -> Self { + Self { + values: [EMPTY_REGISTER; N], + count: 0, + } + } + + fn push(&mut self, number: u32, value: u64) -> Result<(), M5RenderBuildError> { + if self.count == N { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + self.values[self.count] = RegisterWrite { number, value }; + self.count += 1; + Ok(()) + } + + fn finish(self) -> Result<[RegisterWrite; N], M5RenderBuildError> { + if self.count != N { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + Ok(self.values) + } +} + +fn value(registers: &[RegisterWrite], number: u32) -> Result { + registers + .iter() + .find(|r| r.number == number) + .map(|r| r.value) + .ok_or(M5RenderBuildError::SharedRegisterLayout) +} + +const fn ptc_control(maximum: u32) -> u64 { + let maximum = if maximum == 0 { 0x60 } else { maximum }; + ((maximum as u64) << 32) | 0x400020 +} + +fn tile_state(p: &G17pRenderParameters, flags: u64) -> Result { + let tiles_x = (p.width as u64 + 31) / 32; + let tiles_y = (p.height as u64 + 31) / 32; + if !(1..=1024).contains(&tiles_x) || !(1..=1024).contains(&tiles_y) { + return Err(M5RenderBuildError::TileCountOutOfRange); + } + // 25G76 fragment generator: bfi at 89dcb98, 89dcbb8 and 89dcbbc. + let geometry_mask = (0xfffff_u64 << 42) | (0xf_u64 << 38) | (1_u64 << 32); + if flags & geometry_mask != 0 { + return Err(M5RenderBuildError::OverlappingTileFlags); + } + Ok(flags + | ((tiles_x - 1) << 42) + | ((tiles_y - 1) << 52) + | ((p.utile_config & 0xf000) << 26) + | if p.layers > 1 { 1_u64 << 32 } else { 0 }) +} + +fn tile_blocks(pixels: u32) -> u64 { + let tiles = (pixels as u64 + 31) / 32; + let first = (((tiles + 3) / 4) + 3) & !3; + // Native TA host +1208/+120c supplies 0x40200c for the reference: + // boundaries 12, 8, 4 in ten-bit fields, versus G17P's nine-bit fields. + (3 * first) | ((2 * first) << 10) | (first << 20) +} + +pub(crate) fn build_ta_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], M5RenderBuildError> { + // M5's captured register program addresses heap metadata directly. + // G17P's layered path now uses a separate layer table in these slots. + let mut common_parameters = *p; + common_parameters.layermeta = p.heapmeta; + let common = g17_render::build_ta_registers(&common_parameters)?; + if controls.ta_dispatch_control & 1 != 0 { + return Err(M5RenderBuildError::DispatchBitAlreadySet); + } + // Validate the M5 tile fields even when only constructing the TA half. + tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterList::new(); + for mut register in common { + match register.number { + 0x0a309 | 0x0a5a1 | 0x0d419 => continue, + 0x1c8f8 => { + out.push(0x0d499, controls.ta.resource_flags)?; + out.push(0x0d419, controls.ta.resource_control)?; + out.push(0x0a5a1, ptc_control(controls.ta.ptc_max))?; + register.value = controls.ta_tile_control; + } + 0x10171 => { + for (number, source) in [(0x1ca50, 0x101b9), (0x1ca58, 0x10149), (0x1ca60, 0x10141)] + { + out.push(number, value(&common, source)?)?; + } + out.push(0x1ca70, tile_blocks(p.width))?; + out.push(0x1ca78, tile_blocks(p.height))?; + out.push(0x1ca68, value(&common, 0x1c081)?)?; + } + 0x1c069 => register.value = tile_blocks(p.width), + 0x1c071 => register.value = tile_blocks(p.height), + 0x1c898 => { + if register.value > 1 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + register.value |= controls.ta_dispatch_control; + } + 0x10799 => out.push(0x1a149, 0)?, + 0x10209 => { + out.push(0x0a739, 0)?; + register.value = controls.ta.work_key; + } + 0x1c9f0 | 0x14320 => register.value = controls.ta.work_key, + 0x14308 => register.value = p.usc_flist_hardware_buffer_id as i32 as i64 as u64, + _ => {} + } + out.push(register.number, register.value)?; + if register.number == 0x01740 { + out.push(0x0a739, 0)?; + } + } + out.finish() +} + +pub(crate) fn build_fragment_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], M5RenderBuildError> { + // M5's captured register program addresses heap metadata directly. + // G17P's layered path now uses a separate layer table in these slots. + let mut common_parameters = *p; + common_parameters.layermeta = p.heapmeta; + let common = g17_render::build_fragment_registers(&common_parameters)?; + let geometry = tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterList::new(); + let mut dispatch_writes = 0; + let mut tile_writes = 0; + for mut register in common { + match register.number { + 0x0a301 | 0x15109 | 0x15101 | 0x160a8 | 0x0a5a9 | 0x0d429 => continue, + 0x100b1 => register.number = 0x16469, + 0x100d9 => register.number = 0x16471, + 0x100b8 => register.number = 0x16138, + 0x10791 => register.value = controls.fragment_raster_control, + 0x15321 => { + out.push(0x0d491, controls.fragment.resource_flags)?; + out.push(0x0d429, controls.fragment.resource_control)?; + out.push(0x0a5a9, ptc_control(controls.fragment.ptc_max))?; + } + 0x16068 => { + tile_writes += 1; + if tile_writes == 1 { + register.value = geometry; + } + } + 0x16020 => { + dispatch_writes += 1; + if dispatch_writes == 3 { + if register.value != 0 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + continue; + } + } + 0x160e0 => out.push(0x1a151, 0)?, + 0x10211 => { + out.push(0x0a741, 0)?; + register.value = controls.fragment.work_key; + } + 0x10420 => register.value = controls.fragment.work_key, + 0x14048 => register.value = p.usc_flist_hardware_buffer_id as i32 as i64 as u64, + _ => {} + } + out.push(register.number, register.value)?; + if register.number == 0x15381 { + out.push(0x153b1, controls.store_pipeline_2_bind)?; + out.push(0x153b9, controls.store_pipeline_2)?; + } + if register.number == 0x01731 { + out.push(0x0a741, 0)?; + } + } + if dispatch_writes != 3 || tile_writes != 2 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + out.finish() +} + +fn partial_registers( + common: &[RegisterWrite], + insert_after: u32, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; N], M5RenderBuildError> { + let mut out = RegisterList::new(); + for register in common { + let bits = if register.number == 0x10039 { + controls.partial_tile_flags + } else { + 0 + }; + out.push(register.number, register.value | bits)?; + if register.number == insert_after { + out.push(0x153b1, controls.partial_store_pipeline_2_bind)?; + out.push(0x153b9, controls.partial_store_pipeline_2)?; + } + } + out.finish() +} + +pub(crate) fn build_partial_store_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 18], M5RenderBuildError> { + partial_registers(&g17_render::partial_store_registers(p), 0x15381, controls) +} + +pub(crate) fn build_partial_resume_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 25], M5RenderBuildError> { + partial_registers(&g17_render::partial_resume_registers(p), 0x15381, controls) +} + +pub(crate) fn build_partial_load_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 12], M5RenderBuildError> { + partial_registers(&g17_render::partial_load_registers(p), 0x15309, controls) +} + +fn work_key( + p: &G17pRenderParameters, + metadata: RenderDescriptorMetadata, +) -> Result { + if metadata.context_id == 0 || metadata.context_id > 255 { + return Err(M5RenderBuildError::InvalidContext); + } + if metadata.ta_hardware_buffer_id != p.ta_hardware_buffer_id { + return Err(M5RenderBuildError::OwnershipMismatch); + } + metadata + .submission_ordinal + .checked_add(metadata.submission_ordinal / 2) + .ok_or(M5RenderBuildError::Common( + RenderBuildError::ArithmeticOverflow, + ))?; + Ok(((metadata.context_id as u64) << 8) | (metadata.command_buffer_state_word & 0x3f) as u64) +} + +fn alias_offset(alias: u64, offset: u64) -> Result { + alias.checked_add(offset).ok_or(M5RenderBuildError::Common( + RenderBuildError::ArithmeticOverflow, + )) +} + +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Construct the M5 TA record's common layout and all register programs. +/// Runtime ownership/completion fields must be populated before publication. +#[inline(never)] +pub(crate) fn build_ta_descriptor_base( + p: &G17pRenderParameters, + controls: &M5RenderControls, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + low_alias: u64, + raw: &mut [u8], +) -> Result<(), M5RenderBuildError> { + let mut controls = *controls; + controls.ta.work_key = work_key(p, metadata)?; + let program = alias_offset(low_alias, 0x60)?; + alias_offset(low_alias, g17_render::TA_DESCRIPTOR_SIZE as u64)?; + let registers = build_ta_registers(p, &controls)?; + g17_render::build_ta_descriptor_base(p, objects, metadata, ®isters, raw)?; + // Native boundaries: main work ends before the final a739/deflake/dispatch. + for (offset, count) in [ + (0x38, TA_REGISTER_COUNT - 3), + (0x3a, TA_REGISTER_COUNT), + (0x3c, TA_REGISTER_COUNT), + ] { + put_u16(raw, offset, count as u16); + } + put_u64(raw, 0x760, program); + Ok(()) +} + +/// Construct the M5 fragment record, including all three partial programs. +/// The shared encoder is pure and does not apply Neo diagnostic overrides. +#[inline(never)] +pub(crate) fn build_fragment_descriptor_base( + p: &G17pRenderParameters, + controls: &M5RenderControls, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + low_alias: u64, + raw: &mut [u8], +) -> Result<(), M5RenderBuildError> { + let mut controls = *controls; + controls.fragment.work_key = work_key(p, metadata)?; + alias_offset(low_alias, g17_render::FRAGMENT_DESCRIPTOR_SIZE as u64)?; + let main = build_fragment_registers(p, &controls)?; + let store = build_partial_store_registers(p, &controls)?; + let resume = build_partial_resume_registers(p, &controls)?; + let load = build_partial_load_registers(p, &controls)?; + g17_render::build_fragment_descriptor_base( + p, + objects, + metadata, + g17_render::G17pFragmentRegisters { + main: &main, + store: &store, + resume: &resume, + load: &load, + }, + raw, + )?; + put_u32(raw, 0x50, 1); + put_u32(raw, 0x90, 1); + for (offset, count) in [ + (0x80, FRAGMENT_REGISTER_COUNT - 3), + (0x82, FRAGMENT_REGISTER_COUNT - 1), + (0x84, FRAGMENT_REGISTER_COUNT - 1), + (0x88, FRAGMENT_REGISTER_COUNT), + ] { + put_u16(raw, offset, count as u16); + } + for (offset, target) in [ + (0x7a0, 0xa0), + (0xec0, 0x7c0), + (0x15e0, 0xee0), + (0x1d00, 0x1600), + ] { + put_u64(raw, offset, alias_offset(low_alias, target)?); + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/m5_render_target.rs b/drivers/gpu/drm/asahi/g18/fw/m5_render_target.rs new file mode 100644 index 0000000000000..c91d8bca493cf --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/m5_render_target.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Render-target allocation geometry and M5 viewport state. + +pub(crate) const PAGE_BYTES: usize = 0x4000; +pub(crate) const AUX_BYTES: usize = 0x8000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct M5RenderTargetLayout { + pub(crate) tilemap_bytes: usize, + pub(crate) heapmeta_offset: usize, + pub(crate) heapmeta_bytes: usize, + pub(crate) target_bytes: usize, + pub(crate) tpc_bytes: usize, +} + +fn align(value: usize, alignment: usize) -> Option { + Some(value.checked_add(alignment - 1)? & !(alignment - 1)) +} + +impl M5RenderTargetLayout { + pub(crate) fn new( + width: u32, height: u32, layers: u16, utile_width: u8, utile_height: u8, + clusters: u32, + ) -> Option { + if !(1..=16384).contains(&width) || !(1..=16384).contains(&height) + || !(1..=2048).contains(&layers) || clusters == 0 + || !matches!((utile_width, utile_height), (32, 32) | (32, 16) | (16, 16)) { + return None; + } + // Same 4x4 macrotile geometry as the register builders. M5 packs its + // boundaries in ten-bit fields; that does not change allocation size. + let tiles_x = (width as usize + 31) / 32; + let tiles_y = (height as usize + 31) / 32; + let per_mtile = align((tiles_x + 3) / 4, 4)? + .checked_mul(align((tiles_y + 3) / 4, 4)?)?; + let utiles = (32 / utile_width as usize) * (32 / utile_height as usize); + let tilemap_bytes = align(5usize.checked_mul(per_mtile)?.checked_mul(utiles)?, 4)? + .checked_mul(16)?.checked_mul(layers as usize)?; + let heapmeta_offset = align(tilemap_bytes, 0x1000)?; + let heapmeta_bytes = if layers > 1 { 0x300 } else { 0x200 }; + let target_bytes = align(heapmeta_offset.checked_add(heapmeta_bytes)?, PAGE_BYTES)?; + let tpc_bytes = align(8usize.checked_mul(per_mtile)?.checked_mul(utiles)? + .checked_mul(16)?.checked_mul(layers as usize)? + .checked_mul(clusters as usize)?, PAGE_BYTES)?; + Some(Self { tilemap_bytes, heapmeta_offset, heapmeta_bytes, target_bytes, tpc_bytes }) + } +} + +fn half_float_bits(value: u32) -> u32 { + // The admitted integer dimensions are exactly representable in binary32. + let highest = 31 - value.leading_zeros(); + ((highest + 126) << 23) | ((value << (23 - highest)) & 0x7fffff) +} + +/// Native M5 deflake page: viewport at +0x9c0, command header 0x3000. +/// G17P uses +0x900/0xc00 and must not supply this page. The eight nonzero +/// dwords match the M5 25G76 resource capture; dimensions are caller-owned. +pub(crate) fn initialize_deflake(page: &mut [u8], width: u32, height: u32) -> Option<()> { + if page.len() != PAGE_BYTES || !(1..=16384).contains(&width) + || !(1..=16384).contains(&height) { return None; } + page.fill(0); + let x = half_float_bits(width); + let y = half_float_bits(height); + for (offset, value) in [(0x9c0, 0x3000), + (0x9c4, 0x80000000 | ((width + 31) / 32 - 1)), + (0x9c8, (height + 31) / 32 - 1), (0x9d0, x), (0x9d4, x), + (0x9d8, y), (0x9dc, y | 0x80000000), (0x9e4, 0x3f800000)] { + page[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + Some(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reference_target_geometry() { + let layout = M5RenderTargetLayout::new(256, 256, 1, 32, 32, 1).unwrap(); + assert_eq!(layout.tilemap_bytes, 0x500); + assert_eq!(layout.heapmeta_offset, 0x1000); + assert_eq!(layout.heapmeta_bytes, 0x200); + assert_eq!(layout.target_bytes, PAGE_BYTES); + assert_eq!(layout.tpc_bytes, PAGE_BYTES); + } + + #[test] + fn larger_layered_targets_increase_capacity() { + let a = M5RenderTargetLayout::new(4096, 2048, 1, 32, 32, 1).unwrap(); + let b = M5RenderTargetLayout::new(4096, 2048, 2, 16, 16, 2).unwrap(); + assert_eq!(b.tilemap_bytes, a.tilemap_bytes * 8); + assert_eq!(b.tpc_bytes, a.tpc_bytes * 16); + assert!(b.heapmeta_offset >= b.tilemap_bytes); + assert!(b.target_bytes >= b.heapmeta_offset + b.heapmeta_bytes); + } + + #[test] + fn invalid_dimensions_leave_output_untouched() { + let mut page = [0xa5; PAGE_BYTES]; + assert_eq!(initialize_deflake(&mut page, 0, 256), None); + assert!(page.iter().all(|b| *b == 0xa5)); + assert!(M5RenderTargetLayout::new(16385, 1, 1, 32, 32, 1).is_none()); + assert!(M5RenderTargetLayout::new(1, 1, 0, 32, 32, 1).is_none()); + assert!(M5RenderTargetLayout::new(1, 1, 1, 8, 32, 1).is_none()); + assert!(M5RenderTargetLayout::new(1, 1, 1, 32, 32, 0).is_none()); + } + + #[test] + fn m5_viewport_is_distinct_and_dimension_derived() { + for (width, height) in [(256, 256), (1920, 1080), (1, 16384)] { + let mut page = [0xa5; PAGE_BYTES]; + initialize_deflake(&mut page, width, height).unwrap(); + let word = |off| u32::from_le_bytes(page[off..off + 4].try_into().unwrap()); + assert_eq!(word(0x900), 0); + assert_eq!(word(0x9c0), 0x3000); + assert_eq!(word(0x9d0), (width as f32 / 2.0).to_bits()); + assert_eq!(word(0x9d8), (height as f32 / 2.0).to_bits()); + assert_eq!(word(0x9dc), (-(height as f32 / 2.0)).to_bits()); + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/mod.rs b/drivers/gpu/drm/asahi/g18/fw/mod.rs new file mode 100644 index 0000000000000..6a951db464f42 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/mod.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 firmware record and register encoders. Nothing here allocates or publishes. + +#[allow(dead_code)] +pub(crate) mod compute; +#[allow(dead_code)] +pub(crate) mod compute_memory; +#[allow(dead_code)] +pub(crate) mod compute_params; +#[allow(dead_code)] +pub(crate) mod compute_state; +#[allow(dead_code)] +pub(crate) mod config; +#[allow(dead_code)] +pub(crate) mod graphics_payload; +#[allow(dead_code)] +pub(crate) mod initdata; +#[allow(dead_code)] +pub(crate) mod m5_render; +#[allow(dead_code)] +pub(crate) mod m5_render_target; +#[allow(dead_code)] +pub(crate) mod parameter_buffer; +#[allow(dead_code)] +pub(crate) mod render; +#[allow(dead_code)] +pub(crate) mod render_assembly; +#[allow(dead_code)] +pub(crate) mod render_command; +#[allow(dead_code)] +pub(crate) mod render_params; +#[allow(dead_code)] +pub(crate) mod render_passthrough; +#[allow(dead_code)] +pub(crate) mod scheduler; +#[allow(dead_code)] +pub(crate) mod sksm; +#[allow(dead_code)] +pub(crate) mod store; +#[allow(dead_code)] +pub(crate) mod submission; +#[allow(dead_code)] +pub(crate) mod uma; +#[allow(dead_code)] +pub(crate) mod vm_layout; +#[allow(dead_code)] +pub(crate) mod events; +#[allow(dead_code)] +pub(crate) mod rtkit; diff --git a/drivers/gpu/drm/asahi/g18/fw/parameter_buffer.rs b/drivers/gpu/drm/asahi/g18/fw/parameter_buffer.rs new file mode 100644 index 0000000000000..5f256a1e20a0f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/parameter_buffer.rs @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 (26A428) parameter buffer: PB state object, page list and PB-table row. +//! +//! G18 uses AGXParameterManagementVirtual (halNewParameterManager +//! 0xfffffe0008c50c78). Each PB block is 0x80000 bytes (PM +0x70, constant +//! 0xfffffe000714f800) of 32 KiB pages; the page list holds one u32 page +//! number (user GPU VA >> 15) per page. The encoders below reproduce, byte for +//! byte, the original writers executed by the native parameter-buffer audit: +//! AGXHWParamBufferManager::resetHWParamBufferState (0xfffffe0008c5e06c), +//! AGXParameterManagement::buildPageList (0xfffffe0008c5dc98) and +//! AGXParameterManagement::loadPBDesc (0xfffffe0008c60918) with the HAL400 +//! halLoadPBDesc (0xfffffe0008c40b40). +//! +//! Pure encoders only: nothing here allocates, maps or publishes. The kernel +//! owner (G18ParameterBuffer) lives in g18_graphics_queues.rs, which is a +//! g17_resources submodule and can use its MappedObject. + +#![allow(dead_code)] + +/// Native InternalResource size for the PB state (createNewHWPBManager +/// 0xfffffe0008c5fd90). Firmware caches the first 0x90 bytes (b000 0x13764) +/// and writes the GPU context ID to +0x08. +pub(crate) const G18_PB_STATE_BYTES: usize = 0xc0; +pub(crate) const G18_PB_STATE_FIRMWARE_BYTES: usize = 0x90; +pub(crate) const G18_PB_ROW_BYTES: usize = 16; +/// Native 0x4000-byte PB table (g18_resources.rs PBDESC_TABLE_SIZE): 0x400 rows. +pub(crate) const G18_PB_TABLE_ROWS: u32 = 0x400; +pub(crate) const G18_PB_BLOCK_BYTES: u64 = 0x80000; +pub(crate) const G18_PB_PAGE_BYTES: u64 = 0x8000; +pub(crate) const G18_PB_PAGES_PER_BLOCK: u32 = (G18_PB_BLOCK_BYTES / G18_PB_PAGE_BYTES) as u32; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum G18PbError { + Granule, + BlockIndex, + PageListTooSmall, + Slot, +} + +/// Parameter-management fields consumed by the state/page-list writers. +/// Names are native AGXParameterManagement offsets. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18PmConfig { + /// Block ring capacity (+0x148, u32): accelerator PB maximum >> 19. + pub pm_148: u32, + /// Current PB bytes (+0x138); state +0x34 = (pm_138 >> 19) * pm_1d8. + pub pm_138: u64, + /// Pages per block (+0x1d8; device PM config word 0, 0x10 on G18). + pub pm_1d8: u32, + /// Block granule (+0x70), 0x80000 on G18; state +0x58. + pub pm_70: u64, + pub pm_c0: u64, + pub pm_c8: u64, + /// State +0x80 = pm_154 << 4; +0x7c = pm_15c << 4. + pub pm_154: u32, + pub pm_15c: u32, + /// State +0x10 and TA command +0x1c. + pub pm_1d4: u32, +} + +/// Addresses and ring positions of one PB state. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18PbStateInputs { + pub config: G18PmConfig, + /// State +0x20: firmware VA of the page list (converted, identity on G18). + pub page_list_firmware: u64, + /// State +0x28: GPU VA of the page list; also encoded in the PB row. + pub page_list_gpu: u64, + /// State +0x30: page-list resource length word (resource +0x78). + pub page_list_bytes: u32, + /// State +0x64: GPU VA of the 4-byte submitted-operations counter. + pub counter_gpu: u64, + /// State +0x84: GPU VA of the 0x8000-byte discard page. + pub discard_gpu: u64, + /// PB table slot (TA HardwareBufferID), state +0x0c. + pub slot: u32, + /// Block ring positions: first block (PM +0x98 -> +8) and staged end. + pub start_block: u32, + pub end_block: u32, +} + +/// AGXSPBDescRingState, the HWPB manager's +0x34..+0x44 words. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18PbRing { + pub words: [u32; 4], +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn get32(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} + +/// PB state after the native first-TA sequence: resetHWParamBufferState, +/// buildPageList and (when the manager was not loaded before) loadPBDesc. +/// Bytes +0x08..+0x0c (firmware-owned context), +0x1c..+0x20 and +/// +0x90..+0xc0 are not written, as native. +pub(crate) fn encode_g18_pb_state( + out: &mut [u8; G18_PB_STATE_BYTES], + p: &G18PbStateInputs, + load: bool, +) { + let c = &p.config; + // resetHWParamBufferState 0x8c5e094..0x8c5e1e4. + put32(out, 0x38, c.pm_148); + put64(out, 0x3c, 0); + put64(out, 0x44, c.pm_c0); + put64(out, 0x4c, c.pm_c8); + put64(out, 0x20, p.page_list_firmware); + put64(out, 0x28, p.page_list_gpu); + put32(out, 0x30, p.page_list_bytes); + put32(out, 0x34, 0); + put64(out, 0x5c, 0); + put32(out, 0x54, u32::MAX); + put32(out, 0x58, c.pm_70 as u32); + put64(out, 0x64, p.counter_gpu); + put64(out, 0, 0); + out[0x6c..0x7c].fill(0); + put32(out, 0x8c, 0); + put64(out, 0x14, 0); + put32(out, 0x80, c.pm_154.wrapping_shl(4)); + put32(out, 0x7c, c.pm_15c.wrapping_shl(4)); + put64(out, 0x84, p.discard_gpu); + put32(out, 0x10, c.pm_1d4); + // buildPageList 0x8c5dd54..0x8c5dd74 and 0x8c5df3c. + let pages = g18_pb_page_count(c); + put32(out, 0x34, pages); + put32(out, 0x54, pages.wrapping_sub(1)); + put32(out, 0x3c, p.end_block); + put32(out, 0x40, p.start_block); + // loadPBDesc 0x8c609b8 on a manager that was never loaded. + if load { + put32(out, 0x0c, p.slot); + } +} + +/// State +0x34, the PB page count encoded in the row. +pub(crate) fn g18_pb_page_count(c: &G18PmConfig) -> u32 { + ((c.pm_138 >> 19) as u32).wrapping_mul(c.pm_1d8) +} + +/// buildPageList, G18 fast path (granule 0x80000): 16 consecutive page +/// numbers per ring slot from `start` up to, not including, `end` (mod the +/// ring capacity), written from the start of the list. `page_bases[slot]` is +/// that slot's block page base (block user GPU VA >> 15, 25 bits). +pub(crate) fn g18_pb_page_list( + out: &mut [u8], + c: &G18PmConfig, + page_bases: &[u32], + start: u32, + end: u32, +) -> Result { + if c.pm_70 != G18_PB_BLOCK_BYTES { + return Err(G18PbError::Granule); + } + if start >= c.pm_148 || end >= c.pm_148 || page_bases.len() < c.pm_148 as usize { + return Err(G18PbError::BlockIndex); + } + let mut slot = start; + let mut offset = 0usize; + while slot != end { + let base = page_bases[slot as usize]; + for page in 0..G18_PB_PAGES_PER_BLOCK { + let entry = out.get_mut(offset..offset + 4).ok_or(G18PbError::PageListTooSmall)?; + entry.copy_from_slice(&base.wrapping_add(page).to_le_bytes()); + offset += 4; + } + slot = (slot + 1) % c.pm_148; + } + Ok(offset / 4) +} + +/// halLoadPBDesc row. `page_list_gpu`/`page_count` are PB state +0x28/+0x34 +/// at load time; `ring` is the manager's +0x34..+0x44 words (all zero after +/// buildPageList, so zero for every first TA). `old` is the row's previous +/// contents: native keeps bits 1..2 of word 0, 22..27 of word 1, 22..31 of +/// word 2 and 22..30 of word 3. +pub(crate) fn g18_pb_row( + page_list_gpu: u64, + page_count: u32, + ring: G18PbRing, + old: [u8; G18_PB_ROW_BYTES], +) -> [u8; G18_PB_ROW_BYTES] { + let list = page_list_gpu; + let rebase = if list & (1 << 42) == 0 { 0x70_0000_0000 } else { 0 }; + let segment = (rebase + list) & 0x70_0000_0000 | (list >> 3) & 0x80_0000_0000; + let old = [get32(&old, 0), get32(&old, 4), get32(&old, 8), get32(&old, 12)]; + let word0 = ((list >> 4) as u32 & !7) | (old[0] & 7); + let word0 = (word0 & !1) | u32::from(ring.words[3] != 0); + let word1 = ((segment >> 8) as u32 & !0x0fff_ffff) | (old[1] & 0x0fff_ffff); + let word1 = (word1 & !0x3f_ffff) | (page_count & 0x3f_ffff); + let word2 = (old[2] & !0x3f_ffff) | (ring.words[0] & 0x3f_ffff); + let word3 = (old[3] & !0x3f_ffff) | (ring.words[1] & 0x3f_ffff); + let word3 = (word3 & !(1 << 31)) | ((ring.words[2] & 1) << 31); + let mut row = [0; G18_PB_ROW_BYTES]; + for (index, word) in [word0, word1, word2, word3].into_iter().enumerate() { + put32(&mut row, index * 4, word); + } + row +} + +/// A page-list VA without bit 42 is stored minus 64 GiB in a 3-bit segment, +/// so only [64 GiB, 512 GiB) round-trips; stage33c's 0x200_0000 decoded to +/// 0x80_0200_0000 and faulted in the PM (see the H1 parameter-buffer fault notes). +pub(crate) const fn g18_pb_row_address_valid(page_list_gpu: u64) -> bool { + page_list_gpu & 0x7f == 0 + && page_list_gpu & (1 << 42) == 0 + && page_list_gpu >= 0x10_0000_0000 + && page_list_gpu < 0x80_0000_0000 +} + +/// The page-list VA the Parameter Manager reads back from a row with word 1 +/// bit 31 clear. `None` for the bit-42 form, which Linux never programs. +pub(crate) fn g18_pb_row_decoded_address(row: &[u8; G18_PB_ROW_BYTES]) -> Option { + let (word0, word1) = (get32(row, 0), get32(row, 4)); + if word1 & (1 << 31) != 0 { + return None; + } + Some(((u64::from(word1 >> 28) & 7) << 36) + (u64::from(word0 & !7) << 4) + 0x10_0000_0000) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/render.rs b/drivers/gpu/drm/asahi/g18/fw/render.rs new file mode 100644 index 0000000000000..0daeb4af7da47 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/render.rs @@ -0,0 +1,811 @@ +// SPDX-License-Identifier: MIT +//! Pure 26A428 G18 render register lowering over the unchanged M5 builder. +//! +//! See the G18 render-register notes for the oracle, supported envelope and unresolved +//! runtime policy. All compact pointers here are already in the native G18 +//! user-process view. The job, not this codec, chooses/maps that address view. + +use crate::g18::fw::render_params::{G17pRenderParameters, RegisterWrite}; +use crate::g18::fw::m5_render::{self as m5_render, M5RenderControls, M5RenderStageControls}; +use crate::g18::fw::m5_render_target as m5_render_target; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G18RenderError { + M5(m5_render::M5RenderBuildError), + RegisterLayout, + UnsupportedGeometry, + CompactAddress, + UnsupportedFeature, +} + +impl From for G18RenderError { + fn from(e: m5_render::M5RenderBuildError) -> Self { + Self::M5(e) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderStageControls { + /// TA descriptor +1188 | (+1190 << 16); fragment +820 | (+828 << 16). + /// This is NOT the M5 d499/d419/a5a1 tuple. Runtime must supply pool policy. + pub(crate) resource_word: u64, + /// Native literal window triple (TA a109/a129/a149; fragment a101/a121/a141). + /// 8c4f4d4..8c4f56c / 8c3b358..8c3b3f0; no descriptor read. + pub(crate) address_windows: [u64; 3], + /// (TA +12c4 / fragment +154) << 8 | (+12d0 / +988) & 63. + pub(crate) work_key: u64, + /// Optional USC statistics mapping from +1468 / +bb0 pointee chain. + /// Native helper 8c4bdb8: absent -> 0, present -> VA & 7ffffffffc0, + /// OR context id << 56 and bit 0. This is NOT ta/fragment_status | 1. + pub(crate) usc_statistics: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderControls { + pub(crate) ta: G18RenderStageControls, + pub(crate) fragment: G18RenderStageControls, + /// Native TA +12d4/+12d5, device +4d4/+6b9, base 10c08860. + pub(crate) ta_tile_control: u64, + /// Native device +10ac4 and descriptor +1659; bit 0 belongs to dispatch. + pub(crate) ta_dispatch_control: u64, + /// Native fragment +819 and policy/device state; baseline 200000ff03ef. + pub(crate) fragment_raster_control: u64, + /// Native 16068 nongeometry bits; +8cb/+ca0, device +6b0. + pub(crate) fragment_tile_flags: u64, + /// Native fragment +982/+983 and device +4d4/+6b9; baseline 8860. + pub(crate) fragment_tile_control: u64, + /// Native descriptor +ac8 & 0x7f, emitted at 8c3c208..8c3c230. + /// Independent of work-key low bits (+988) and utile dimensions. + pub(crate) fragment_control: u64, + /// Native +770 / register 10051 sample control. This is independent of + /// the application's tile-storage block count used by metadata +4d0. + pub(crate) fragment_sample_control: u64, + /// Native +8cb and device +10aa4; first 160a8, final has bit 5 set. + pub(crate) fragment_dispatch_control: u64, + /// Native +7a0/+7a8. The second EOT binding is independent of +790/+798. + pub(crate) store_pipeline_2_bind: u64, + pub(crate) store_pipeline_2: u64, + /// Native partial helper uses +7b0/+7b8 for both EOT binding pairs. + pub(crate) partial_tile_flags: u64, + /// Native TA loader +11f8/+1200 (second tilemap/layer metadata). + pub(crate) ta_cluster_tilemap: u64, + pub(crate) ta_cluster_layermeta: u64, + /// Native TA loader +1218/+1220/+1228/+1238 (optional cluster resources). + pub(crate) ta_cluster_meta: [u64; 4], + /// Native TA loader +1240/+1248/+1250, copied from target +d8/+e0/+e8. + pub(crate) ta_cluster_control: [u64; 3], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18Program { + pub(crate) registers: [RegisterWrite; N], + pub(crate) rce_count: u16, + pub(crate) command_offset: u32, +} + +impl G18Program { + pub(crate) const fn empty() -> Self { + Self { + registers: [RegisterWrite { + number: 0, + value: 0, + }; N], + rce_count: 0, + command_offset: 0, + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18FragmentPrograms { + pub(crate) main: G18Program<92>, + pub(crate) store: G18Program<17>, + pub(crate) resume: G18Program<24>, + pub(crate) load: G18Program<12>, +} + +struct List<'a, const N: usize> { + program: &'a mut G18Program, + len: usize, +} +impl<'a, const N: usize> List<'a, N> { + fn new(program: &'a mut G18Program) -> Self { + Self { program, len: 0 } + } + fn push(&mut self, number: u32, value: u64) -> Result<(), G18RenderError> { + if self.len == N { + return Err(G18RenderError::RegisterLayout); + } + self.program.registers[self.len] = RegisterWrite { number, value }; + self.len += 1; + Ok(()) + } + fn finish(self, rce_count: u16, command_offset: u32) -> Result<(), G18RenderError> { + if self.len != N { + return Err(G18RenderError::RegisterLayout); + } + self.program.rce_count = rce_count; + self.program.command_offset = command_offset; + Ok(()) + } +} + +/// msaa163: the one multisample render the G18 path admits, Apple's 4x +/// render of run112 workload 8 (the MSAA scope +/// notes, section 2): 32x16 utiles and four samples (register 0x10009 = +/// 0x6002), the rotated-grid sample table 0xeaa26e26 (0x10019 / 3D +0x48), +/// eight-byte samples (3D +0x1fb0 = 8). Only `g18_application` admits it, +/// and only with `g18_admit_msaa4=1`. `p.samples` stays 1: that field is +/// the unused `set_samples` convention (32x32 utiles, Y-doubled geometry), +/// whereas the UAPI already names the halved utile. +pub(crate) const G18_MSAA4_UTILE: (u8, u8) = (32, 16); +pub(crate) const G18_MSAA4_UTILE_CONFIG: u64 = 0x6002; +pub(crate) const G18_MSAA4_MULTISAMPLE_CONTROL: u64 = 0xeaa2_6e26; +pub(crate) const G18_MSAA4_SAMPLES: u8 = 4; +pub(crate) const G18_MSAA4_SAMPLE_SIZE: u8 = 8; +/// Native331 RGBA32 two/four-sample renders use 16-byte samples, with the +/// same geometry and sample control. +pub(crate) const G18_MSAA_WIDE_SAMPLE_SIZE: u8 = 16; +/// Native464 retains three RGBA32 targets: 48-byte samples in a 96-KiB tile. +pub(crate) const G18_MSAA_MRT_SAMPLE_SIZE: u8 = 48; +/// Register 0x10051 / 3D +0x1f38: samples << 16 (native 0x40000 at 4x). +pub(crate) const G18_MSAA4_SAMPLE_CONTROL: u64 = (G18_MSAA4_SAMPLES as u64) << 16; +/// Partial-load list 0x10039 bit 16: native 0x200280 at 1x, 0x210280 at 4x. +pub(crate) const G18_MSAA4_LOAD_TILE_FLAG: u64 = 0x1_0000; + +/// Native326: two samples retain a 32x32 utile and use diagonal positions. +pub(crate) const G18_MSAA2_UTILE: (u8, u8) = (32, 32); +pub(crate) const G18_MSAA2_UTILE_CONFIG: u64 = 0xa001; +pub(crate) const G18_MSAA2_MULTISAMPLE_CONTROL: u64 = 0x44cc; + +pub(crate) fn g18_msaa2(p: &G17pRenderParameters) -> bool { + (p.utile_width, p.utile_height) == G18_MSAA2_UTILE + && p.utile_config == G18_MSAA2_UTILE_CONFIG + && p.multisample_control == G18_MSAA2_MULTISAMPLE_CONTROL + && p.samples == 1 + // Native673 combines the two-sample positions and 32x32 utiles with + // layered target metadata and all five complete register lists. + && (1..=2048).contains(&p.layers) +} + +pub(crate) fn g18_multisample(p: &G17pRenderParameters) -> bool { + g18_msaa2(p) || g18_msaa4(p) +} + +/// Whether `p` is the admitted 4x render (see [`G18_MSAA4_UTILE`]). +pub(crate) fn g18_msaa4(p: &G17pRenderParameters) -> bool { + (p.utile_width, p.utile_height) == G18_MSAA4_UTILE + && p.utile_config == G18_MSAA4_UTILE_CONFIG + && p.multisample_control == G18_MSAA4_MULTISAMPLE_CONTROL + && p.samples == 1 + // Native618 compares layered 32x16 targets and all five register + // lists with the same four-sample positions and partial-load mode. + && (1..=2048).contains(&p.layers) +} + +/// Rasterization samples (3D +0x50): 4 for the admitted 4x render, else the +/// existing `p.samples`. +pub(crate) fn g18_sample_count(p: &G17pRenderParameters) -> u32 { + if g18_msaa4(p) { + u32::from(G18_MSAA4_SAMPLES) + } else if g18_msaa2(p) { + 2 + } else { + u32::from(p.samples) + } +} + +/// 3D +0x54 macro size in utiles, (x << 16) | y: the register builders' +/// macro_size (g17_render.rs geometry). 4x splits the utile in Y, so y +/// doubles (native 0x40004 -> 0x40008); 32x32 renders keep the one-utile +/// formula unchanged. +pub(crate) fn g18_macro_size(p: &G17pRenderParameters) -> u32 { + let tiles_x = (p.width + 31) / 32; + let tiles_y = (p.height + 31) / 32; + let macro_x = ((tiles_x + 3) / 4 + 3) & !3; + let macro_y = ((tiles_y + 3) / 4 + 3) & !3; + if g18_msaa4(p) { + let utiles_x = 32 / u32::from(G18_MSAA4_UTILE.0); + let utiles_y = 32 / u32::from(G18_MSAA4_UTILE.1); + ((macro_x * utiles_x) << 16) | (macro_y * utiles_y) + } else { + (macro_x << 16) | macro_y + } +} + +fn validate(p: &G17pRenderParameters) -> Result<(), G18RenderError> { + if !(1..=16384).contains(&p.width) + || !(1..=16384).contains(&p.height) + || !(1..=2048).contains(&p.layers) + || p.samples != 1 + || (!g18_multisample(p) + && ((p.utile_width, p.utile_height) != (32, 32) + || p.utile_config != 0xa000 + || p.multisample_control != 0x88)) + { + return Err(G18RenderError::UnsupportedGeometry); + } + if p.sampler_count != 0 + || p.sampler_array != 0 + || p.occlusion_query_base & 0xf != 0 + || p.ta_timestamp_start != 0 + || p.ta_timestamp_end != 0 + || p.fragment_timestamp_start != 0 + || p.fragment_timestamp_end != 0 + || p.absolute_pointers + || p.native_ta_registers != 0 + || p.bg_resource_prefix != 0 + || (p.layers == 1 && (p.stencil_aux_buffer != 0 + || p.stencil_stride != 0 || p.stencil_aux_stride != 0)) + || p.stencil_clear_value > 0xff + { + return Err(G18RenderError::UnsupportedFeature); + } + Ok(()) +} + +fn m5_controls(c: &G18RenderControls) -> M5RenderControls { + let stage = |s: G18RenderStageControls| M5RenderStageControls { + resource_flags: 0, + resource_control: 0, + ptc_max: 0, + work_key: s.work_key, + }; + M5RenderControls { + ta: stage(c.ta), + fragment: stage(c.fragment), + ta_tile_control: c.ta_tile_control, + ta_dispatch_control: c.ta_dispatch_control, + fragment_raster_control: c.fragment_raster_control, + fragment_tile_flags: c.fragment_tile_flags, + store_pipeline_2_bind: c.store_pipeline_2_bind, + store_pipeline_2: c.store_pipeline_2, + partial_store_pipeline_2_bind: 0, + partial_store_pipeline_2: 0, + partial_tile_flags: c.partial_tile_flags, + } +} + +/// Convert ONLY compact-address inputs for the shared M5 arithmetic. The +/// emitted values remain the caller's G18 view; no physical alias is invented. +fn m5_parameters(p: &G17pRenderParameters) -> Result { + validate(p)?; + if p.encoder >= 1 << 39 + || p.encoder & 3 != 0 + || [p.deflake_1, p.deflake_2, p.deflake_3] + .iter() + .any(|&v| v > u32::MAX as u64 || v & 3 != 0) + { + return Err(G18RenderError::CompactAddress); + } + let mut m = *p; + m.encoder += 0x10_0000_0000; + m.deflake_1 += 0x10_0000_0000; + m.deflake_2 += 0x10_0000_0000; + m.deflake_3 += 0x10_0000_0000; + Ok(m) +} + +#[inline(never)] +fn common_ta( + p: &G17pRenderParameters, + c: &M5RenderControls, +) -> Result<[RegisterWrite; 82], m5_render::M5RenderBuildError> { + m5_render::build_ta_registers(p, c) +} + +#[inline(never)] +fn common_fragment( + p: &G17pRenderParameters, + c: &G18RenderControls, +) -> Result<[RegisterWrite; 90], m5_render::M5RenderBuildError> { + m5_render::build_fragment_registers(p, &m5_controls(c)) +} + +#[inline(never)] +pub(crate) fn build_g18_ta_program( + p: &G17pRenderParameters, + c: &G18RenderControls, +) -> Result, G18RenderError> { + let mut program = G18Program::empty(); + build_g18_ta_program_into(&mut program, p, c)?; + Ok(program) +} + +/// H1 writes into heap-owned storage rather than nesting return-value arrays. +#[inline(never)] +pub(crate) fn build_g18_ta_program_into(program: &mut G18Program<91>, + p: &G17pRenderParameters, c: &G18RenderControls) -> Result<(), G18RenderError> { + let common = common_ta(&m5_parameters(p)?, &m5_controls(c))?; + let mut out = List::new(program); + for r in &common { + let (mut n, mut v) = (r.number, r.value); + match n { + 0x1748 | 0xd499 | 0xd419 | 0xa5a1 | 0xd471 | 0x14320 | 0x1740 => continue, + 0x1c8f8 => out.push(0xa5e9, c.ta.resource_word)?, + // Fixed J873 device capability, independent of user PPP flags. + // Native device +6b0 bit 16 is emitted as bit 9 (8c4f004). + 0x10121 => v |= 0x200, + 0x1c079 | 0x1c9d8 => { + v = p + .layermeta + .checked_sub(p.context_base) + .ok_or(G18RenderError::CompactAddress)? + } + 0x1a099 | 0x1a0a1 | 0x1a069 | 0x1a071 | 0x1a0c9 | 0x1a0d1 => n -= 0x10, + 0x1a0f1 => n = 0x1a0e1, + 0x1a149 => { + n = 0x1a111; + v = 1; + } + // Native owner +28 masks the TA cycle/scene slot to 64 bytes. + 0x1ca30 | 0x16c39 => v &= !63, + 0x14a1 => n = 0x11201, + 0x14308 => n = 0x10249, + 0x14318 => { + n = 0xa619; + v = c.ta.usc_statistics; + } + _ => {} + } + out.push(n, v)?; + match n { + 0x1c9c8 => { + out.push(0x1c041, c.ta_cluster_tilemap)?; + out.push(0x1c9d0, c.ta_cluster_tilemap)?; + } + 0x1c880 if out.len < 30 => { + out.push(0x1c948, c.ta_cluster_meta[0])?; + out.push(0x1c888, c.ta_cluster_meta[1])?; + out.push(0x1c890, c.ta_cluster_control[0])?; + out.push(0x1c918, c.ta_cluster_control[1])?; + } + 0x1c9d8 => { + out.push(0x1c089, c.ta_cluster_layermeta)?; + out.push(0x1c9e0, c.ta_cluster_layermeta)?; + out.push(0x16c41, c.ta_cluster_meta[2])?; + out.push(0x1ca40, c.ta_cluster_meta[2])?; + out.push(0x1c9a8, c.ta_cluster_control[2])?; + out.push(0x1c920, c.ta_cluster_meta[3])?; + } + 0x10169 => { + for (n, v) in [0xa109, 0xa129, 0xa149].iter().zip(c.ta.address_windows) { + out.push(*n, v)?; + } + } + _ => {} + } + } + // Native descriptor +1608; the final three writes are outside the RCE tag. + out.finish(88, 0x60)?; + Ok(()) +} + +#[inline(never)] +fn fragment_main( + program: &mut G18Program<92>, + p: &G17pRenderParameters, + c: &G18RenderControls, +) -> Result<(), G18RenderError> { + validate(p)?; + let common = common_fragment(p, c)?; + let mut out = List::new(program); + let mut tiles = 0; + for r in &common { + let (mut n, mut v) = (r.number, r.value); + match n { + 0xd491 | 0xd429 | 0xa5a9 | 0xd469 => continue, + 0x1739 => n = 0x8139, + 0x1731 => n = 0x8131, + 0x10791 => { + for (n, v) in [0xa101, 0xa121, 0xa141] + .iter() + .zip(c.fragment.address_windows) + { + out.push(*n, v)?; + } + } + 0x15321 => out.push(0xa5f1, c.fragment.resource_word)?, + 0x16060 => v = p.layermeta, + 0x16138 => v = c.fragment_tile_control, + 0x101e9 => v = c.fragment_control & 0x7f, + 0x10051 => v = c.fragment_sample_control, + 0x1ca28 => v &= !15, + 0x16068 => { + tiles += 1; + out.push( + 0x160a8, + c.fragment_dispatch_control | if tiles == 2 { 0x20 } else { 0 }, + )?; + } + 0x1a0a9 | 0x1a0b1 | 0x1a079 | 0x1a081 | 0x1a0d9 | 0x1a0e1 => n -= 0x10, + 0x1a0f9 => n = 0x1a0e9, + 0x1a151 => { + n = 0x1a119; + v = 1; + } + 0x1499 => n = 0x111f9, + 0x14048 => n = 0x10251, + 0x14080 => { + n = 0xa621; + v = c.fragment.usc_statistics; + } + _ => {} + } + out.push(n, v)?; + } + out.finish(88, 0xa0) +} + +#[inline(never)] +pub(crate) fn build_g18_fragment_programs( + p: &G17pRenderParameters, + c: &G18RenderControls, +) -> Result { + let mut programs = G18FragmentPrograms { + main: G18Program::empty(), + store: G18Program::empty(), + resume: G18Program::empty(), + load: G18Program::empty(), + }; + build_g18_fragment_programs_into(&mut programs, p, c)?; + Ok(programs) +} + +#[inline(never)] +pub(crate) fn build_g18_fragment_programs_into(programs: &mut G18FragmentPrograms, + p: &G17pRenderParameters, c: &G18RenderControls) -> Result<(), G18RenderError> { + fragment_main(&mut programs.main, p, c)?; + fragment_partials(programs, p, c) +} + +#[inline(never)] +fn fragment_partials( + programs: &mut G18FragmentPrograms, + p: &G17pRenderParameters, + c: &G18RenderControls, +) -> Result<(), G18RenderError> { + let mut mc = m5_controls(c); + mc.partial_store_pipeline_2_bind = p.partial_store_pipeline_bind; + mc.partial_store_pipeline_2 = p.partial_store_pipeline; + fn partial( + r: &[RegisterWrite], + offset: u32, + ) -> Result, G18RenderError> { + let mut program = G18Program::empty(); + let mut out = List::new(&mut program); + for r in r { + if r.number != 0x15359 { + out.push(r.number, r.value)?; + } + } + out.finish(N as u16, offset)?; + Ok(program) + } + let mut partial_p = *p; + // G18 helper 8c4c1d0 drops main-pass +8d2/+8d3 flags for a split. + partial_p.tile_config &= 0x1ffff; + partial_p.partial_depth_flags = p.depth_flags; + let mut load_mc = mc; + // Original G18 partial-load helper reads +790/+798 (main EOT), whereas + // store/resume read +7b0/+7b8. M5 used the partial pair for all three. + load_mc.partial_store_pipeline_2_bind = p.store_pipeline_bind; + load_mc.partial_store_pipeline_2 = p.store_pipeline; + programs.store = partial( + &m5_render::build_partial_store_registers(&partial_p, &mc)?, + 0x7c0, + )?; + programs.resume = partial(&m5_render::build_partial_resume_registers(p, &mc)?, 0xee0)?; + programs.load = partial( + &m5_render::build_partial_load_registers(p, &load_mc)?, + 0x1600, + )?; + // msaa163: native 4x keeps tile_config bit 16 in the partial-load list + // (0x10039 = 0x210280 against 0x200280 at 1x, MSAA notes s2.1; the M5 + // 4x load word +0x161c moves the same way). Store/resume do not move. + if g18_multisample(p) { + // Native326 retains the same partial-load flag at two samples. + for register in &mut programs.load.registers { + if register.number == 0x10039 { + register.value |= G18_MSAA4_LOAD_TILE_FLAG; + } + } + } + // Native G18 partial helpers retain main stencil flags and add the + // transfer required by each program when a plane is present. The + // ordinary main list already forwards these fields unchanged. + fn stencil_flags(program: &mut G18Program, + p: &G17pRenderParameters, transfer: u64, store: bool) { + let mask = (1 << 4) | (1 << 8) | (1 << 14) | (1 << 18); + let flags = (p.depth_flags & mask) + | if p.stencil_buffer != 0 { transfer } else { 0 } + | if store && p.stencil_aux_buffer != 0 { 1 << 8 } else { 0 }; + for register in &mut program.registers { + if register.number == 0x15319 { + register.value = (register.value & !mask) | flags; + } + } + } + stencil_flags(&mut programs.store, p, 1 << 18, true); + stencil_flags(&mut programs.resume, p, (1 << 14) | (1 << 18), true); + stencil_flags(&mut programs.load, p, 1 << 14, false); + Ok(()) +} + +pub(crate) fn g18_default_controls(p: &G17pRenderParameters) -> G18RenderControls { + let stage = G18RenderStageControls { + resource_word: 0, + address_windows: [0xffff_0000_0000, 0x70_0000_0000, 0x71_0000_0000], + work_key: 0, + usc_statistics: 0, + }; + G18RenderControls { + ta: stage, + fragment: stage, + ta_tile_control: 0x10c08860, + ta_dispatch_control: 0x700, + fragment_raster_control: 0x200000ff03ef, + fragment_tile_flags: 0x20_0003_6011, + fragment_tile_control: 0x8860, + // Original a/b/g1 user descriptors and working native run 112. + fragment_control: 0x1c, + // Capture parameters already carry native sample control here. + fragment_sample_control: p.tib_blocks, + fragment_dispatch_control: 0, + store_pipeline_2_bind: p.store_pipeline_bind, + store_pipeline_2: p.store_pipeline, + partial_tile_flags: 0x200000, + ta_cluster_tilemap: 0, + ta_cluster_layermeta: 0, + ta_cluster_meta: [0; 4], + ta_cluster_control: [0; 3], + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G18Capture { + ColorA, + ColorB, + GearsG1, + FragmentWitness, +} + +/// Captured user fields plus explicitly unallocated kernel-owned fields. +/// The job MUST replace resource addresses, IDs, cycle/metrics and policy. +/// Geometry is constructed for the capture's 256x256 single-layer target; +/// native oracle parity injects that same geometry into the owned RT loader. +pub(crate) fn g18_capture_parameters( + capture: G18Capture, +) -> (G17pRenderParameters, G18RenderControls) { + let mut p = G17pRenderParameters::source_rsrc8(); + p.width = 256; + p.height = 256; + p.context_base = 0; + p.tilemap = 0; + p.layermeta = 0; + p.heapmeta = 0; + p.tpc = 0; + p.deflake_1 = 0x68980; + p.deflake_2 = 0x680c0; + p.deflake_3 = 0x68000; + p.encoder = 0x18000; + p.ta_status = 0; + p.fragment_status = 0; + p.store_pipeline_bind = 0; + p.partial_store_pipeline_bind = 0; + p.load_pipeline_bind = 0xffff800000000040; + p.partial_load_pipeline_bind = p.load_pipeline_bind; + let (shader, aux, id) = match capture { + G18Capture::ColorA => (0x10000168000, 0x10000280000, 0x0e0260a2u64), + G18Capture::ColorB => (0x10000168000, 0x10000280000, 0x1c0268ba), + G18Capture::GearsG1 => (0x100001b0000, 0x100002c8000, 0x4a13d3dc), + G18Capture::FragmentWitness => (0x10000170000, 0x10000288000, 0x0c0060a2), + }; + p.load_pipeline = shader + 0x240; + p.partial_load_pipeline = shader; + p.store_pipeline = shader + 0x480; + p.partial_store_pipeline = p.store_pipeline; + p.scissor_array = 0; + p.depth_bias_array = 0; + p.aux_fb = aux; + p.utile_config = 0xa000; + p.multisample_control = 0x88; + // Native J873 device +6b0 bit 16 becomes PPP bit 9 (8c4f004). + // Keep the same 0x202 used by live native draws. + p.ppp_control = 0x202; + p.tib_blocks = 0; + p.tile_config = 0xc10280; + p.aux_fb_flags = if capture == G18Capture::GearsG1 { + 0x30c000 + } else { + 0x30c001 + }; + p.cycle = 0; + p.record_index = 0x7000000001; + p.lifecycle = id << 32; + p.fragment_lifecycle = id << 32; + p.merge_upper_x_bits = 0; + p.merge_upper_y_bits = 0; + p.ta_timestamp_start = 0; + p.ta_timestamp_end = 0; + p.fragment_timestamp_start = 0; + p.fragment_timestamp_end = 0; + if capture == G18Capture::GearsG1 { + p.depth_buffer = 0x100000c8000; + p.depth_aux_buffer = 0x10000108000; + p.depth_flags = 0x80044; + p.partial_depth_flags = 0x88044; + p.depth_dimensions = 0xff00ff; + } + let c = g18_default_controls(&p); + (p, c) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderTargetLayout { + pub(crate) tilemap_bytes: usize, + pub(crate) heapmeta_offset: usize, + pub(crate) heapmeta_bytes: usize, + pub(crate) target_bytes: usize, + pub(crate) tpc_bytes: usize, + pub(crate) deflake_bytes: usize, + pub(crate) aux_bytes: usize, +} + +/// Original G18 geometry and updateGtpOnlyBuffers packing for one layer, +/// 32x32 utiles and one sample. `kind` selects clustering, not physical +/// cluster count: native TPC sizing still uses both J873 clusters for kind 1. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18NativeTargetLayout { + pub layout: G18RenderTargetLayout, + pub slice_sizes: [usize; 8], + pub slice_offsets: [usize; 8], + pub cluster_control: [u64; 3], +} + +/// Native target layout for one layer. msaa163: `utile` is the render's +/// utile. The tilemap and the TPC slice scale with utiles per 32x32 tile, the +/// M5 rule (`m5_render_target.rs:36-43`) and the same term that doubles +/// size1/size3 (TA 0x1c0b1/0x1c850/0x1c0a9). Native 4x (32x16) measures TA +/// +0x788 = 0x2000 against 0x1000 at 1x (MSAA notes s2.2). Only the witnessed +/// 32x16 single-target layout (kind 1) is admitted besides 32x32, which is +/// unchanged. +pub(crate) fn g18_native_target_layout(w: u32, h: u32, kind: u32, utile: (u8, u8)) + -> Option { + g18_native_target_layout_layers(w, h, 1, kind, utile) +} + +/// Original G18 target preparation also allocates a tilemap and TPC for each +/// layer. Layer metadata is one separate 0x100-byte region, not one per layer. +/// Native598 executes the original parser and preparation through 2048 layers, +/// including the 32x16 four-sample kind-1 geometry. Submission admission remains +/// separate: allocation arithmetic alone does not qualify layered rendering. +pub(crate) fn g18_native_target_layout_layers( + w: u32, h: u32, layers: u16, kind: u32, utile: (u8, u8), +) -> Option { + if !(1..=16384).contains(&w) || !(1..=16384).contains(&h) + || !(1..=2048).contains(&layers) || !(1..=2).contains(&kind) { + return None; + } + let utiles = match utile { + (32, 32) => 1, + G18_MSAA4_UTILE if kind == 1 => (32 / G18_MSAA4_UTILE.0 as usize) + * (32 / G18_MSAA4_UTILE.1 as usize), + _ => return None, + }; + let mx = ((((w as usize + 31) / 32 + 3) / 4) + 3) & !3; + let my = ((((h as usize + 31) / 32 + 3) / 4) + 3) & !3; + let layers = usize::from(layers); + let tilemap = (5usize * 16 * mx * my * utiles).checked_mul(layers)?; + let layermeta_bytes = if layers > 1 { 0x100usize } else { 0 }; + // Explicit experiment policy, not inferred live macOS configuration: + // segment mode 4 (native table gives 64), range differences zero, 8 PB + // blocks of 512 KiB. The original generators receive these same inputs. + let mut slice_sizes = [0; 8]; + slice_sizes[0] = (2usize * 128 * mx * my * utiles).checked_mul(layers)?; + if kind == 2 { + slice_sizes[1] = tilemap.checked_mul(64)?; + slice_sizes[2] = layermeta_bytes * 64; + slice_sizes[4] = 0x680 + 2 * 0x300; + slice_sizes[5] = 2 * 0x5c0; + slice_sizes[6] = (((0x400000usize / 0x8000 + 0x17ff) / 0x1800) + 1) * 8 * 64; + slice_sizes[7] = 0x80; + } + let alignments = if kind == 2 { [0x40, 0x40, 0x100, 0, 0x40, 0x40, 0x80, 0x40] } + else { [0x40, 0, 0, 0, 0, 0, 0, 0] }; + let mut slice_offsets = [0; 8]; + let mut bytes = 0usize; + for n in 0..8 { + let align = alignments[n]; + if align != 0 { bytes = bytes.checked_add(align - 1)? & !(align - 1); } + slice_offsets[n] = bytes; + bytes = bytes.checked_add(slice_sizes[n])?; + } + let tilemap_bytes = tilemap.checked_add(0xfff)? & !0xfff; + let heapmeta_offset = tilemap_bytes.checked_add(layermeta_bytes)?; + let target_bytes = heapmeta_offset.checked_add(0x200 + 0x3fff)? & !0x3fff; + Some(G18NativeTargetLayout { + layout: G18RenderTargetLayout { tilemap_bytes, heapmeta_offset, + heapmeta_bytes: 0x200, target_bytes, + tpc_bytes: bytes.checked_add(0x3fff)? & !0x3fff, deflake_bytes: 0xc000, aux_bytes: 0x10000 }, + slice_sizes, slice_offsets, + cluster_control: if kind == 1 { [1, 0, 0x80] } else { [0x40, 4, 0x80] }, + }) +} + +/// Correctly rounded positive IEEE-754 division of 0x3fddb3d9 by a target +/// dimension. Integer arithmetic keeps the kernel free of FP/SIMD use. +pub(crate) fn g18_merge_coefficient(dimension: u32) -> Option { + if !(1..=16384).contains(&dimension) { return None; } + let b = 31 - dimension.leading_zeros(); + let shift = b + u32::from(0xddb3d9u64 < (u64::from(dimension) << (23 - b))); + let numerator = 0xddb3d9u64 << shift; + let denominator = u64::from(dimension); + let mut mantissa = numerator / denominator; + let remainder = numerator % denominator; + if remainder * 2 > denominator || (remainder * 2 == denominator && mantissa & 1 != 0) { + mantissa += 1; + } + Some(((127 - shift) << 23) + mantissa as u32 - (1 << 23)) +} + +/// Conservative 2x M5 storage pending native G18 geometry qualification. +/// Only H1/H2 geometry is admitted; clusters must be one or two. +pub(crate) fn g18_render_target_layout( + w: u32, + h: u32, + layers: u16, + utile: (u8, u8), + samples: u8, + clusters: u32, +) -> Option { + if layers != 1 || utile != (32, 32) || samples != 1 || !(1..=2).contains(&clusters) { + return None; + } + let m = m5_render_target::M5RenderTargetLayout::new(w, h, layers, utile.0, utile.1, clusters)?; + let tilemap_bytes = m.tilemap_bytes.checked_mul(2)?; + let heapmeta_offset = tilemap_bytes.checked_add(0xfff)? & !0xfff; + let heapmeta_bytes = m.heapmeta_bytes.checked_mul(2)?; + let target_bytes = core::cmp::max( + m.target_bytes.checked_mul(2)?, + heapmeta_offset + .checked_add(heapmeta_bytes)? + .checked_add(0x3fff)? + & !0x3fff, + ); + Some(G18RenderTargetLayout { + tilemap_bytes, + heapmeta_offset, + heapmeta_bytes, + target_bytes, + tpc_bytes: m.tpc_bytes.checked_mul(2)?, + deflake_bytes: 0xc000, + aux_bytes: 0x10000, + }) +} + +/// Full 0xc000-byte captured G18 deflake allocation. Viewport is at +2c00, +/// not M5's +9c0. Both colour and gears captures witness these eight dwords. +pub(crate) fn initialize_g18_deflake(page: &mut [u8], w: u32, h: u32) -> Option<()> { + if page.len() != 0xc000 || !(1..=16384).contains(&w) || !(1..=16384).contains(&h) { + return None; + } + fn half(v: u32) -> u32 { + let b = 31 - v.leading_zeros(); + ((b + 126) << 23) | ((v << (23 - b)) & 0x7fffff) + } + page.fill(0); + for (off, v) in [ + (0x2c00, 0x3000), + (0x2c04, 0x80000000 | ((w + 31) / 32 - 1)), + (0x2c08, (h + 31) / 32 - 1), + (0x2c10, half(w)), + (0x2c14, half(w)), + (0x2c18, half(h)), + (0x2c1c, half(h) | 0x80000000), + (0x2c24, 0x3f800000), + ] { + page[off..off + 4].copy_from_slice(&v.to_le_bytes()); + } + Some(()) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/render_assembly.rs b/drivers/gpu/drm/asahi/g18/fw/render_assembly.rs new file mode 100644 index 0000000000000..86ad9028ad369 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/render_assembly.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Pure runtime assembly shared by the kernel owner and the host dry-run. +//! Register arithmetic belongs to g18_render; command fields to render-cmd. +#![allow(dead_code)] +use crate::{g18::fw::render_params::G17pRenderParameters, g18::address_map as va, + g18::fw::graphics_payload::fnv64, g18::fw::render as regs, g18::fw::render_command as cmd, + g18::fw::sksm::M5SksmRce}; + +pub(crate) const TA_OFFSET: usize = 0; +pub(crate) const FRAGMENT_OFFSET: usize = 0x4000; +pub(crate) const TARGET_OFFSETS: [u64; 6] = [0, 0x20000, 0x40000, 0x50000, 0x60000, 0x80000]; +// Fixed virtual reservations, not eager allocations. Native tilemap/TPC sizes +// scale with layers: even a 32x32x64 target needs a 0x40000-byte TPC. Keep +// addresses stable when an application grows an already-owned target. +const NATIVE_TARGET_OFFSETS: [u64; 6] = + [0, 0x400000, 0xc00000, 0xc10000, 0xc20000, 0xc30000]; +pub(crate) const TARGET_NAMES: [&str; 6] = ["tilemap-heapmeta", "tpc", "deflake", "ta-status", "3d-status", "auxiliary"]; +pub(crate) const DIGEST_NAMES: [&str; 7] = ["ta-command", "3d-command", "ta-registers", + "3d-main", "3d-store", "3d-resume", "3d-load"]; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct TargetPlan { + pub layout: regs::G18RenderTargetLayout, + pub sizes: [usize; 6], + pub offsets: [u64; 6], + pub native: Option, +} + +pub(crate) fn target_plan(p: &G17pRenderParameters) -> Option { + target_plan_for_kind(p, crate::g18::uapi::render_policy::target_kind()) +} + +pub(crate) fn target_plan_for_kind(p: &G17pRenderParameters, kind: u32) -> Option { + // msaa163: the admitted 4x render (32x16 utiles) plans only with the + // native single-target layout, whose TPC/tilemap scale with its utiles; + // the legacy (kind 0) and clustered (kind 2) layouts have no 4x witness. + let msaa4 = regs::g18_msaa4(p); + if !(1..=2048).contains(&p.layers) || p.samples != 1 + || (!msaa4 && (p.utile_width, p.utile_height) != (32, 32)) + || (msaa4 && kind != 1) { + return None; + } + let native = if kind == 0 { None } else { + Some(regs::g18_native_target_layout_layers(p.width, p.height, p.layers, kind, + (p.utile_width, p.utile_height))?) + }; + let layout = if let Some(n) = native { n.layout } else { + regs::g18_render_target_layout(p.width, p.height, p.layers, + (p.utile_width, p.utile_height), p.samples, 2)? + }; + // Stable VAs across application resizes. The extra slices share the TPC + // allocation; reserve 4 MiB for it, covering 1280x720 without overlap. + let offsets = match kind { + 1 => NATIVE_TARGET_OFFSETS, + 2 => [0, 0x100000, 0x500000, 0x510000, 0x520000, 0x600000], + _ => TARGET_OFFSETS, + }; + let sizes = [layout.target_bytes, layout.tpc_bytes, layout.deflake_bytes, + 0x4000, 0x4000, layout.aux_bytes]; + for (i, size) in sizes.iter().enumerate() { + let end = offsets[i].checked_add(*size as u64)?; + if i + 1 < sizes.len() && end > offsets[i + 1] { return None; } + if end > va::GRAPHICS_TARGET_BYTES as u64 { return None; } + } + Some(TargetPlan { layout, sizes, offsets, native }) +} + +/// Native 3D uses full target VAs; TA subtracts the mapping's context base. +/// The owner maps identical backing at the compact and biased views. +pub(crate) fn apply_target_plan(p: &mut G17pRenderParameters, plan: &TargetPlan) { + let a = plan.offsets.map(|offset| va::GRAPHICS_TARGET_OFFSET + offset); + p.context_base = va::COMPACT_BIAS; + p.tilemap = va::COMPACT_BIAS + a[0]; + p.heapmeta = p.tilemap + plan.layout.heapmeta_offset as u64; + // Native target preparation places layer metadata directly after the + // rounded tilemap; the heap follows it. Legacy kind 0 keeps its layout. + p.layermeta = if plan.native.is_some() { + p.tilemap + plan.layout.tilemap_bytes as u64 + } else { p.heapmeta }; + p.tpc = va::COMPACT_BIAS + a[1]; + // Deflake has a separate task-address encoding; keep these compact. + p.deflake_3 = a[2]; p.deflake_2 = a[2] + 0xc0; p.deflake_1 = a[2] + 0x980; + p.ta_status = a[3]; p.fragment_status = a[4]; p.aux_fb = a[5]; + // Generator does not admit timestamp registers yet; command timestamp + // pointers are separately supplied by the owned command inputs. + p.ta_timestamp_start = 0; p.ta_timestamp_end = 0; + p.fragment_timestamp_start = 0; p.fragment_timestamp_end = 0; +} + +pub(crate) fn apply_target_controls(p: &mut G17pRenderParameters, + c: &mut regs::G18RenderControls, plan: &TargetPlan) { + if let Some(n) = plan.native { + let pointer = |i: usize| if n.slice_sizes[i] == 0 { 0 } + else { p.tpc - p.context_base + n.slice_offsets[i] as u64 }; + c.ta_cluster_tilemap = pointer(1); + c.ta_cluster_layermeta = pointer(2); + c.ta_cluster_meta = [pointer(4), pointer(5), pointer(6), pointer(7)]; + c.ta_cluster_control = n.cluster_control; + p.merge_upper_x_bits = regs::g18_merge_coefficient(p.width).expect("validated width"); + p.merge_upper_y_bits = regs::g18_merge_coefficient(p.height).expect("validated height"); + } +} + +#[derive(Debug)] +pub(crate) enum AssemblyError { Registers(regs::G18RenderError), Command(cmd::G18RenderCommandError) } +impl From for AssemblyError { fn from(e: regs::G18RenderError) -> Self { Self::Registers(e) } } +impl From for AssemblyError { fn from(e: cmd::G18RenderCommandError) -> Self { Self::Command(e) } } + +pub(crate) struct EncodedJob { pub rce: [[M5SksmRce; 4]; 2], pub digests: [u64; 7] } + +// Only integer fields: zero is a valid initial representation. Kept on the +// heap by the kernel caller, including across the register-builder calls. +pub(crate) struct Workspace { pub ta: regs::G18Program<91>, pub fragment: regs::G18FragmentPrograms } + +fn view(p: ®s::G18Program) -> cmd::G18ProgramRef<'_> { + cmd::G18ProgramRef { registers: &p.registers, rce_count: p.rce_count, command_offset: p.command_offset } +} + +#[inline(never)] +#[cfg(test)] +pub(crate) fn encode( + ta: &mut [u8; cmd::G18_TA_COMMAND_BYTES], fragment: &mut [u8; cmd::G18_3D_COMMAND_BYTES], + p: &G17pRenderParameters, controls: ®s::G18RenderControls, + inputs: &cmd::G18RenderCommandInputs, +) -> Result { + let mut workspace = Workspace { ta: regs::G18Program::empty(), fragment: regs::G18FragmentPrograms { + main: regs::G18Program::empty(), store: regs::G18Program::empty(), + resume: regs::G18Program::empty(), load: regs::G18Program::empty() } }; + encode_into(ta, fragment, p, controls, inputs, &mut workspace) +} + +#[inline(never)] +pub(crate) fn encode_into( + ta: &mut [u8; cmd::G18_TA_COMMAND_BYTES], fragment: &mut [u8; cmd::G18_3D_COMMAND_BYTES], + p: &G17pRenderParameters, controls: ®s::G18RenderControls, + inputs: &cmd::G18RenderCommandInputs, workspace: &mut Workspace, +) -> Result { + ta.fill(0); fragment.fill(0); + regs::build_g18_ta_program_into(&mut workspace.ta, p, controls)?; + regs::build_g18_fragment_programs_into(&mut workspace.fragment, p, controls)?; + let t = &workspace.ta; + let f = &workspace.fragment; + // Native main boundaries N-4 / N-2 / N, per render-cmd §2. + let programs = cmd::G18FragmentProgramRefs { main: view(&f.main), main_mid_count: 90, + store: view(&f.store), resume: view(&f.resume), load: view(&f.load) }; + let fr_rce = cmd::encode_g18_3d_command(fragment, + va::GRAPHICS_JOB_LOW + FRAGMENT_OFFSET as u64, inputs, &programs)?; + // The linked 3D tuple in inputs is finalized before encoding TA. + let ta_rce = cmd::encode_g18_ta_command(ta, va::GRAPHICS_JOB_LOW, inputs, &view(&t))?; + let digests = [fnv64(ta), fnv64(fragment), fnv64(&ta[0x60..0x60 + 91 * 12]), + fnv64(&fragment[0xa0..0xa0 + 92 * 12]), fnv64(&fragment[0x7c0..0x7c0 + 17 * 12]), + fnv64(&fragment[0xee0..0xee0 + 24 * 12]), fnv64(&fragment[0x1600..0x1600 + 12 * 12])]; + Ok(EncodedJob { rce: [ta_rce, fr_rce], digests }) +} + +/// Runtime variants are deliberately a closed set. H2 repeats three frames; +/// no caller may infer retirement from an output hash before the next frame. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RenderVariant { Color, TaOnly, Clear, GearsThreeFrames } +impl RenderVariant { + pub(crate) fn parse(value: u32) -> Option { match value { + 0 => Some(Self::Color), 1 => Some(Self::TaOnly), 2 => Some(Self::Clear), + 3 => Some(Self::GearsThreeFrames), _ => None, + } } + pub(crate) fn stage_count(self) -> u32 { if self == Self::TaOnly { 1 } else { 2 } } + pub(crate) fn frames(self) -> u32 { if self == Self::GearsThreeFrames { 3 } else { 1 } } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/render_command.rs b/drivers/gpu/drm/asahi/g18/fw/render_command.rs new file mode 100644 index 0000000000000..8acdb17cca17c --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/render_command.rs @@ -0,0 +1,925 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 (J873/T8152, 26A428 HAL400) TA and 3D firmware commands, the M6 +//! channel configuration, SKSM kick-entry operands and tag14 kicks. +//! +//! Every byte follows the original AGXTAChannelSKSM::submitBuffer +//! (0xfffffe0008c837ac) and AGX3DChannelSKSM::submitBuffer (0xfffffe0008bbb500) +//! field writers, the register-list placement of the HAL400 generators +//! (TAChannelSKSM::generateRegisterList 0xfffffe0008c4e1b4, +//! 3DChannelSKSM::generateRegisterListFor3D 0xfffffe0008c3abe8) and the +//! KSMKickQueue TA/3D wrappers (0xfffffe0008c4a9e4 / 0xfffffe0008c4a794). +//! Parity is established by a native render-command audit, +//! which executes those functions unmodified. +//! +//! These are pure serializers. They never compute register values (the +//! programs come from g18_render), never allocate, map or publish, and never +//! choose an address: every kernel-owned address is an input that the job +//! must own and have mapped in the context that dereferences it. Native field +//! names are kept as descriptor offsets (`dNNNN` = native host descriptor +//! +0xNNNN) wherever a guessed meaning would be unproven. +//! +//! Command selectors (first u32 of every inner-ring command): TA 0, 3D 1, +//! 3D fast blit 2 (AGFIChannelCommandFastBlit, 0x9e0 bytes, emitted only when +//! 3D descriptor +0x840 bit 0 is set; not encoded here), compute 3, +//! tag15 channel configuration 15, tag14 add-kicks 14. + +#![allow(dead_code)] + +use crate::g18::fw::render_params::RegisterWrite; +use crate::g18::fw::sksm::{self as m5_sksm, M5SksmDependency, M5SksmOperands, M5SksmQueue, M5SksmRce}; + +pub(crate) const G18_TA_COMMAND_BYTES: usize = 0x980; +pub(crate) const G18_3D_COMMAND_BYTES: usize = 0x2240; +/// Selector-2 3D fast-blit form. Recorded for completeness; never emitted. +pub(crate) const G18_FAST_BLIT_COMMAND_BYTES: usize = 0x9e0; +pub(crate) const G18_CHANNEL_CONFIG_BYTES: usize = 0x98; +pub(crate) const G18_M5_CHANNEL_CONFIG_BYTES: usize = 0x86; +pub(crate) const G18_KICK_BYTES: usize = 21; + +pub(crate) const G18_TA_SELECTOR: u32 = 0; +pub(crate) const G18_3D_SELECTOR: u32 = 1; +pub(crate) const G18_FAST_BLIT_SELECTOR: u32 = 2; +pub(crate) const G18_CONFIG_SELECTOR: u32 = 15; + +/// TA register program: one list at command +0x60. +pub(crate) const G18_TA_PROGRAM_OFFSET: u32 = 0x60; +/// 3D main, partial store, partial resume and partial load programs. +pub(crate) const G18_3D_PROGRAM_OFFSETS: [u32; 4] = [0xa0, 0x7c0, 0xee0, 0x1600]; +/// Each list slot: 0x700 bytes of 12-byte entries, then the GPU address of the +/// list (+0x700, u64) and its entry count (+0x708, u16). +const PROGRAM_AREA: usize = 0x700; +const REGISTER_BYTES: usize = 12; +pub(crate) const G18_PROGRAM_CAPACITY: usize = PROGRAM_AREA / REGISTER_BYTES; +/// The TA generator mirrors the first 0x1c898 value into command +0x85e and +/// later emits 0x1c899 with bit 0 set (0xfffffe0008c4f690 / 0x8c4fff0). +const TA_DISPATCH_REGISTER: u32 = 0x1c898; +/// generateRegisterListFor3D mirrors its 0x10038 value (appended as the +/// 64-bit entry 0x10039) into command +0x1f58 (0x8c3bfe8), then clears +0x1f60 +/// (0x8c3c0d0). Register numbers are matched without the width bit 0. +const FRAGMENT_TILE_REGISTER: u32 = 0x10038; + +/// Native KSMKickQueue QoS class table 0xfffffe00071524d8, index 3/4/5 for +/// TA/3D/CL when the scheduler has no override (+0xc68 == 0). +pub(crate) const G18_TA_QOS_CLASS: u8 = 12; +pub(crate) const G18_3D_QOS_CLASS: u8 = 24; +/// encodeTAKickEntry / encode3DKickEntry dependency-type argument. +pub(crate) const G18_TA_DEPENDENCY_TYPE: u8 = 2; +pub(crate) const G18_3D_DEPENDENCY_TYPE: u8 = 3; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum G18RenderCommandError { + ProgramTooLong, + ProgramOffset, + RceCount, + MissingDispatchMirror, + AddressOverflow, + InvalidCompletionState, + Kick, + Sksm, +} + +/// A borrowed register program, the slice form of g18_render's +/// `G18Program` (registers, native RCE metadata count, list offset). +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18ProgramRef<'a> { + pub registers: &'a [RegisterWrite], + /// Native generator metadata slot count, used as the SKSM RCE tag and the + /// command's first boundary (TA: N-3; 3D main: N-4). + pub rce_count: u16, + pub command_offset: u32, +} + +/// The four 3D programs. Native always encodes all four. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18FragmentProgramRefs<'a> { + /// `rce_count` is the +0x80 boundary (native N-4). + pub main: G18ProgramRef<'a>, + /// Second main boundary, +0x82 and +0x84 (native N-2, after 0x16020). + pub main_mid_count: u16, + pub store: G18ProgramRef<'a>, + pub resume: G18ProgramRef<'a>, + pub load: G18ProgramRef<'a>, +} + +/// Native per-stage completion identity copied into the command tail. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18StageCompletion { + /// Firmware-written stamp words: stamp-memory A/B base plus + /// IOGPUEventMachine::getStampOffset(queue) = queue * 4. + pub stamp_addresses: [u64; 2], + /// Event-machine stamp of this submission (vendevtCommandRec +0). + pub stamp: u32, + /// Kick-queue index (vendevtCommandRec +4; the qid on Linux). + pub queue_index: u32, + /// Channel +0x38. + pub ordinal: u32, + /// Host descriptor +0x48 nonzero. + pub has_wait: bool, + /// Hardware completion word, [`g18_render_completion`]. M5's + /// `(1 << 40) | generation` is this value for pass 1, stamp 0x100. + pub completion: u64, + /// Submission key: TA descriptor +0xe68 / 3D descriptor +0x100. Also + /// the key argument of the command-pointer publication. + pub key: u64, +} + +/// Hardware-stamp conversion shared by completion words and barrier stamps +/// (HAL400 convertEventMachineTSToHWTS arithmetic; identical to compute). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18HardwareStamp { + pub stamp: u32, + /// Kick-queue entry count (+0x10). + pub entries: u32, + /// Event-machine generation of the stamp's queue, half selected by the + /// stamp's sign (+8 for stamp < 0x80000000, else +16). + pub generation: u64, + /// Kick-queue ring base (+0x90). + pub base: u64, +} + +pub(crate) fn g18_hardware_stamp(p: &G18HardwareStamp) -> Result { + if p.entries == 0 { + return Err(G18RenderCommandError::InvalidCompletionState); + } + let value = (u64::from(p.stamp >> 8) | p.generation << 24).wrapping_sub(1); + let quotient = (value / u64::from(p.entries)) as u32; + let remainder = (value as u32).wrapping_sub(quotient.wrapping_mul(p.entries)); + let slot = (p.base as u32 & 0xff) + (remainder & 0xff); + let carry = slot / p.entries; + let index = (slot - carry * p.entries) & 0xff; + let phase = ((p.base >> 8) as u32) + .wrapping_add(quotient) + .wrapping_add(carry); + Ok(u64::from(index) | u64::from(phase) << 8) +} + +/// Completion word: hardware stamp | completion scratch payload << 40. The +/// payload is the 3D channel's per-pass counter (+0x1d4, incremented by the +/// 3D submit); the TA of the same pass carries the same value. +pub(crate) fn g18_render_completion( + p: &G18HardwareStamp, + pass: u8, +) -> Result { + Ok(g18_hardware_stamp(p)? | u64::from(pass) << 40) +} + +/// TA fields for the TA's own linked 3D submission (AGX3DWorkQueue:: +/// submitCommand 0x8c93938..0x8c93a50 writes them after the 3D submit). +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18TaLinked3d { + /// 3D channel +0x18 (its queue index) -> TA +0x79c. + pub queue_index: u32, + /// 3D channel +0x1d0 (the 3D's event stamp) -> TA +0x7a0. + pub stamp: u32, + /// TA descriptor +0x1580 bit 0 -> TA +0x7a4. + pub flag: u8, + /// 3D completion word -> TA +0x7a8 (40 bits) and +0x7b0/+0x7b1 (pass). + pub completion: u64, +} + +/// Native TA +0x89e policy bit: accelerator +0x6c0 against descriptor +/// +0x1658/+0x1198/+0x11a7 (0x8c83f6c..0x8c83fa4, 0x8c84448..0x8c8446c). +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18TaPolicy { + pub a6c0: u32, + pub d1658: u8, + pub d1198: u32, +} + +/// Optional four-address tail (+0x95f..+0x97f), as compute's +0xfe5. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ExtraAddresses { + /// Descriptor count word (TA +0x13d8, 3D +0xa90). + pub count: u32, + pub addresses: [u64; 4], + /// Accelerator +0x116b8 bits 0..1. + pub policy: u8, +} + +/// Every TA command field outside the register program. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18TaCommandFields { + /// AGXCommandQueue +0x628 before its increment. + pub sequence: u64, + pub completion: G18StageCompletion, + /// TA HardwareBufferID (PB descriptor-table slot), d1280. + pub pb_slot: u32, + /// Parameter management +0x1d4, d12a8. + pub pm_1d4: u32, + /// PB state object (firmware VA), d1288. Same value as tag15 +0x7a. + pub pb_state: u64, + /// Parameter management +0xa8[+0x2c], d1290. + pub pm_scene: u64, + /// Parameter management +0xb8, d12a0. + pub pm_b8: u64, + pub d12d0: u32, + pub d12c0: u32, + pub d12c8: u64, + pub d11b8: u64, + pub d11d8: u64, + pub linked_3d: G18TaLinked3d, + pub d1068: u64, + pub d1070: u64, + pub d10f8: u64, + pub d08a0: u8, + pub d08a4: u32, + pub d08a8: u32, + pub d08ac: u32, + pub d08b0: u64, + pub d1418: [u64; 2], + pub d1428: u64, + pub d1430: u64, + /// UMA pool state firmware VA (UMA data record +0x20), d1458. + pub uma_state: u64, + pub d1474: u8, + pub d11a4: u8, + pub d11a7: u8, + pub policy: G18TaPolicy, + pub extra: G18ExtraAddresses, +} + +/// Every 3D command field outside the four register programs. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G183dCommandFields { + pub sequence: u64, + pub completion: G18StageCompletion, + /// Completion scratch payload (3D channel +0x1d4 after this submit) -> +0x90. + pub pass: u8, + pub d158: u64, + pub d150: u32, + /// Descriptor +0x980 == 2 -> +0x2174. + pub d980_is_2: bool, + pub d08cf: u8, + pub dbbc: u8, + /// 3D side of the shared PM state (firmware VA of d940) -> +0x28. + pub pm_state: u64, + /// Parameter management +0xa8[+0x2c], d948 -> +0x30. + pub pm_scene: u64, + /// Parameter management +0xb8, d958 -> +0x38. + pub pm_b8: u64, + pub d08d8: u64, + pub d08f0: u64, + pub d08f8: u32, + pub d0900: u32, + pub d0908: u32, + pub d0910: [u64; 2], + pub d0920: u64, + pub d0930: u32, + pub d04e0: u64, + pub d04e8: u64, + pub d04f0: u64, + pub d0510: u64, + /// d518..d688 as 23 16-byte pairs, copied to +0x1d58..+0x1ec8. + pub d0518: [[u64; 2]; 23], + pub d0690: u64, + pub d0698: u64, + pub d06a0: [u64; 2], + pub d06b0: u64, + pub d06b8: [u64; 2], + pub d06d8: u64, + pub d06f0: u64, + pub d06f8: u64, + pub d0708: u64, + pub d0728: u64, + pub d0730: u64, + pub d0750: u64, + pub d0758: u64, + pub d0770: [u64; 2], + pub d0780: [u64; 2], + pub d0790: [u64; 2], + pub d07b0: [u64; 2], + pub d07c0: u64, + pub d07c8: u32, + pub d07cc: u64, + pub d07d4: u32, + pub d07d8: u64, + pub d07e8: u64, + pub d07f8: u32, + pub d04d0: u32, + pub d08c4: u8, + pub dc98: u32, + pub dc9c: u8, + pub d08a0: u8, + pub d08a4: u32, + pub d08a8: u32, + pub d08ac: u32, + pub dad0: [u64; 2], + pub dae0: u64, + pub dae8: u64, + /// UMA pool state firmware VA, dba0. + pub uma_state: u64, + /// Timestamp pair, d8b0/d8b8 -> +0x2198/+0x21a0. + pub d08b0: [u64; 2], + /// d8c5, d8c6, d8c7, d8cb bit 0 -> +0x2124..+0x2130. + pub d08c5_flags: [u8; 4], + pub d08cd: u8, + pub extra: G18ExtraAddresses, +} + +/// Inputs shared by both commands of a render pass (contract §6.2). +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18RenderCommandInputs { + /// TA descriptor +0x12c4 / 3D descriptor +0x154 (the GPU context ID). + pub context_id: u32, + /// Descriptor +0x198: the 0xc8-byte scheduler record (TA +0x10, 3D +0x20). + pub scheduler_record: u64, + pub ta: G18TaCommandFields, + pub fragment: G183dCommandFields, +} + +fn put8(out: &mut [u8], offset: usize, value: u8) { + out[offset] = value; +} + +fn put16(out: &mut [u8], offset: usize, value: u16) { + out[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn put128(out: &mut [u8], offset: usize, value: [u64; 2]) { + put64(out, offset, value[0]); + put64(out, offset + 8, value[1]); +} + +fn read32(out: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(out[offset..offset + 4].try_into().unwrap()) +} + +fn completion_tail(out: &mut [u8], base: usize, c: &G18StageCompletion) { + put64(out, base, c.stamp_addresses[0]); + put64(out, base + 8, c.stamp_addresses[1]); + put32(out, base + 0x10, c.stamp); + put32(out, base + 0x14, c.queue_index); + put32(out, base + 0x18, c.ordinal); + put32(out, base + 0x1c, u32::from(c.has_wait)); +} + +fn extra_tail(out: &mut [u8], flags: usize, addresses: usize, e: &G18ExtraAddresses) { + out[flags] &= 0xf2; + if e.count != 0 || e.addresses.iter().any(|&value| value != 0) { + out[flags] |= 1 | ((e.policy & 3) << 2); + for (index, value) in e.addresses.iter().enumerate() { + put64(out, addresses + index * 8, *value); + } + } +} + +/// Validate one program slot before any byte is written. +fn check_program(p: &G18ProgramRef<'_>, offset: u32) -> Result<(), G18RenderCommandError> { + if p.command_offset != offset { + return Err(G18RenderCommandError::ProgramOffset); + } + if p.registers.len() > G18_PROGRAM_CAPACITY { + return Err(G18RenderCommandError::ProgramTooLong); + } + if usize::from(p.rce_count) > p.registers.len() || p.rce_count > 0x7f { + return Err(G18RenderCommandError::RceCount); + } + Ok(()) +} + +/// Native RCEBufferEncoder entry: the header keeps the destination word's +/// bits 0xfffc0006, then the 64-bit value follows unaligned at +4. +fn place_program(out: &mut [u8], low_alias: u64, p: &G18ProgramRef<'_>) -> Result { + let offset = p.command_offset as usize; + let address = low_alias + .checked_add(u64::from(p.command_offset)) + .ok_or(G18RenderCommandError::AddressOverflow)?; + put16(out, offset + PROGRAM_AREA + 8, 0); + put64(out, offset + PROGRAM_AREA, address); + for (index, register) in p.registers.iter().enumerate() { + let entry = offset + index * REGISTER_BYTES; + let header = (read32(out, entry) & 0xfffc0006) | (register.number & 0x3fff9); + put32(out, entry, header); + put64(out, entry + 4, register.value); + } + put16(out, offset + PROGRAM_AREA + 8, p.registers.len() as u16); + Ok(address) +} + +/// Encode the TA firmware command (selector 0) and return the SKSM RCE list. +/// +/// `out` must hold the command allocation's current contents: like native, +/// untouched bytes and header bits 0xfffc0006 of each list entry are kept. +/// `low_alias` is the command's GPU address in the job context (native +/// descriptor +0x1008); the RCE list and +0x760 name it. +pub(crate) fn encode_g18_ta_command( + out: &mut [u8; G18_TA_COMMAND_BYTES], + low_alias: u64, + i: &G18RenderCommandInputs, + ta: &G18ProgramRef<'_>, +) -> Result<[M5SksmRce; 4], G18RenderCommandError> { + check_program(ta, G18_TA_PROGRAM_OFFSET)?; + let dispatch = ta + .registers + .iter() + .find(|r| r.number & !1 == TA_DISPATCH_REGISTER) + .ok_or(G18RenderCommandError::MissingDispatchMirror)? + .value; + let t = &i.ta; + put32(out, 0, G18_TA_SELECTOR); + put64(out, 4, t.sequence); + put32(out, 0xc, i.context_id); + put64(out, 0x937, t.d12c8); + put32(out, 0x8d6, t.d12c0); + put32(out, 0x48, t.d12d0); + put32(out, 0x18, t.pb_slot); + put32(out, 0x1c, t.pm_1d4); + put64(out, 0x20, t.pb_state); + put64(out, 0x28, t.pm_scene); + put64(out, 0x30, t.pm_b8); + put64(out, 0x10, i.scheduler_record); + put64(out, 0x780, t.d11b8); + put64(out, 0x788, t.d11d8); + put32(out, 0x79c, t.linked_3d.queue_index); + put32(out, 0x7a0, t.linked_3d.stamp); + put32(out, 0x7a4, u32::from(t.linked_3d.flag & 1)); + put64(out, 0x7a8, t.linked_3d.completion & 0xff_ffff_ffff); + let pass = (t.linked_3d.completion >> 40) as u8; + put8(out, 0x7b1, pass); + put8(out, 0x7b0, pass); + put64(out, 0x7b6, t.d1068); + put64(out, 0x7be, t.d1070); + put64(out, 0x7d6, t.d10f8); + put128(out, 0x7de, [0, 0]); + put64(out, 0x7ee, 0); + put32(out, 0x7b2, 0); + put32(out, 0x86a, u32::from(t.d08a0 & 1)); + put32(out, 0x86e, t.d08a4); + put32(out, 0x872, t.d08a8); + put32(out, 0x876, t.d08ac); + put128(out, 0x87a, [0, 0]); + put128(out, 0x90e, t.d1418); + put64(out, 0x91e, t.d1428); + put64(out, 0x926, t.d1430); + put64(out, 0x92e, t.uma_state); + put8(out, 0x936, t.d1474 & 1); + put64(out, 0x8fe, t.d08b0); + put32(out, 0x892, u32::from(t.d11a4 & 1)); + put32(out, 0x89a, u32::from(t.d11a7 & 1)); + put32(out, 0x89e, u32::from(g18_ta_policy_bit(&t.policy, t.d11a7))); + put32(out, 0x8a2, 0); + put64(out, 0x8f6, 0); + completion_tail(out, 0x8a6, &t.completion); + put64(out, 0x8c6, t.completion.completion); + put64(out, 0x8ce, t.completion.key); + // Register generator placement (zeroing first, as 0x8c4fd28..0x8c4fd30). + put32(out, 0x38, 0); + put16(out, 0x3c, 0); + put32(out, 0x40, 0); + let address = place_program(out, low_alias, ta)?; + put16(out, 0x38, ta.rce_count); + put16(out, 0x3a, ta.registers.len() as u16); + put16(out, 0x3c, ta.registers.len() as u16); + put64(out, 0x85e, dispatch); + extra_tail(out, 0x97f, 0x95f, &t.extra); + let empty = M5SksmRce { address: 0, tag: 0 }; + Ok([M5SksmRce { address, tag: ta.rce_count as u8 }, empty, empty, empty]) +} + +/// Encode the 3D firmware command (selector 1) with all four programs and +/// return the SKSM RCE list (main, store, resume, load). +/// +/// `low_alias` is the command's GPU address in the job context (native +/// descriptor +0x470). Untouched bytes of `out` are kept, as native does. +pub(crate) fn encode_g18_3d_command( + out: &mut [u8; G18_3D_COMMAND_BYTES], + low_alias: u64, + i: &G18RenderCommandInputs, + f: &G18FragmentProgramRefs<'_>, +) -> Result<[M5SksmRce; 4], G18RenderCommandError> { + let programs = [f.main, f.store, f.resume, f.load]; + for (program, offset) in programs.iter().zip(G18_3D_PROGRAM_OFFSETS) { + check_program(program, offset)?; + } + for partial in &programs[1..] { + if usize::from(partial.rce_count) != partial.registers.len() { + return Err(G18RenderCommandError::RceCount); + } + } + if f.main_mid_count < f.main.rce_count || usize::from(f.main_mid_count) > f.main.registers.len() { + return Err(G18RenderCommandError::RceCount); + } + let tile = f + .main + .registers + .iter() + .find(|r| r.number & !1 == FRAGMENT_TILE_REGISTER) + .ok_or(G18RenderCommandError::MissingDispatchMirror)? + .value; + let t = &i.fragment; + put32(out, 0, G18_3D_SELECTOR); + put64(out, 4, t.sequence); + put32(out, 0xc, i.context_id); + put64(out, 0x21d1, t.d158); + put32(out, 0x2170, t.d150); + put32(out, 0x2174, u32::from(t.d980_is_2)); + put32(out, 0x2100, u32::from(t.d08cf & 1)); + put32(out, 0x20f8, 0); + // generateFrgBarrierRegister is a G18 stub (0x8c39224): 24 zero bytes. + put128(out, 0x2178, [0, 0]); + put64(out, 0x2188, 0); + put8(out, 0x21d0, t.dbbc & 1); + put64(out, 0x20, i.scheduler_record); + put64(out, 0x28, t.pm_state); + put64(out, 0x30, t.pm_scene); + put64(out, 0x38, t.pm_b8); + put32(out, 0x58, t.d08f8); + put64(out, 0x40, t.d08d8); + put64(out, 0x48, t.d08f0); + put32(out, 0x50, t.d0900); + put32(out, 0x54, t.d0908); + put128(out, 0x60, t.d0910); + put64(out, 0x70, t.d0920); + put32(out, 0x78, t.d0930); + put64(out, 0x1d30, t.d04e8); + put64(out, 0x1d20, t.d04e0); + put64(out, 0x1d40, t.d04f0); + for (index, pair) in t.d0518.iter().enumerate() { + put128(out, 0x1d58 + index * 16, *pair); + } + put64(out, 0x1ec8, t.d0698); + put64(out, 0x1ed0, t.d06f8); + put64(out, 0x1ed8, t.d0730); + put64(out, 0x1f28, t.d06d8); + put128(out, 0x1ee0, t.d06a0); + put64(out, 0x1ef0, t.d0728); + put64(out, 0x1ef8, t.d06b0); + put64(out, 0x1f00, t.d0708); + put64(out, 0x1f08, t.d0758); + put64(out, 0x1f30, t.d06f0); + put128(out, 0x1f10, t.d06b8); + put64(out, 0x1f20, t.d0750); + put64(out, 0x1f68, t.d07c0); + put32(out, 0x1f70, t.d07c8); + put128(out, 0x1f48, t.d0780); + put128(out, 0x1f74, t.d0790); + put64(out, 0x1f88, t.d07cc); + put32(out, 0x1f90, t.d07d4); + put128(out, 0x1f94, t.d07b0); + put64(out, 0x1fa8, t.d07d8); + put32(out, 0x1fb0, t.d04d0); + put128(out, 0x1f38, t.d0770); + put64(out, 0x1d50, t.d0510); + put64(out, 0x1fc0, t.d07e8); + put32(out, 0x1fc8, t.d07f8); + put64(out, 0x1fd0, t.d0690); + out[0x1fdc..0x20dc].fill(0); + put32(out, 0x20dc, 0); + put128(out, 0x20e0, [0, 0]); + put32(out, 0x1fd8, 0); + put32(out, 0x20f0, 0); + put8(out, 0x90, t.pass); + put32(out, 0x20fc, u32::from(t.d08c4 & 1)); + put32(out, 0x21f9, t.dc98); + put8(out, 0x21fd, t.dc9c); + put32(out, 0x2104, u32::from(t.d08a0 & 1)); + put32(out, 0x2108, t.d08a4); + put32(out, 0x210c, t.d08a8); + put32(out, 0x2110, t.d08ac); + put128(out, 0x2114, [0, 0]); + put128(out, 0x21a8, t.dad0); + put64(out, 0x21b8, t.dae0); + put64(out, 0x21c0, t.dae8); + put64(out, 0x21c8, t.uma_state); + put128(out, 0x2198, t.d08b0); + completion_tail(out, 0x2140, &t.completion); + put64(out, 0x2160, t.completion.completion); + put64(out, 0x2190, 0); + for (index, flag) in t.d08c5_flags.iter().enumerate() { + put32(out, 0x2124 + index * 4, u32::from(flag & 1)); + } + put32(out, 0x2138, u32::from(t.d08cd & 1)); + put64(out, 0x2168, t.completion.key); + extra_tail(out, 0x221e, 0x21fe, &t.extra); + put32(out, 0x221f, 0); + // generateRegisterListFor3D placement (zeroing at 0x8c3c7e4..0x8c3c7ec). + put32(out, 0x80, 0); + put16(out, 0x84, 0); + put32(out, 0x88, 0); + let mut rce = [M5SksmRce { address: 0, tag: 0 }; 4]; + for (slot, program) in rce.iter_mut().zip(programs.iter()) { + let address = place_program(out, low_alias, program)?; + *slot = M5SksmRce { address, tag: program.rce_count as u8 }; + } + put64(out, 0x1f58, tile); + put64(out, 0x1f60, 0); + put16(out, 0x80, f.main.rce_count); + put16(out, 0x82, f.main_mid_count); + put16(out, 0x84, f.main_mid_count); + put16(out, 0x88, f.main.registers.len() as u16); + Ok(rce) +} + +fn g18_ta_policy_bit(p: &G18TaPolicy, d11a7: u8) -> bool { + if p.a6c0 == 0 || !(1..=2).contains(&p.d1658) || p.d1198 == 3 { + return false; + } + match p.a6c0 { + 1 | 4 => d11a7 & 1 == 0, + 3 => true, + _ => false, + } +} + +// ---- M6 tag15 channel configuration ---------------------------------------- + +/// Channel-configuration layout. M6 inserts a u32 flag at +0x6e and a u64 at +/// +0x72, moving the parameter-buffer tuple from +0x6e..+0x86 to +0x7a..+0x92. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ChannelForm { + /// T8142/25G76, 0x86 bytes (m5_channel.rs). + M5, + /// T8152/26A428, 0x98 bytes (GFX b000 tag15 handler 0x12824 reads 0x98). + M6, +} + +impl ChannelForm { + pub(crate) const fn bytes(self) -> usize { + match self { + ChannelForm::M5 => G18_M5_CHANNEL_CONFIG_BYTES, + ChannelForm::M6 => G18_CHANNEL_CONFIG_BYTES, + } + } +} + +/// UMA pool tuple (+0x36 state, +0x3e generation, +0x46 slot). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18ConfigUma { + pub state: u64, + /// FList +0x1c0 when the descriptor asks for it, else None (u64::MAX). + pub generation: Option, + pub slot: u32, +} + +/// Parameter-buffer tuple. Firmware b000 0x13708..0x137c0 caches 0x90 +/// bytes of `state` into its slot table (0x71420 + slot * 0x50) when the +/// generation changes, stores the GPU context at state +0x08 and sets the +/// queue/manager bits. Native TA writer 0x8c83a34..0x8c83a9c. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18ConfigParameterBuffer { + /// PB state object firmware VA (TA descriptor +0x1288). + pub state: u64, + /// Descriptor +0xdc8 object's +0x68 (PB generation). + pub generation: u64, + /// TA HardwareBufferID / PB table slot (+0x1280). + pub slot: u32, + /// TA descriptor +0x1578: the linked 3D queue index. + pub manager: u32, +} + +/// All native tag15 fields. Offsets are for the M6 form. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18ChannelConfig { + /// Kick queue +0x80/+0x88: SKSM entry ring low and firmware addresses. + pub ring_low: u64, + pub ring_high: u64, + pub queue_id: u16, + /// +0x1a: install the queue (TA d+0xe9d, 3D d+0x191, compute d+0x181). + pub install: bool, + /// Kick queue +0x1c. + pub priority: u32, + /// +0x22: stage word, TA 0 / 3D 1 / compute 2. Copied to the firmware + /// queue slot +0x20 (b000 0x12240/0x1241c). + pub data_master: u32, + /// +0x26 flag and +0x2a value (TA d+0xeaa/+0xeb0, 3D d+0x308/+0x310). + /// Native stores the value even when the flag is clear. + pub state_update_flag: bool, + pub state_update: u64, + pub context_id: u32, + pub uma: G18ConfigUma, + /// AGXCommandQueue +0x618 -> +0xe0: the scheduler control record. + pub channel_control: u64, + pub update_scheduler: bool, + /// AGXCommandQueue +0x868 -> +0x10. + pub scheduler_index: u32, + /// Channel +0x58 -> +0x44 (AGXChannel::getPid). + pub pid: u32, + /// Channel +0x58 -> +0x3c (AGXChannel::getSubPriority). Compute uses 2. + pub sub_priority: u32, + pub update_context: bool, + /// Descriptor +0x58 gart's +0x2b8 bit 0 (b000 sets/clears context bit). + pub gart_flag: bool, + /// M6 +0x6e flag and +0x72 value (TA d+0xe9f/+0xea0, 3D d+0x2fa/ + /// +0x300); the value is stored regardless of the flag. Absent on M5. + pub extension_flag: bool, + pub extension: u64, + pub parameter_buffer: Option, +} + +/// Encode a tag15 record. Native writes bytes 0..form.bytes() except the +/// trailing M6 padding +0x92..+0x98, which is left to the caller's buffer. +pub(crate) fn encode_channel_config( + form: ChannelForm, + p: &G18ChannelConfig, + out: &mut [u8], +) -> Result { + let bytes = form.bytes(); + if out.len() < bytes { + return Err(G18RenderCommandError::AddressOverflow); + } + put64(out, 0, u64::from(G18_CONFIG_SELECTOR)); + put64(out, 8, p.ring_low); + put64(out, 0x10, p.ring_high); + put16(out, 0x18, p.queue_id); + put32(out, 0x1a, u32::from(p.install)); + put32(out, 0x1e, p.priority); + put32(out, 0x22, p.data_master); + put32(out, 0x26, u32::from(p.state_update_flag)); + put64(out, 0x2a, p.state_update); + put32(out, 0x32, p.context_id); + put64(out, 0x36, p.uma.state); + put64(out, 0x3e, p.uma.generation.unwrap_or(u64::MAX)); + put32(out, 0x46, p.uma.slot); + put64(out, 0x4a, p.channel_control); + put32(out, 0x52, u32::from(p.update_scheduler)); + put32(out, 0x56, p.scheduler_index); + put32(out, 0x5a, p.pid); + put32(out, 0x5e, p.sub_priority); + put32(out, 0x62, u32::from(p.update_context)); + put32(out, 0x66, u32::from(p.gart_flag)); + put32(out, 0x6a, 0); + let pb = match form { + ChannelForm::M6 => { + put32(out, 0x6e, u32::from(p.extension_flag)); + put64(out, 0x72, p.extension); + 0x7a + } + ChannelForm::M5 => 0x6e, + }; + match p.parameter_buffer { + Some(b) => { + put64(out, pb, b.state); + put64(out, pb + 8, b.generation); + put32(out, pb + 0x10, b.slot); + put32(out, pb + 0x14, b.manager); + } + None => { + put64(out, pb, 0); + put64(out, pb + 8, u64::MAX); + put64(out, pb + 0x10, u64::MAX); + } + } + Ok(bytes) +} + +// ---- SKSM kick entry and tag14 kick ----------------------------------------- + +/// Graphics stage of a kick queue. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum G18RenderStage { + Ta, + ThreeD, +} + +impl G18RenderStage { + const fn dependency_type(self) -> u8 { + match self { + G18RenderStage::Ta => G18_TA_DEPENDENCY_TYPE, + G18RenderStage::ThreeD => G18_3D_DEPENDENCY_TYPE, + } + } + + pub(crate) const fn qos_class(self) -> u8 { + match self { + G18RenderStage::Ta => G18_TA_QOS_CLASS, + G18RenderStage::ThreeD => G18_3D_QOS_CLASS, + } + } + + const fn data_master(self) -> m5_sksm::M5DataMaster { + match self { + G18RenderStage::Ta => m5_sksm::M5DataMaster::Tiling, + G18RenderStage::ThreeD => m5_sksm::M5DataMaster::Fragment, + } + } +} + +/// One vendevtBarrierRec entry, before hardware-stamp conversion. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18Barrier { + /// Event-machine stamp of the producer (e.g. the TA's 0x100). + pub stamp: u32, + /// Producer's queue index; also its qid on Linux. + pub queue_index: u8, + /// Barrier record flag word: 1 selects the stage dependency type. + pub flag: u32, + /// Event-machine generation of the producer queue for this stamp. + pub generation: u64, +} + +/// AGX3DChannelSKSM::submitBuffer 0x8bbba14..0x8bbba80: move the first +/// barrier on the paired TA queue (3D descriptor +0xe60; 0x80 for fast blit) +/// to the front of the list, swapping it with entry 0. +pub(crate) fn g18_order_3d_barriers(barriers: &mut [G18Barrier], ta_queue_index: u32) { + if let Some(index) = barriers.iter().position(|b| u32::from(b.queue_index) == ta_queue_index) { + barriers.swap(0, index); + } +} + +/// Inputs of encodeTAKickEntry / encode3DKickEntry beyond the kick queue. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18KickEntryInputs<'a> { + pub stage: G18RenderStage, + /// Kick queue id/priority/current stamp/previous stamp (+0xc/+0x1c/ + /// +0x48/+0x98/+0xa0); `submitted` counts entries in flight. + pub queue: M5SksmQueue, + /// Kick queue +0x10 entries and +0x90 base (barrier conversion). + pub entries: u32, + pub base: u64, + /// This submission's event stamp and its queue's event generation, for + /// the implicit predecessor when the queue has no previous stamp. + pub stamp: u32, + pub generation: u64, + pub barriers: &'a [G18Barrier], + /// TA descriptor +0xfdc / 3D +0x444 (AGXSPipelineDependencyType). + pub pipeline_dependency: u32, + /// AGXCommandQueue +0x868 -> +0x10 (QoS byte 32). + pub scheduler_index: u8, + /// Per-stage QoS word (0xffff without a scheduler override). + pub qos_word: u16, + /// Command firmware VA and the channel's +0x80 queue address. + pub payload: [u64; 2], + pub rce: [M5SksmRce; 4], + /// Descriptor +0x4c bit 0 (TA) or (3D +0xc9c == 3 ? 1 : +0x4c) -> bit 1. + pub flag_bit1: bool, + /// TA +0x15f0 / 3D +0xb78 -> bit 61; TA +0x15f1 / 3D +0x838 -> bit 63. + pub flag_bit61: bool, + pub flag_bit63: bool, + /// Event flag word 3: bit 63 config +0xe20, bit 62 extra command + /// present, bit 61 accelerator +0x6b6 & 0xc0. + pub event_flags: u64, + /// Descriptor +0x178, or accelerator +0x11f28[index & 0x7f]. + pub auxiliary: u64, +} + +/// Build the HAL400 kick-entry operands for m5_sksm::prepare_m5_sksm_entry. +/// The dependency list follows encodeDependencyList 0x8c4a5a4. +pub(crate) fn g18_kick_entry_operands<'a>( + p: &G18KickEntryInputs<'_>, + barriers: &'a mut [M5SksmDependency], +) -> Result<(M5SksmOperands<'a>, u64), G18RenderCommandError> { + if barriers.len() < p.barriers.len() || p.barriers.len() >= 32 { + return Err(G18RenderCommandError::Sksm); + } + let kind = |flag: bool| if flag { p.stage.dependency_type() } else { 1 }; + for (out, b) in barriers.iter_mut().zip(p.barriers.iter()) { + let stamp = g18_hardware_stamp(&G18HardwareStamp { + stamp: b.stamp, entries: p.entries, generation: b.generation, base: p.base, + })?; + *out = M5SksmDependency { queue_id: b.queue_index & 0x7f, stamp, kind: kind(b.flag == 1) }; + } + let first = g18_hardware_stamp(&G18HardwareStamp { + stamp: p.stamp, entries: p.entries, generation: p.generation, base: p.base, + })? + .wrapping_sub(1); + let count = p.barriers.len(); + Ok((M5SksmOperands { + barriers: &barriers[..count], + first_predecessor: first, + implicit_dependency_type: kind(p.pipeline_dependency == 1), + mcache: None, + flag_bit1: p.flag_bit1, + flag_bit61: p.flag_bit61, + flag_bit63: p.flag_bit63, + payload: p.payload, + qos_byte_32: p.scheduler_index, + qos_class_40: p.stage.qos_class(), + qos_word_48: p.qos_word, + event_mask: [0, 0, 0, p.event_flags], + rce_kind: 0, + rce: p.rce, + auxiliary: p.auxiliary, + }, first)) +} + +/// Encode the kick entry into its 0x200-byte slot image. +pub(crate) fn encode_g18_kick_entry( + p: &G18KickEntryInputs<'_>, +) -> Result { + let mut storage = [M5SksmDependency { queue_id: 0, stamp: 0, kind: 0 }; 31]; + let (operands, _) = g18_kick_entry_operands(p, &mut storage)?; + m5_sksm::prepare_m5_sksm_entry(p.queue, operands).map_err(|_| G18RenderCommandError::Sksm) +} + +/// Native tag14 add-kicks record (TA 0x8c8430c..0x8c84334, 3D 0x8bbc8f0.. +/// 0x8bbc91c): count is the descriptor's submitted-kick total (NOP entries +/// plus this one), DM 0 for TA and 1 for 3D. +pub(crate) fn encode_g18_kick( + stage: G18RenderStage, + queue_id: u8, + priority: u8, + stamp: u64, + count: u16, +) -> Result<[u8; G18_KICK_BYTES], G18RenderCommandError> { + m5_sksm::encode_m5_sksm_add_kicks_for(stage.data_master(), queue_id, priority, stamp, count) + .map_err(|_| G18RenderCommandError::Kick) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/render_params.rs b/drivers/gpu/drm/asahi/g18/fw/render_params.rs new file mode 100644 index 0000000000000..65bd30235ecf4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/render_params.rs @@ -0,0 +1,2877 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! Pure T8140/G17P render-command construction. +//! +//! The byte layout comes from m1n1 +//! `916548424c00c72176326e3811badedce5714a99`. The source profile carries the +//! hardware-tested TA and fragment register recipes, the compiler-derived EOT +//! resource binding `0x8`, full-width load binding, and native-aligned BG/EOT +//! program pointers. Descriptor construction also writes the duplicated structural +//! pipeline fields. Queue construction returns ordered memory writes only; +//! allocation, cache maintenance, barriers, doorbells, MMIO, and completion +//! stay with the retained runtime. + +const TA_REGISTER_COUNT: usize = 73; +const FRAGMENT_REGISTER_COUNT: usize = 89; + +/// Both embedded register lists match the kext EXACTLY -- same writes, same +/// unique registers, same order -- verified against 2048-byte captures of the +/// native TA and 3D command records (macos-trace recs2048-gfx.txt): +/// +/// TA 73 writes, 71 unique, list ends at +0x3cc +/// 3D 89 writes, 86 unique, list ends at +0x4cc +/// +/// A 1024-byte capture cuts the 3D list at exactly +0x400 and hides its last +/// 17 writes, which briefly looked like 14 registers "Metal never writes" -- +/// they were entries 72..85. Assert the counts so a future truncated capture +/// cannot produce that conclusion again. +const _: () = assert!(TA_REGISTER_COUNT == 73); +const _: () = assert!(FRAGMENT_REGISTER_COUNT == 89); +const PARTIAL_STORE_REGISTER_COUNT: usize = 16; +const PARTIAL_RESUME_REGISTER_COUNT: usize = 23; +const PARTIAL_LOAD_REGISTER_COUNT: usize = 10; +const REGISTER_BYTES: usize = 12; + +pub(crate) const TA_DESCRIPTOR_SIZE: usize = 0x9c0; +pub(crate) const FRAGMENT_DESCRIPTOR_SIZE: usize = 0x2240; +/// Native render-setup field copied through command descriptor +0xc00 to the +/// final 3D command tail at +0x2110. This is independent of SGX's GPC +/// performance-state map register. +const G17P_FRAGMENT_TAIL_SETUP_WORD: u32 = u32::MAX; +/// J700 shared-runtime flag snapshot copied by native 3D submit into type-1 +/// command lanes +0x2124/+0x2128. +/// +/// The kext resolves the accelerator's shared event-stamp/HWDSID internal +/// resource at +0x1a130, obtains its CPU mapping, reads byte bit 0 at mapping +/// +0x2ad/+0x2ae, and widens them before per-draw register generation. The +/// working Neo Metal snapshot contains 1/1. +const G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT: [u32; 2] = [1, 1]; +pub(crate) const G17P_EOT_RESOURCE_SPEC: u64 = 0x8; +pub(crate) const G17P_LOAD_RESOURCE_SPEC: u64 = 0x0007_8000_0000_0040; +pub(crate) const G17P_SOURCE_BG_PIPELINE: u64 = 0x0000_0100_0199_0240; +pub(crate) const G17P_SOURCE_EOT_PIPELINE: u64 = 0x0000_0100_0199_0640; +/// Native `mcacheApertureBufferSetup` writes its packed write ranges here in +/// the 3D firmware command record. The KSM entry points at this table. +pub(crate) const G17P_FRAGMENT_MCACHE_OFFSET: usize = 0x6a0; +/// The tiler's MCache aperture table. `mcacheApertureBufferSetup` is called for +/// the TA with `ksmBufferInfo = recTA+0x60` and writes its entries at +/// `ksmBufferInfo + 0x600 + i*0x10`, i.e. `recTA + 0x660` -- the exact analogue +/// of the 3D's `rec3D + 0x6a0`. +pub(crate) const G17P_TILING_MCACHE_OFFSET: usize = 0x660; +pub(crate) const G17P_MCACHE_MAPPING_SIZE: usize = 0x10; +/// Temporary diagnostic mirror of the four Register Command Encoder programs. +/// Native keeps the complete programs in the 3D command itself: the first is +/// at command +0xa0 and the remaining phases follow at a 0x720-byte stride. +/// Each phase's +0x700 trailer points back at that same embedded phase and its +/// +0x708 word carries the entry count. KSM must bind those complete embedded +/// structures, not this entries-only mirror. +pub(crate) const G17P_FRAGMENT_RCE_STORAGE_SIZE: usize = 0x4000; +pub(crate) const G17P_FRAGMENT_RCE_PROGRAM_OFFSET: u64 = 0xa0; +pub(crate) const G17P_FRAGMENT_RCE_PROGRAM_STRIDE: u64 = 0x720; + +const TILE_WIDTH: u64 = 32; +const TILE_HEIGHT: u64 = 32; +const MACRO_TILES_X: u64 = 4; +const MACRO_TILES_Y: u64 = 4; +const REGION_ENTRY_SIZE: u64 = 5; +const EVENT_SELECTOR: u32 = 0x0e; +const EVENT_SUBTYPE_BASE: u32 = 0x0001_0000; +const EVENT_RECORD_SIZE: usize = 0x40; +const RING_SLOT_SIZE: usize = 0x18; +const QUEUE_WRITE_OFFSET: u64 = 0x40; +/// Base added by G17P's compact VDM control-stream register (0x1c880). +/// The register retains the legacy 40-bit {TTBR selector, 39-bit offset} +/// shape, so lower-root user addresses are encoded relative to this base. +const G17P_USER_BASE: u64 = 0x10_0000_0000; +const G17P_VDM_USER_WINDOW_SIZE: u64 = 1 << 39; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RenderBuildError { + ZeroDimensions, + LayerCountOutOfRange, + UtileDimensions, + AddressBeforeContext, + UserAddressOutOfRange, + VdmAddressOutOfRange, + BufferTooSmall, + ArithmeticOverflow, + QueueGridOutOfRange, + RingSlotOutOfRange, + McacheAddressOutOfRange, + McacheRangeEmpty, + McacheSelectorOutOfRange, +} + +impl RenderBuildError { + /// Stable short name for one descriptor-build refusal. Callers flatten this + /// enum to `EINVAL`, which erased the distinction between a geometry + /// rejection and an address that sits below the render context base. + pub(crate) fn name(&self) -> &'static str { + match self { + Self::ZeroDimensions => "zero-dimensions", + Self::LayerCountOutOfRange => "layer-count-out-of-range", + Self::UtileDimensions => "utile-dimensions", + Self::AddressBeforeContext => "address-before-context", + Self::UserAddressOutOfRange => "user-address-out-of-range", + Self::VdmAddressOutOfRange => "vdm-address-out-of-range", + Self::BufferTooSmall => "buffer-too-small", + Self::ArithmeticOverflow => "arithmetic-overflow", + Self::QueueGridOutOfRange => "queue-grid-out-of-range", + Self::RingSlotOutOfRange => "ring-slot-out-of-range", + Self::McacheAddressOutOfRange => "mcache-address-out-of-range", + Self::McacheRangeEmpty => "mcache-range-empty", + Self::McacheSelectorOutOfRange => "mcache-selector-out-of-range", + } + } +} + +/// One resource-derived G17P MCache write-range mapping. +/// +/// This is the exact input shape consumed by +/// `AGXAcceleratorG16::halMCacheApertureSetup`: address and size are in +/// 128-byte cache lines; `hwsid` is the allocated hardware DSID; `selector` +/// names the command's GART/UAT context; and `range_shift` is the native +/// write-range class (clamped to four by the kext). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PMcacheMapping { + pub(crate) address: u64, + pub(crate) size_units: u32, + pub(crate) hwsid: u8, + pub(crate) selector: u8, + pub(crate) range_shift: u16, +} + +/// Encode the two qwords written by the G17P kext at 3D descriptor +/// `+0x6a0 + index * 0x10`. +pub(crate) fn encode_g17p_mcache_mapping( + mapping: G17PMcacheMapping, +) -> Result<[u8; G17P_MCACHE_MAPPING_SIZE], RenderBuildError> { + if mapping.address & 0x7f != 0 || mapping.address >> 48 != 0 { + return Err(RenderBuildError::McacheAddressOutOfRange); + } + if mapping.size_units == 0 { + return Err(RenderBuildError::McacheRangeEmpty); + } + if mapping.selector > 0x3f { + return Err(RenderBuildError::McacheSelectorOutOfRange); + } + + let start = mapping.address >> 7; + let end = start + .checked_add(mapping.size_units as u64) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + // qword 1 reserves bits 41..46 for the range policy. + if start >> 41 != 0 || end >> 41 != 0 { + return Err(RenderBuildError::McacheAddressOutOfRange); + } + + let q0 = start + | ((mapping.selector as u64) << 41) + | ((mapping.hwsid as u64) << 47) + | (1u64 << 55); + let range_shift = core::cmp::min(mapping.range_shift, 4) as u32; + let q1 = end | (1u64 << 41) | (1u64 << (42 + range_shift)); + let mut out = [0u8; G17P_MCACHE_MAPPING_SIZE]; + out[..8].copy_from_slice(&q0.to_le_bytes()); + out[8..].copy_from_slice(&q1.to_le_bytes()); + Ok(out) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWrite { + pub(crate) number: u32, + pub(crate) value: u64, +} + +impl RegisterWrite { + const fn new(number: u32, value: u64) -> Self { + Self { number, value } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17pRenderParameters { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) context_base: u64, + /// Allocation epoch of the trusted execution-context lease, shared by TA/3D. + pub(crate) context_generation: u8, + pub(crate) tilemap: u64, + /// Per-layer tiler metadata, which native places between the tilemap and + /// the heap metadata. It is empty, and so equal to `heapmeta`, for a + /// single-layer target. + pub(crate) layermeta: u64, + pub(crate) heapmeta: u64, + pub(crate) tpc: u64, + /// Geometry size of the owned TPC subregion, before capacity page rounding. + pub(crate) tpc_size: u64, + pub(crate) deflake_1: u64, + pub(crate) deflake_2: u64, + pub(crate) deflake_3: u64, + pub(crate) encoder: u64, + pub(crate) ta_status: u64, + pub(crate) fragment_status: u64, + pub(crate) store_pipeline_bind: u64, + pub(crate) store_pipeline: u64, + pub(crate) load_pipeline_bind: u64, + pub(crate) load_pipeline: u64, + pub(crate) partial_store_pipeline_bind: u64, + pub(crate) partial_store_pipeline: u64, + pub(crate) partial_load_pipeline_bind: u64, + pub(crate) partial_load_pipeline: u64, + pub(crate) scissor_array: u64, + pub(crate) depth_bias_array: u64, + pub(crate) aux_fb: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth_buffer: u64, + pub(crate) stencil_buffer: u64, + pub(crate) depth_aux_buffer: u64, + pub(crate) stencil_aux_buffer: u64, + pub(crate) depth_stride: u64, + pub(crate) stencil_stride: u64, + pub(crate) depth_aux_stride: u64, + pub(crate) stencil_aux_stride: u64, + pub(crate) depth_clear_value_bits: u32, + pub(crate) stencil_clear_value: u32, + pub(crate) depth_flags: u64, + /// ZLS control used by every firmware partial-store/load/resume phase. + /// Unlike the pass-boundary control, an attached depth/stencil surface + /// must be both loaded and stored across an internal split. + pub(crate) partial_depth_flags: u64, + pub(crate) depth_dimensions: u64, + pub(crate) merge_upper_x_bits: u32, + pub(crate) merge_upper_y_bits: u32, + pub(crate) layers: u16, + /// Sample count: 1, 2 or 4. Defaults to 1, so every existing parameter + /// vector keeps its exact register program. + pub(crate) samples: u8, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) utile_config: u64, + pub(crate) multisample_control: u64, + pub(crate) ppp_control: u64, + pub(crate) tib_blocks: u64, + pub(crate) tile_config: u64, + pub(crate) aux_fb_flags: u64, + pub(crate) aux_fb_page_count: u64, + pub(crate) cycle: u64, + pub(crate) record_index: u64, + /// TA event correlation written to 0x1ca10/0x014a1/0x0a349. + /// Native sources this independently from the fragment correlation. + pub(crate) lifecycle: u64, + /// Fragment event correlation written to 0x160e0/0x01499/0x0a341. + pub(crate) fragment_lifecycle: u64, + pub(crate) sampler_array: u64, + pub(crate) sampler_count: u32, + pub(crate) process_empty_tiles: bool, + pub(crate) ta_timestamp_start: u64, + pub(crate) ta_timestamp_end: u64, + pub(crate) fragment_timestamp_start: u64, + pub(crate) fragment_timestamp_end: u64, + /// Write the VDM control-stream register (0x1c880) as a full GPU virtual + /// address instead of the native user-process compact form. + /// + /// SCOPE CORRECTED against the macOS oracle. This flag once covered every + /// register `ta_offset` feeds, justified from the legacy G14X path where + /// all of them are whole pointers. A real Metal render pass on THIS chip + /// says otherwise: the captured TA command record writes tilemap, TPC and + /// heapmeta as small context-relative offsets (see `ta_offset`). The legacy + /// encoding does not transfer to G17P, so those six registers are always + /// relative now and this flag applies to the VDM stream alone. Native first + /// lowers the task VA to a task-relative offset, maps the same backing at + /// 0x1000000000 + offset in the GART window, and writes that offset here. + /// Linux userspace supplies the full GART-alias VA; this kernel subtracts + /// the architectural 0x1000000000 base. Hardware adds it back for compact + /// pointers whose bit 39 is clear. + pub(crate) absolute_pointers: bool, + /// + /// Legacy diagnostic OR-mask for the BG resource-specifier high 17 bits. + /// Native calls this field PassControlMaskType and derives it from shader + /// outputs, including extension of the highest active pass bit; it is not + /// an address class or a universal constant. Main and partial shaders can + /// legitimately differ. Default zero preserves the userspace mask; + /// a nonzero value is an explicit diagnostic override only. + pub(crate) bg_resource_prefix: u64, + /// Historical opt-in diagnostic changing byte 2 of the USC private-memory + /// budget qword at TA +0x93d. It must not override +0x94d: that byte is the + /// allocation generation of the execution context, not a PM mode flag. + /// The captured budget is not yet derived from our pool allocation. + pub(crate) native_pm_bytes: bool, + /// Hardware-buffer ID allocated for this TA command. Native allocates it + /// from the accelerator's 0x7f-entry `AGXHardwareBufferIDManager`, stores + /// it in the host-side `AGXTACommandDescriptor` at +0x1248/+0x928, and the + /// register generator emits that value as register 0x1c830. Those two host + /// offsets are not fields in this final 0x9c0-byte firmware command. + pub(crate) ta_hardware_buffer_id: u32, + /// HardwareBufferID of the retained USC private-memory FList shared by TA + /// and 3D. Native allocates this from the USC manager (accelerator + /// +0x133c8), independently of the TA parameter-buffer manager above, and + /// emits the same ID in TA register 0x14308 and fragment register 0x14048. + pub(crate) usc_flist_hardware_buffer_id: u32, + /// Diagnostic capture-value bitmask retained for fields whose proper + /// inputs are not modeled yet. In particular, the 0xa5a* values are + /// assembled from workload-derived USC pool limits. Their captured values + /// are not architectural literals and must not become defaults. + pub(crate) native_ta_registers: u32, + /// Retained GPU-ID diagnostic snapshot, not the fragment parameter-buffer + /// owner. The old 0x1c838 use confused host descriptor +0x928 with a + /// different object's transient raw-command +0x928 field. + pub(crate) gpc_perf_state_map: u32, + /// Low nibble of SGX 0xe01480, copied by native 3D submit to tail +0x210c. + pub(crate) gpc_perf_state_map_low: u32, + /// Bit 0 of SGX 0xe0141c, copied by native 3D submit to tail +0x2104. + pub(crate) gpc_perf_state_control: u32, + /// GPU address of the selected native 0x80 Scene-state entry, or 0 for + /// none. The command-record binding is carried by `RenderDescriptorObjects` + /// in both TA and 3D; this copy is retained for diagnostics. + pub(crate) parameter_buffer: u64, +} + +impl G17pRenderParameters { + /// Apply the witnessed multisample register triple for a sample count. + /// + /// Captured constants, not derived: from Apple's own renders on this part + /// (logs/m5-msaa-native-hv-20260915-101837) at one, two and four samples, + /// with a same-mode control in a second oracle process establishing the + /// noise floor. `multisample_control` is the sample POSITION table -- + /// 0x88 is the centre (8,8), 0x44cc is (4,4),(12,12), and 0xeaa26e26 + /// unpacks to (14,10),(10,2),(6,14),(2,6), the rotated-grid 4x pattern in + /// 1/16 pixel units. `utile_config`'s low nibble is log2(samples); + /// `tib_blocks` is samples << 16, zero at one sample. + pub(crate) fn set_samples(&mut self, samples: u8) -> bool { + let (multisample_control, utile_config, tib_blocks) = match samples { + 1 => (0x88, 0xa000, 0), + 2 => (0x44cc, 0xa001, 0x2_0000), + 4 => (0xeaa2_6e26, 0x6002, 0x4_0000), + _ => return false, + }; + self.samples = samples; + self.multisample_control = multisample_control; + self.utile_config = utile_config; + self.tib_blocks = tib_blocks; + true + } + + pub(crate) const fn source_rsrc8() -> Self { + Self { + width: 64, + height: 64, + context_base: 0x0000_0010_0000_0000, + context_generation: 1, + tilemap: 0x0000_0010_001b_0000, + layermeta: 0x0000_0010_001b_5000, + heapmeta: 0x0000_0010_001b_5000, + tpc: 0x0000_0010_0024_0000, + tpc_size: 0x800, + deflake_1: 0x0000_0010_0006_82a0, + deflake_2: 0x0000_0010_0006_8020, + deflake_3: 0x0000_0010_0006_8000, + encoder: 0x0000_0010_0001_8000, + ta_status: 0x0000_0010_0007_8000, + fragment_status: 0x0000_0010_001a_8000, + store_pipeline_bind: G17P_EOT_RESOURCE_SPEC, + store_pipeline: G17P_SOURCE_EOT_PIPELINE, + load_pipeline_bind: G17P_LOAD_RESOURCE_SPEC, + load_pipeline: G17P_SOURCE_BG_PIPELINE, + partial_store_pipeline_bind: 0, + partial_store_pipeline: 0, + partial_load_pipeline_bind: 0, + partial_load_pipeline: 0, + scissor_array: 0x0000_0100_019a_0000, + depth_bias_array: 0x0000_0100_01af_8000, + aux_fb: 0x0000_0100_01aa_8000, + occlusion_query_base: 0, + depth_buffer: 0, + stencil_buffer: 0, + depth_aux_buffer: 0, + stencil_aux_buffer: 0, + depth_stride: 0, + stencil_stride: 0, + depth_aux_stride: 0, + stencil_aux_stride: 0, + depth_clear_value_bits: 0x3f80_0000, + stencil_clear_value: 0, + depth_flags: 0, + partial_depth_flags: 0, + depth_dimensions: 0, + merge_upper_x_bits: 0x3cdd_b3d9, + merge_upper_y_bits: 0x3cdd_b3d9, + layers: 1, + samples: 1, + utile_width: 32, + utile_height: 32, + utile_config: 0xa000, + multisample_control: 0x88, + ppp_control: 0x202, + tib_blocks: 8, + tile_config: 0x10280, + aux_fb_flags: 0xc001, + aux_fb_page_count: 0x10_0000, + cycle: 0x17_8020, + record_index: 0x8_0005, + lifecycle: 0, + fragment_lifecycle: 0, + sampler_array: 0, + sampler_count: 0, + process_empty_tiles: true, + ta_timestamp_start: 0xffff_fc20_0002_4c68, + ta_timestamp_end: 0, + fragment_timestamp_start: 0xffff_fc20_0002_4c68, + fragment_timestamp_end: 0xffff_fc20_0002_4c70, + absolute_pointers: false, + bg_resource_prefix: 0, + native_pm_bytes: false, + ta_hardware_buffer_id: 0, + usc_flist_hardware_buffer_id: 0, + native_ta_registers: 0, + gpc_perf_state_map: 0, + gpc_perf_state_map_low: 0, + gpc_perf_state_control: 0, + parameter_buffer: 0, + } + } +} + +#[derive(Debug, Copy, Clone)] +struct Geometry { + tiles_x: u64, + tiles_y: u64, + size1: u64, + size2: u64, + size3: u64, + x_blocks: u64, + y_blocks: u64, + screen: u64, + pixels: u64, + macro_size: u64, +} + +const fn align(value: u64, alignment: u64) -> u64 { + (value + alignment - 1) & !(alignment - 1) +} + +fn geometry(parameters: &G17pRenderParameters) -> Result { + if parameters.width == 0 || parameters.height == 0 { + return Err(RenderBuildError::ZeroDimensions); + } + if parameters.layers == 0 || parameters.layers > 2048 { + return Err(RenderBuildError::LayerCountOutOfRange); + } + + let width = parameters.width as u64; + let height = parameters.height as u64; + let tiles_x = (width + TILE_WIDTH - 1) / TILE_WIDTH; + let tiles_y = (height + TILE_HEIGHT - 1) / TILE_HEIGHT; + let (utile_width, utile_height) = (parameters.utile_width, parameters.utile_height); + if !matches!((utile_width, utile_height), (32, 32) | (32, 16) | (16, 16)) { + return Err(RenderBuildError::UtileDimensions); + } + let utiles_per_tile_x = TILE_WIDTH / utile_width as u64; + let utiles_per_tile_y = TILE_HEIGHT / utile_height as u64; + // Four samples split the utile in Y. Measured against Apple's own + // four-sample render: size1 0x14 -> 0x28, size3 0x20 -> 0x40 and + // macro_size 0x40004 -> 0x40008, where only the y term moves. All three + // follow from doubling this one term; two samples do not split. + let utiles_per_tile_y = if parameters.samples == 4 { + utiles_per_tile_y * 2 + } else { + utiles_per_tile_y + }; + let utiles_per_tile = utiles_per_tile_x * utiles_per_tile_y; + let mtile_x1 = align((tiles_x + MACRO_TILES_X - 1) / MACRO_TILES_X, 4); + let mtile_y1 = align((tiles_y + MACRO_TILES_Y - 1) / MACRO_TILES_Y, 4); + let mtile_x2 = 2 * mtile_x1; + let mtile_x3 = 3 * mtile_x1; + let mtile_y2 = 2 * mtile_y1; + let mtile_y3 = 3 * mtile_y1; + let mtile_stride = mtile_x1 * mtile_y1; + let size1 = (REGION_ENTRY_SIZE * mtile_stride * utiles_per_tile + 3) / 4; + + Ok(Geometry { + tiles_x, + tiles_y, + size1, + size2: mtile_stride, + size3: 2 * mtile_stride * utiles_per_tile, + x_blocks: mtile_x3 | (mtile_x2 << 9) | (mtile_x1 << 18), + y_blocks: mtile_y3 | (mtile_y2 << 9) | (mtile_y1 << 18), + screen: ((tiles_y - 1) << 12) | (tiles_x - 1), + pixels: (width - 1) | ((height - 1) << 16), + macro_size: mtile_y1 * utiles_per_tile_y | ((mtile_x1 * utiles_per_tile_x) << 16), + }) +} + +/// Context-relative offset. ALWAYS relative -- a real Metal render pass on this +/// chip writes small offsets into these registers, not addresses. +/// +/// Oracle: the TA command record captured from `mtlg.m` (macos-trace +/// `recs-gfx.txt`, dumped by `recs.d` at `requestChannelCommandTA`) carries an +/// embedded register list that decodes field-for-field against this function's +/// output -- same registers, same order, 12-byte entries of u32 number plus u64 +/// value -- with these values: +/// +/// 0x01748 = 1 (matches) +/// 0x10141 = 0x200 (matches) +/// 0x1c039 = 0x0119ba00 tilemap <- OFFSET, not a VA +/// 0x1c9c8 = 0x0119ba00 tilemap +/// 0x1c0a1 = 0x01290000 tpc <- OFFSET +/// 0x1c031 = 0x800000000119ca00 heapmeta | bit63 (matches the form) +/// 0x1c9c0 = 0x800000000119ca00 heapmeta +/// 0x1c051 = 0x003a0012006b0003 (matches the constant exactly) +/// +/// Writing full virtual addresses here -- which is what the legacy G14X path +/// does, and what an earlier version of this driver was changed to do -- points +/// the tiler's tilemap, TPC and heapmeta at values the firmware cannot use. +fn ta_resource_offset( + parameters: &G17pRenderParameters, + address: u64, +) -> Result { + address + .checked_sub(parameters.context_base) + .ok_or(RenderBuildError::AddressBeforeContext) +} + +/// Compact user/GART address consumed by the TA register and descriptor +/// fields. Native maps these objects in the architectural 0x1000000000 window +/// and publishes the offset, not the full address. Descriptor +0x7d6 is only +/// 32 bits wide, so reject aliases that cannot be represented consistently. +fn ta_user_offset(address: u64) -> Result { + let offset = address + .checked_sub(G17P_USER_BASE) + .ok_or(RenderBuildError::UserAddressOutOfRange)?; + + if offset > u32::MAX as u64 { + return Err(RenderBuildError::UserAddressOutOfRange); + } + + Ok(offset) +} + +fn vdm_user_offset(address: u64) -> Result { + let offset = address + .checked_sub(G17P_USER_BASE) + .ok_or(RenderBuildError::VdmAddressOutOfRange)?; + + if offset >= G17P_VDM_USER_WINDOW_SIZE { + return Err(RenderBuildError::VdmAddressOutOfRange); + } + + Ok(offset) +} + +pub(crate) fn build_ta_registers( + p: &G17pRenderParameters, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], RenderBuildError> { + build_ta_registers_for_work_key(p, render_work_key(1, 0)) +} + +fn build_ta_registers_for_work_key( + p: &G17pRenderParameters, + work_key: u64, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], RenderBuildError> { + let g = geometry(p)?; + let tilemap = ta_resource_offset(p, p.tilemap)?; + let layermeta = ta_resource_offset(p, p.layermeta)?; + let heapmeta = ta_resource_offset(p, p.heapmeta)?; + let tpc = ta_resource_offset(p, p.tpc)?; + let deflake_1 = ta_user_offset(p.deflake_1)?; + let deflake_2 = ta_user_offset(p.deflake_2)?; + let deflake_3 = ta_user_offset(p.deflake_3)?; + // The VDM control stream is the one pointer the 192-byte capture window + // does not reach (register 0x1c880 sits past the end of the dump), and it + // is the one that is not relative to the driver-private `context_base`. + // macOS first lowers the task VA to a task-relative offset, then maps the + // backing at 0x1000000000 + offset in the GART window. Linux userspace gives + // us that full alias VA, so this hardware encoder subtracts the GART base. + // Bit 39 selects the upper UAT root; refuse an offset that would silently + // change roots. Keep the full-address escape hatch diagnostic-only. + let encoder = if p.absolute_pointers { + p.encoder + } else { + vdm_user_offset(p.encoder)? + }; + // generateTARegisterListG16Common: the layer count less one with bit 15, + // plus bit 14 and a process-empty-tiles select for a layered target. The + // pipe mirror 0x1c9e8 keeps only the layered bit and the count. + let mut layer_mode = 0x8000 | (p.layers as u64 - 1); + if p.layers > 1 { + layer_mode |= 0x4000 | if p.process_empty_tiles { 0x2000 } else { 0x1000 }; + } + + Ok([ + RegisterWrite::new(0x01748, 1), + // Native N=1 uses 0x200 and N=2 uses 0x0. The one-layer source profile + // selects the N=1 value, matching descriptor byte +0x789 above. + RegisterWrite::new(0x10141, 0x200), + RegisterWrite::new(0x1c039, tilemap), + RegisterWrite::new(0x1c9c8, tilemap), + RegisterWrite::new(0x1c0a1, tpc), + RegisterWrite::new(0x1c031, heapmeta | 0x8000_0000_0000_0000), + RegisterWrite::new(0x1c9c0, heapmeta | 0x8000_0000_0000_0000), + RegisterWrite::new(0x1c051, 0x003a_0012_006b_0003), + RegisterWrite::new(0x1c061, 1), + RegisterWrite::new(0x10149, p.utile_config), + RegisterWrite::new(0x10139, p.multisample_control), + RegisterWrite::new(0x10111, deflake_1), + RegisterWrite::new(0x1c9b0, deflake_1), + RegisterWrite::new(0x10119, deflake_2), + RegisterWrite::new(0x1c9b8, deflake_2), + RegisterWrite::new(0x1c958, 1), + RegisterWrite::new(0x1c950, deflake_3 | 0x0004_0000_0000_0000), + RegisterWrite::new(0x1c930, 0), + RegisterWrite::new(0x1c880, encoder), + RegisterWrite::new(0x1c079, layermeta), + RegisterWrite::new(0x1c9d8, layermeta), + RegisterWrite::new(0x10151, 0), + RegisterWrite::new(0x1c199, 0), + RegisterWrite::new(0x1c1a1, 0), + RegisterWrite::new(0x1c1a9, 0), + RegisterWrite::new(0x1c1b1, 0), + RegisterWrite::new(0x1c1b9, 0), + RegisterWrite::new(0x1c8f8, 0x8860), + RegisterWrite::new(0x1c0b1, g.size1), + RegisterWrite::new(0x1c850, g.size1), + // Silicon holds 0x10131 at 0x88 at ONE, TWO and FOUR samples while + // 0x10139 carries the real sample position table (all five captures in + // logs/m5-msaa-native-hv-20260915-101837). Driving this from + // multisample_control would write the position table here, which + // Apple never does. + RegisterWrite::new(0x10131, 0x88), + RegisterWrite::new(0x10121, p.ppp_control), + RegisterWrite::new(0x10129, g.pixels), + RegisterWrite::new(0x101b9, g.screen), + RegisterWrite::new(0x1c069, g.x_blocks), + RegisterWrite::new(0x1c071, g.y_blocks), + RegisterWrite::new(0x1c081, g.size2), + RegisterWrite::new(0x1c0a9, g.size3), + RegisterWrite::new(0x10171, 0x100), + RegisterWrite::new(0x10169, layer_mode), + RegisterWrite::new(0x0a309, 0), + RegisterWrite::new(0x1c8e0, u64::MAX), + RegisterWrite::new(0x1c8e8, u64::MAX), + RegisterWrite::new(0x1c898, 0), + RegisterWrite::new(0x101e1, 0x1c), + RegisterWrite::new(0x1c9e8, layer_mode & 0x47ff), + RegisterWrite::new(0x1a099, 0), + RegisterWrite::new(0x1a0a1, 0), + RegisterWrite::new(0x1a069, 0), + RegisterWrite::new(0x1a071, 0), + RegisterWrite::new(0x1a0c9, 0), + RegisterWrite::new(0x1a0d1, 0), + RegisterWrite::new(0x101c9, 0), + RegisterWrite::new(0x0d471, 0), + RegisterWrite::new(0x1a0f1, 8), + RegisterWrite::new(0x10799, 0xff_0000), + // Native sources this from AGXHardwareBufferIDManager. It is neither a + // PBDesc ring index nor a capture-stable literal. + RegisterWrite::new(0x1c830, p.ta_hardware_buffer_id as u64), + RegisterWrite::new(0x1ca30, p.cycle), + RegisterWrite::new(0x16c39, p.cycle), + RegisterWrite::new(0x1c910, p.record_index), + // Native derives the high word from the TA AGXUMADataRec PTC/DUPM + // maximum at host descriptor +0x1488, copied through command +0x1158. + // halUpdatePTCMaxCount computes max(0x60, min(pool_input / active_kicks, + // 0x200)); Linux does not yet carry that pool input. Follow the kext's + // exact zero-source fallback 0x60_0040_0020. Bit 1 retains one working + // Metal trace's workload-derived 0xfe value as a diagnostic only. + RegisterWrite::new( + 0x0a5a1, + if p.native_ta_registers & 2 != 0 { + 0x0000_00fe_0040_0020 + } else { + 0x0000_0060_0040_0020 + }, + ), + RegisterWrite::new(0x0d419, 0x0000_0002_0000_0001), + RegisterWrite::new(0x1ca10, p.lifecycle), + RegisterWrite::new(0x014a1, p.lifecycle), + RegisterWrite::new(0x0a349, p.lifecycle), + RegisterWrite::new(0x10209, work_key), + RegisterWrite::new(0x1c9f0, work_key), + RegisterWrite::new(0x14320, work_key), + // AGXTACommandDescriptor::prepare receives the shared USC FList ID + // through descriptor +0x1410. Optional GART/feature payloads are absent + // in Linux, so the retained owner's ID is the complete value. + RegisterWrite::new(0x14308, p.usc_flist_hardware_buffer_id as u64), + RegisterWrite::new(0x14318, p.ta_status | 1), + RegisterWrite::new(0x01740, 1), + RegisterWrite::new(0x1c880, deflake_1), + RegisterWrite::new(0x1c898, 1), + ]) +} + +/// DIAGNOSTIC. Write native's value into TA-descriptor fields where ours differs +/// structurally at MATCHED 256x256 geometry (recs3.txt TA record, stride 0x9c0). +/// The tiler bins ~60 bytes of a 1.28 MB TVB with every input verified correct, +/// so these are what is left that we demonstrably send differently. +fn apply_g17p_ta_oracle(raw: &mut [u8]) { + let mask = crate::g18::fw::config::TA_ORACLE; + if mask == 0 { + return; + } + if mask & 0x01 != 0 { + put_u64(raw, 0x370, 0x500); + put_u64(raw, 0x388, 0x500); + put_u64(raw, 0x8d0, 0x500); + } + if mask & 0x02 != 0 { + put_u64(raw, 0x018, 3); + } + if mask & 0x04 != 0 { + put_u32(raw, 0x33c, 0xa9); + } + if mask & 0x08 != 0 { + put_u64(raw, 0x7d8, 0x6); + put_u32(raw, 0x798, 0xa); + } +} + +pub(crate) fn build_fragment_registers( + p: &G17pRenderParameters, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], RenderBuildError> { + build_fragment_registers_for_work_key(p, render_work_key(1, 1)) +} + +/// As `build_fragment_registers`, plus the live entry count. The count is +/// always `FRAGMENT_REGISTER_COUNT`; the pair form is kept because the +/// descriptor's count word at +0x7a8 must agree with the slice actually +/// written, and having one source for both makes them impossible to desync. +pub(crate) fn build_fragment_registers_counted( + p: &G17pRenderParameters, +) -> Result<([RegisterWrite; FRAGMENT_REGISTER_COUNT], usize), RenderBuildError> { + Ok(( + build_fragment_registers_for_work_key(p, render_work_key(1, 1))?, + FRAGMENT_REGISTER_COUNT, + )) +} + +fn build_fragment_registers_for_work_key( + p: &G17pRenderParameters, + work_key: u64, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], RenderBuildError> { + let g = geometry(p)?; + let tile_state = 0x3717f + | ((g.tiles_x - 1) << 44) + | ((g.tiles_y - 1) << 53) + | 0x20_0000_0000 + | (if p.layers > 1 { 0x1_0000_0000 } else { 0 }) + | ((p.utile_config & 0xf000) << 28); + + Ok([ + RegisterWrite::new(0x01739, 1), + RegisterWrite::new(0x10009, p.utile_config), + RegisterWrite::new(0x15379, p.store_pipeline_bind), + RegisterWrite::new(0x15381, p.store_pipeline), + RegisterWrite::new(0x15369, p.load_pipeline_bind), + RegisterWrite::new(0x15371, p.load_pipeline), + RegisterWrite::new(0x15131, p.merge_upper_x_bits as u64), + RegisterWrite::new(0x15139, p.merge_upper_y_bits as u64), + RegisterWrite::new(0x100a1, 0), + RegisterWrite::new(0x15069, 0), + RegisterWrite::new(0x15071, 0), + RegisterWrite::new(0x16058, 0), + RegisterWrite::new(0x10019, p.multisample_control), + RegisterWrite::new(0x100b1, g.macro_size), + RegisterWrite::new(0x16030, g.macro_size), + RegisterWrite::new(0x100d9, g.screen), + RegisterWrite::new(0x0a301, 0), + RegisterWrite::new(0x10791, 0xff_0200), + RegisterWrite::new(0x16098, p.heapmeta), + RegisterWrite::new(0x15109, p.scissor_array), + RegisterWrite::new(0x15101, p.depth_bias_array), + RegisterWrite::new(0x15021, p.aux_fb_flags), + RegisterWrite::new(0x15211, ((p.height as u64) << 32) | p.width as u64), + RegisterWrite::new(0x15049, p.aux_fb_page_count), + RegisterWrite::new(0x10051, p.tib_blocks), + RegisterWrite::new(0x15321, p.depth_dimensions), + RegisterWrite::new(0x15301, p.depth_clear_value_bits as u64), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15311, p.occlusion_query_base), + RegisterWrite::new(0x15319, p.depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + RegisterWrite::new(0x15351, 0), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x15231, 0), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15239, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15401, p.depth_stride), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15409, p.stencil_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15411, p.depth_aux_stride), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15419, p.stencil_aux_stride), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x16429, p.tilemap), + RegisterWrite::new(0x16060, p.layermeta), + RegisterWrite::new(0x16431, (4 * g.size1) << 24), + RegisterWrite::new(0x10039, p.tile_config), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16451, 0), + RegisterWrite::new(0x15359, 0), + RegisterWrite::new(0x100b8, 0x8860), + RegisterWrite::new(0x16461, p.aux_fb), + RegisterWrite::new(0x16090, p.aux_fb), + RegisterWrite::new(0x101e9, 0x1c), + RegisterWrite::new(0x160a8, 0), + RegisterWrite::new(0x16068, tile_state), + RegisterWrite::new(0x1a0a9, 0), + RegisterWrite::new(0x1a0b1, 0), + RegisterWrite::new(0x1a079, 0), + RegisterWrite::new(0x1a081, 0), + RegisterWrite::new(0x1a0d9, 0), + RegisterWrite::new(0x1a0e1, 0), + RegisterWrite::new(0x101c1, 0), + RegisterWrite::new(0x0d469, 0), + RegisterWrite::new(0x1a0f9, 8), + // Fragment analog of TA 0x0a5a1. Native derives its high word from the + // 3D AGXUMADataRec PTC/DUPM maximum at host descriptor +0xc00, copied + // through command +0x818. Linux has no owned pool-size input yet, so + // use the exact 0x60 zero-source fallback. Bit 3 retains one captured, + // workload-derived 0xd5 value only as a diagnostic. + RegisterWrite::new( + 0x0a5a9, + if p.native_ta_registers & 8 != 0 { + 0x0000_00d5_0040_0020 + } else { + 0x0000_0060_0040_0020 + }, + ), + RegisterWrite::new(0x0d429, 0x0000_0002_0000_0001), + RegisterWrite::new(0x160e0, p.fragment_lifecycle), + RegisterWrite::new(0x01499, p.fragment_lifecycle), + RegisterWrite::new(0x0a341, p.fragment_lifecycle), + // The fragment must use the same parameter-buffer owner as TA 1c830. + // Native TA::prepare stores its allocated HWPB ID at host +928; the + // same derived host descriptor is submitted through both channels. + // generate3DRegisterList reads that HOST field at 8a5c7cc. Its wrapper + // at 8a515d8 tail-calls with x1=raw command, x2=host descriptor: the + // unused channel argument in the mangled common-function name is NOT + // present in the optimized calling convention. Raw command +928 has + // a different producer and must not be used to infer this register. + RegisterWrite::new(0x1c838, p.ta_hardware_buffer_id as u64), + RegisterWrite::new(0x1ca28, p.cycle), + RegisterWrite::new(0x10211, work_key), + RegisterWrite::new(0x10420, work_key), + // AGX3DCommandDescriptor::prepare receives this ID from the USC pool's + // HardwareBufferIDManager and generate3DRegisterList ORs it with an + // optional GART payload and feature bit. Linux currently has neither + // optional source, so publish the retained USC FList owner's exact ID. + RegisterWrite::new(0x14048, p.usc_flist_hardware_buffer_id as u64), + RegisterWrite::new(0x14080, p.fragment_status | 1), + RegisterWrite::new(0x01731, 1), + RegisterWrite::new(0x16020, 1), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16068, 0x4_0000), + ]) +} + +/// Size of one userspace global-ID group reserved by +/// `IOGPUDeviceUserClient::s_get_next_gid_group` on macOS. +pub(crate) const G17P_RENDER_GID_GROUP_STRIDE: u32 = 0x0100_0000; + +/// Compose the predecessor/current IDs used by one native render pair. +/// +/// macOS reserves a 0x01000000-ID range and gives its old global-counter value +/// to Metal. Metal returns that predecessor in render setup; the kernel then +/// allocates consecutive current IDs from the advanced global counter, 3D +/// first and TA second. The RCE programs and command tails both carry the +/// resulting `{predecessor,current}` pair. These are object IDs, not queue +/// submission ordinals. +pub(crate) const fn g17p_render_lifecycle_pair( + predecessor: u32, + fragment_current: u32, +) -> Option<(u64, u64)> { + if predecessor == 0 || fragment_current == 0 { + return None; + } + let tiling_current = match fragment_current.checked_add(1) { + Some(value) if value != 0 => value, + _ => return None, + }; + let predecessor = (predecessor as u64) << 32; + Some(( + predecessor | fragment_current as u64, + predecessor | tiling_current as u64, + )) +} + +pub(crate) fn partial_store_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_STORE_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15379, p.partial_store_pipeline_bind), + RegisterWrite::new(0x15381, p.partial_store_pipeline), + RegisterWrite::new(0x10039, p.tile_config), + RegisterWrite::new(0x15359, 0x20), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +pub(crate) fn partial_resume_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_RESUME_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15379, p.partial_store_pipeline_bind), + RegisterWrite::new(0x15381, p.partial_store_pipeline), + RegisterWrite::new(0x15369, p.partial_load_pipeline_bind), + RegisterWrite::new(0x15371, p.partial_load_pipeline), + RegisterWrite::new(0x10039, p.tile_config & 0xffff), + RegisterWrite::new(0x15359, 0x20), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +pub(crate) fn partial_load_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_LOAD_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15369, p.partial_load_pipeline_bind), + RegisterWrite::new(0x15371, p.partial_load_pipeline), + RegisterWrite::new(0x10039, p.tile_config & 0xffff), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +fn put_u8(raw: &mut [u8], offset: usize, value: u8) { + raw[offset] = value; +} + +#[cfg(test)] +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +// v91 WORD_STORES: aligned fields use one store (see g17_store). +#[cfg(not(test))] +#[inline(always)] +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + crate::g18::fw::store::put_u16(raw, offset, value) +} + +#[cfg(test)] +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +// v91 WORD_STORES: aligned fields use one store (see g17_store). +#[cfg(not(test))] +#[inline(always)] +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + crate::g18::fw::store::put_u32(raw, offset, value) +} + +#[cfg(test)] +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +// v91 WORD_STORES: aligned fields use one store (see g17_store). +#[cfg(not(test))] +#[inline(always)] +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + crate::g18::fw::store::put_u64(raw, offset, value) +} + +fn encode_registers(raw: &mut [u8], offset: usize, registers: &[RegisterWrite]) { + for (index, register) in registers.iter().enumerate() { + let entry = offset + index * REGISTER_BYTES; + put_u32(raw, entry, register.number); + put_u64(raw, entry + 4, register.value); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RenderDescriptorKind { + Tiling, + Fragment, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderDescriptorObjects { + pub(crate) record_a: u64, + pub(crate) shared: u64, + pub(crate) record_b: u64, + pub(crate) zero: u64, +} + +impl RenderDescriptorObjects { + /// Select one 0x100-byte Pool-A record while leaving every other object + /// pointer unchanged. + pub(crate) fn with_pool_a_record_index( + mut self, + record_index: usize, + ) -> Result { + let offset = record_index + .checked_mul(0x100) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + self.record_a = self + .record_a + .checked_add(offset as u64) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + Ok(self) + } + + /// Replace the stage-local shared object without changing the other three + /// descriptor pointers. Final native publishes keep TA +0x20 and Fragment + /// +0x28 identical; callers must therefore apply any HWPB substitution to + /// the shared object used by both descriptors, not to Fragment alone. + pub(crate) const fn with_shared(mut self, shared: u64) -> Self { + self.shared = shared; + self + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderDescriptorMetadata { + pub(crate) context_id: u32, + pub(crate) submission_ordinal: u64, + /// Monotonic state tag allocated by the command buffer's 36-entry state + /// ring. This is neither the selected ring index nor any QID/HWID. Native + /// retains the full word in the selected state object and places only its + /// low six bits in the stage work key. + pub(crate) command_buffer_state_word: u32, + /// TA HardwareBufferID copied to firmware descriptor +0x18. + /// + /// Every working native descriptor corpus couples this field to register + /// 0x1c830 exactly (for example 2/2, 4/4, and 5/5). The same allocator + /// result is also PBState +0x0c and the descriptor-ring index consumed by + /// `halLoadPBDesc`. + pub(crate) ta_hardware_buffer_id: u32, + pub(crate) submit_sequence: u64, +} + +impl RenderDescriptorMetadata { + pub(crate) const fn source(kind: RenderDescriptorKind) -> Self { + Self { + context_id: 1, + submission_ordinal: 0, + command_buffer_state_word: match kind { + RenderDescriptorKind::Tiling => 0, + RenderDescriptorKind::Fragment => 1, + }, + ta_hardware_buffer_id: 0, + submit_sequence: match kind { + RenderDescriptorKind::Tiling => 1, + RenderDescriptorKind::Fragment => 0, + }, + } + } +} + +/// Native's TA/3D register-list producer couples the command-buffer state tag +/// to the descriptor's allocated application-GART context. The physical QID, +/// QoS HardwareBufferID, descriptor-array slot and 36-entry state-ring index +/// are independent namespaces and must not be substituted here. +const fn render_work_key(context_id: u32, command_buffer_state_word: u32) -> u64 { + ((context_id as u64) << 8) | ((command_buffer_state_word & 0x3f) as u64) +} + +/// The completion record must name the same generation as the rendered writes. +/// Native nextCommandBufferState stores the allocated word at EventControl +8; +/// generate3DRegisterList uses its low six bits in 0x10211/0x10420. B1 uses those +/// same bits in the cache-maintenance request before publishing the stamp. +/// Keep the full word here: physical slot ordinals and ring indices are not +/// substitutes for the command-buffer allocator's generation. +pub(crate) fn set_render_event_control_generation( + event_control: &mut [u8], + command_buffer_state_word: u32, +) -> Result<(), RenderBuildError> { + let generation = event_control + .get_mut(8..12) + .ok_or(RenderBuildError::BufferTooSmall)?; + generation.copy_from_slice(&command_buffer_state_word.to_le_bytes()); + Ok(()) +} + +fn descriptor_work_ordinal(ordinal: u64) -> u32 { + (ordinal as u32).wrapping_add((ordinal / 2) as u32) +} + +fn write_common_descriptor( + kind: RenderDescriptorKind, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + registers: &[RegisterWrite], + raw: &mut [u8], +) { + let (selector, pointers, register_offset, pointer_gap) = match kind { + RenderDescriptorKind::Tiling => (0u32, 0x10usize, 0x60usize, 8usize), + RenderDescriptorKind::Fragment => (1u32, 0x20usize, 0xa0usize, 0usize), + }; + put_u32(raw, 0, selector); + put_u64(raw, 4, metadata.submit_sequence); + put_u32(raw, 0x0c, metadata.context_id); + + let addresses = [ + objects.record_a, + objects.shared, + objects.record_b, + objects.zero, + ]; + let mut offset = pointers; + for (index, address) in addresses.iter().enumerate() { + put_u64(raw, offset, *address); + offset += 8; + if index == 0 { + offset += pointer_gap; + } + } + encode_registers(raw, register_offset, registers); + + let work = descriptor_work_ordinal(metadata.submission_ordinal); + match kind { + RenderDescriptorKind::Tiling => { + put_u32(raw, 0x18, metadata.ta_hardware_buffer_id); + put_u32(raw, 0x48, work); + } + RenderDescriptorKind::Fragment => {} + } +} + +pub(crate) fn build_ta_descriptor( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < TA_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + let registers = build_ta_registers_for_work_key( + parameters, + render_work_key(metadata.context_id, metadata.command_buffer_state_word), + )?; + build_ta_descriptor_base(parameters, objects, metadata, ®isters, raw)?; + apply_g17p_ta_oracle(raw); + Ok(()) +} + +pub(crate) fn build_ta_descriptor_base( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + registers: &[RegisterWrite], + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < TA_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + if registers.is_empty() || registers.len() > (0x660 - 0x60) / REGISTER_BYTES { + return Err(RenderBuildError::BufferTooSmall); + } + raw[..TA_DESCRIPTOR_SIZE].fill(0); + write_common_descriptor( + RenderDescriptorKind::Tiling, + objects, + metadata, + registers, + raw, + ); + + // Like the fragment boundaries, these are generated for every native TA + // command (25G83 88bdfc4/88be090/88be1a4), not only a queue's first kick. + // The final three entries are 01740, 1c880, 1c898; firmware reads +38 and + // +3a/+3c to select the corresponding embedded RCE boundaries. + put_u16(raw, 0x38, (TA_REGISTER_COUNT - 2) as u16); + put_u16(raw, 0x3a, TA_REGISTER_COUNT as u16); + put_u16(raw, 0x3c, TA_REGISTER_COUNT as u16); + + let deflake = ta_user_offset(parameters.deflake_1)?; + let sampler_max = if parameters.sampler_count == 0 { + 0 + } else { + parameters.sampler_count + 1 + }; + put_u32( + raw, + 0x0768, + (((registers.len() * REGISTER_BYTES) as u32) << 16) | registers.len() as u32, + ); + put_u64(raw, 0x0780, parameters.tpc); + // Native copies RTMemory's geometry-derived TPC size beside its pointer + // on every submission (25G83 88fd970..974), independent of queue lifetime. + put_u64(raw, 0x0788, parameters.tpc_size); + put_u32(raw, 0x07d6, deflake as u32); + put_u32(raw, 0x0876, u32::MAX); + put_u64(raw, 0x087a, parameters.sampler_array); + put_u32(raw, 0x0882, parameters.sampler_count); + put_u32(raw, 0x0886, sampler_max); + // Present in 436/438 valid G17P TA records, including both exact 32x32 + // geometry matches. This belongs to the TA output/activation tail; leaving + // it zero is a structural mismatch before the linked 3D job is generated. + put_u32(raw, 0x0892, 1); + put_u8(raw, 0x0932, 0x44); + put_u8(raw, 0x093c, 1); + put_u8(raw, 0x094d, parameters.context_generation); + put_u64(raw, 0x08fe, parameters.ta_timestamp_start); + put_u64(raw, 0x0906, parameters.ta_timestamp_end); + // +0x090e/+0x0916 are caller-visible timestamp destinations. Firmware + // writes them only inside its timestamp aperture, so they stay zero; the + // completion publishes the pass envelope to userspace instead. + // AGXTAChannelSKSM::submitBuffer mirrors the same dependency/current pair + // consumed by the TA RCE program into the firmware command. These fields + // are deliberately unaligned in the native 0x9c0-byte command layout. + put_u32(raw, 0x086e, (parameters.lifecycle >> 32) as u32); + put_u64(raw, 0x08ce, parameters.lifecycle as u32 as u64); + Ok(()) +} + +/// Build the TA descriptor used by the proven native cold opening. +/// +/// The first-work path applies its pool/lifecycle fields after the common +/// builder has generated the RCE boundaries required by every command. +pub(crate) fn build_cold_opening_ta_descriptor( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + context_id: u32, + command_buffer_state_word: u32, + ta_hardware_buffer_id: u32, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + let mut metadata = RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling); + metadata.context_id = context_id; + metadata.command_buffer_state_word = command_buffer_state_word; + metadata.ta_hardware_buffer_id = ta_hardware_buffer_id; + build_ta_descriptor(parameters, objects, metadata, raw)?; + put_u8(raw, 0x93e, 0xd0); + put_u8( + raw, + 0x93f, + if parameters.native_pm_bytes { + 0x87 + } else { + 0x91 + }, + ); + Ok(()) +} + +fn put_embedded_registers( + raw: &mut [u8], + header_offset: usize, + program_offset: usize, + registers: &[RegisterWrite], +) { + let bytes = registers.len() * REGISTER_BYTES; + put_u32( + raw, + header_offset, + ((bytes as u32) << 16) | registers.len() as u32, + ); + encode_registers(raw, program_offset, registers); +} + +pub(crate) struct G17pFragmentRegisters<'a> { + pub(crate) main: &'a [RegisterWrite], + pub(crate) store: &'a [RegisterWrite], + pub(crate) resume: &'a [RegisterWrite], + pub(crate) load: &'a [RegisterWrite], +} + +pub(crate) fn build_fragment_descriptor( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < FRAGMENT_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + let registers = build_fragment_registers_for_work_key( + parameters, + render_work_key(metadata.context_id, metadata.command_buffer_state_word), + )?; + let store = partial_store_registers(parameters); + let resume = partial_resume_registers(parameters); + let load = partial_load_registers(parameters); + build_fragment_descriptor_base( + parameters, + objects, + metadata, + G17pFragmentRegisters { + main: ®isters, + store: &store, + resume: &resume, + load: &load, + }, + raw, + )?; + apply_g17p_fragment_oracle(raw); + Ok(()) +} + +pub(crate) fn build_fragment_descriptor_base( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + programs: G17pFragmentRegisters<'_>, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < FRAGMENT_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + if programs.main.is_empty() + || programs.main.len() > (0x6a0 - 0xa0) / REGISTER_BYTES + || programs.store.len() > (0xec0 - 0x7c0) / REGISTER_BYTES + || programs.resume.len() > (0x15e0 - 0xee0) / REGISTER_BYTES + || programs.load.len() > (0x1d00 - 0x1600) / REGISTER_BYTES + { + return Err(RenderBuildError::BufferTooSmall); + } + raw[..FRAGMENT_DESCRIPTOR_SIZE].fill(0); + let geometry = geometry(parameters)?; + let register_count = programs.main.len(); + write_common_descriptor( + RenderDescriptorKind::Fragment, + objects, + metadata, + programs.main, + raw, + ); + + put_u64(raw, 0x40, parameters.tilemap); + put_u64(raw, 0x48, parameters.multisample_control); + // Native copies the render target's sample count through host +0x8c8 + // into every fragment descriptor at +0x50 (25G83: 0x8839d80, + // 0x8837af8..0x8837b08). It is not first-work metadata. The UAPI has + // already encoded log2(samples) in UTILE_CONFIG bits [1:0]. + put_u32(raw, 0x50, 1u32 << (parameters.utile_config & 3)); + put_u32(raw, 0x54, geometry.macro_size as u32); + put_u32(raw, 0x68, parameters.merge_upper_x_bits); + put_u32(raw, 0x6c, parameters.merge_upper_y_bits); + put_u64(raw, 0x78, geometry.tiles_x * geometry.tiles_y); + + // RCE boundaries belong to every command, including retained queue kicks. + // Native emits these from the running register count (25G83 88c0740, + // 88c07ac, 88c08c8, 88c0970); firmware consumes +80/+82/+84/+88 when + // selecting an entry in this command's embedded program. The fixed recipe + // ends with 01731=1, 16020=1, 16020=0, 16068=40000. Boundaries are after + // 01731, after the first 16020, that same entry, and the complete program. + // Keeping these only in the cold wrapper left ordinary restages at zero. + put_u16(raw, 0x80, (register_count - 3) as u16); + put_u16(raw, 0x82, (register_count - 2) as u16); + put_u16(raw, 0x84, (register_count - 2) as u16); + put_u16(raw, 0x88, register_count as u16); + + put_u32( + raw, + 0x07a8, + (((register_count * REGISTER_BYTES) as u32) << 16) | register_count as u32, + ); + put_embedded_registers(raw, 0x0ec8, 0x07c0, programs.store); + put_embedded_registers(raw, 0x15e8, 0x0ee0, programs.resume); + put_embedded_registers(raw, 0x1d08, 0x1600, programs.load); + + put_u64(raw, 0x1d20, parameters.depth_bias_array); + put_u64(raw, 0x1d30, parameters.scissor_array); + put_u64(raw, 0x1d40, parameters.occlusion_query_base); + put_u64(raw, 0x1e78, parameters.load_pipeline_bind); + put_u64(raw, 0x1e80, parameters.load_pipeline); + put_u64(raw, 0x1ea8, parameters.partial_load_pipeline_bind); + put_u64(raw, 0x1eb0, parameters.partial_load_pipeline); + put_u32(raw, 0x1ec0, 0x0404_0404); + put_u64(raw, 0x1f38, parameters.tib_blocks); + put_u64(raw, 0x1f40, parameters.aux_fb_flags); + put_u32(raw, 0x1f48, parameters.width); + put_u32(raw, 0x1f4c, parameters.height); + put_u64(raw, 0x1f50, parameters.aux_fb_page_count); + put_u64(raw, 0x1f58, parameters.tile_config); + put_u32(raw, 0x1f78, parameters.store_pipeline_bind as u32); + put_u64(raw, 0x1f7c, parameters.store_pipeline); + put_u32(raw, 0x1f98, parameters.partial_store_pipeline_bind as u32); + put_u64(raw, 0x1f9c, parameters.partial_store_pipeline); + put_u32(raw, 0x1fa8, parameters.depth_clear_value_bits); + put_u64(raw, 0x1fac, (parameters.tib_blocks << 33) | 0x300); + // Native copies isMemorylessRender through the UMD render record +0x240, + // parsed record +0x1b8, and host descriptor +0x8bf into this byte. It is + // not a TIB-size predicate: native's memoryless policy also selects the + // shortened SPM phase programs. The admitted render UAPI represents + // memory-backed resources and has no memoryless-render policy or matching + // SPM-program construction. Keep that policy clear even for >8 TIB + // blocks; supporting memoryless renders requires modeling both together. + // Empty-tile processing remains independent in register 0x10039 bit 16. + put_u8(raw, 0x2100, 0); + // Exact G17P tail copied by AGX3DChannelSKSM::submitBuffer. +0x2104 is the + // accelerator control snapshot and +0x210c its low map nibble. +0x2110 is + // a separate render-setup field: prepare copies descriptor +0xc00 to setup + // +0x818, processRenderSetup carries it to +0x89c, and submitBuffer emits + // it here. Working Metal commands carry the invariant all-ones value. + put_u32(raw, 0x2104, parameters.gpc_perf_state_control); + // AGX3DChannelSKSM::submitBuffer mirrors the same dependency/current pair + // consumed by the fragment RCE program into the firmware command. + put_u32(raw, 0x2108, (parameters.fragment_lifecycle >> 32) as u32); + put_u32(raw, 0x210c, parameters.gpc_perf_state_map_low); + put_u32(raw, 0x2110, G17P_FRAGMENT_TAIL_SETUP_WORD); + // Native submitBuffer zeros +0x2114..+0x2123. Sampler state reaches the + // GPU through its real register/program path, not these command-tail + // lanes. The first two following flag lanes reproduce the working Neo + // shared-runtime snapshot; the final two come from per-command descriptor + // bytes and retain their constructor-zero values until those inputs are + // modeled. + put_u32( + raw, + 0x2124, + G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT[0], + ); + put_u32( + raw, + 0x2128, + G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT[1], + ); + put_u64(raw, 0x2168, parameters.fragment_lifecycle as u32 as u64); + put_u64(raw, 0x2198, parameters.fragment_timestamp_start); + put_u64(raw, 0x21a0, parameters.fragment_timestamp_end); + // +0x21a8/+0x21b0: caller-visible destinations, zero as in the TA record. + put_u8(raw, 0x21cc, 0x53); + put_u8(raw, 0x21d6, 1); + put_u8(raw, 0x21e7, parameters.context_generation); + put_u8(raw, 0x2208, 1); + put_u8(raw, 0x220c, 1); + put_u8(raw, 0x222d, g17p_frg_hwdsid_mode()); + Ok(()) +} + +fn apply_g17p_fragment_oracle(raw: &mut [u8]) { + // Neo diagnostic overrides stay outside the shared encoder so M5 + // register programs and geometry never inherit these substitutions. + let oracle = crate::g18::fw::config::FRAG_ORACLE; + if oracle & 0x01 != 0 { + put_u64(raw, 0x158, 0x7007); + } + if oracle & 0x02 != 0 { + put_u64(raw, 0x488, 1); + } + if oracle & 0x04 != 0 { + // native 0x8000005300000000 against our 0x0000005300000000 + let mut cur = [0u8; 8]; + cur.copy_from_slice(&raw[0x21c8..0x21d0]); + put_u64(raw, 0x21c8, u64::from_le_bytes(cur) | (1u64 << 63)); + } + if oracle & 0x08 != 0 { + put_u64(raw, 0x78, 0x40); + } + if oracle & 0x10 != 0 { + put_u64(raw, 0x458, 3); + put_u64(raw, 0x470, 0x500); + } + if oracle & 0x20 != 0 { + put_u64(raw, 0x1f48, 0x0000_0100_0000_0100); + } + if oracle & 0x40 != 0 { + // fields native leaves ZERO and we populate + put_u64(raw, 0x1d0, 0); + put_u64(raw, 0x1f0, 0); + put_u64(raw, 0x1f8, 0); + } + if oracle & 0x80 != 0 { + put_u32(raw, 0x1d8 + 4, 0x3f80_0000); + } +} + +/// FRG +0x222d mode byte: Linux has always written 1; all 160 native light +/// FRG captures hold 0. Diagnostic parameter `g17p_frg_hwdsid_clear`. +#[cfg(not(test))] +fn g17p_frg_hwdsid_mode() -> u8 { + u8::from(crate::g18::params::g17p_frg_hwdsid_clear.value() == 0 + && !crate::g18::compat::timing_buffer::diag_knob(crate::g18::compat::timing_buffer::DIAG_FRG_222D_CLEAR)) +} + +#[cfg(test)] +fn g17p_frg_hwdsid_mode() -> u8 { 1 } + +/// Build the fragment descriptor used by the proven native cold opening. +pub(crate) fn build_cold_opening_fragment_descriptor( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + context_id: u32, + command_buffer_state_word: u32, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + let mut metadata = RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment); + metadata.context_id = context_id; + metadata.command_buffer_state_word = command_buffer_state_word; + build_fragment_descriptor(parameters, objects, metadata, raw)?; + // +90 is staged with the channel's completion payload at publication. + // A newly published native 3D queue carries 1 here; retained submissions + // clear it. This lifecycle is independent of the outer-slot first-submit + // byte and appears identically on cold QIDs 1 and 8 in both valid traces. + put_u32(raw, 0x215c, 1); + Ok(()) +} + +/// Stage the TA's link to the physical fragment command, not its ring slot. +/// +/// Native TA submit copies host +0x1528 as a low-40-bit qword at +0x7a8 and +/// its bits 47:40 as two independent bytes at +0x7b0/+0x7b1 (8a99220..34). +/// Host +0xe60 supplies the full packed qword at +0x8c6 (8a99414..1c). +/// Its producer converts the TA channel timestamp, whereas +0x1528 converts +/// the fragment channel timestamp (8aa9e30..aaa05c). Both carry the 3D channel +/// payload. Preserve those distinct sources even when queue seeds coincide. +/// Refresh from the actual retained producer immediately before publication; +/// the payload counter wraps independently of the physical timestamp. +pub(crate) fn apply_g17p_ta_linked_completion_scratch( + tiling_timestamp: u64, + fragment_timestamp: u64, + payload: u8, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if tiling_timestamp > 0xff_ffff_ffff || fragment_timestamp > 0xff_ffff_ffff { + return Err(RenderBuildError::ArithmeticOverflow); + } + if raw.len() < TA_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + put_u64(raw, 0x07a8, fragment_timestamp); + put_u8(raw, 0x07b0, payload); + put_u8(raw, 0x07b1, payload); + put_u64(raw, 0x08c6, tiling_timestamp | ((payload as u64) << 40)); + Ok(()) +} + +/// Apply the queue-local fields that change as the canonical three-item group +/// is refreshed in place. `descriptor_gpu_va` is the low VM address firmware +/// uses for the descriptor's self pointers. +pub(crate) fn apply_g17p_retained_descriptor_fields( + kind: RenderDescriptorKind, + descriptor_gpu_va: u64, + runtime_control: u64, + queue_id: u8, + linked_fragment_queue_id: u8, + submission_ordinal: u64, + cold_opening: bool, + native_pm_bytes: bool, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + let item_number = submission_ordinal + .checked_add(1) + .ok_or(RenderBuildError::ArithmeticOverflow)? & 0xffffffffff; + // The native queue timestamp is 40 bits. Stamp-info carries its low + // 24 bits shifted by eight, with ordinary u32 wrap as on M1/M2. + let event_value = (item_number as u32).wrapping_shl(8); + match kind { + RenderDescriptorKind::Tiling => { + if raw.len() < TA_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + put_u64( + raw, + 0x0760, + descriptor_gpu_va + .checked_add(0x60) + .ok_or(RenderBuildError::ArithmeticOverflow)?, + ); + // Native TA links directly to the physical 3D kick queue which + // will consume this submission. This is queue allocator state, + // not the type-1 fragment descriptor's distinct QID-0 RCE field. + put_u32(raw, 0x079c, u32::from(linked_fragment_queue_id)); + put_u32(raw, 0x07a0, event_value); + apply_g17p_ta_linked_completion_scratch( + u64::from(item_number), u64::from(item_number), item_number as u8, raw, + )?; + // Native writes a u32 at +0x8b6 (8a993f4). +0x8b4/+0x8b5 + // belong to the preceding stamp pointer, not this event value. + put_u32(raw, 0x08b6, event_value); + // B1 indexes the registered scheduler-resource table with this + // value while admitting the completed TA, then passes the full + // word to FUN_15424(linked_3d_qid, ta_qid). It is the TA queue's + // own physical QID, not a captured literal: two independent Metal + // commands carry 7 here while +0x79c links QID 8, and 9 here while + // +0x79c links QID 10. Leaving it zero made Linux consult the + // unrelated QID-0 scheduler state at the TA-to-3D handoff. + put_u32( + raw, + 0x08ba, + if crate::g18::fw::config::RENDER_TA_QID_FIELD != 0 { + u32::from(queue_id) + } else { + 0 + }, + ); + // Retain the existing Linux TA drain policy while its visibility + // boundaries are audited. Native +0x8c2 is a pending IO-fence + // boolean (host descriptor +0x48), not a priority/QID encoding. + // Firmware ORs it into TSQ node +0x20 and clears it on a drain; + // this TA flag does not guarantee a later fragment drain. + // Keep this independent of the opt-in priority diagnostic. + put_u32(raw, 0x08c2, 1); + // Native's host +0x1288 source is a u32 at +0x8d6 (8a990b8), + // immediately after the qword lifecycle field beginning +0x8ce. + put_u32(raw, 0x08d6, submission_ordinal as u32); + put_u64(raw, 0x0934, runtime_control); + if cold_opening { + put_u8(raw, 0x093e, 0xd0); + put_u8(raw, 0x093f, if native_pm_bytes { 0x87 } else { 0x91 }); + } + } + RenderDescriptorKind::Fragment => { + if raw.len() < FRAGMENT_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + for (at, target) in [ + (0x07a0, 0x00a0), + (0x0ec0, 0x07c0), + (0x15e0, 0x0ee0), + (0x1d00, 0x1600), + ] { + put_u64( + raw, + at, + descriptor_gpu_va + .checked_add(target) + .ok_or(RenderBuildError::ArithmeticOverflow)?, + ); + } + put_u32(raw, 0x2150, event_value); + // Valid native queues prove +0x2154 is the physical 3D QID: it is + // 1/3/8 for qids 1/3/8, respectively. The old literal only happened + // to match the first captured qid. + put_u32(raw, 0x2154, u32::from(queue_id)); + // Preserve the existing first-submission policy pending a + // visibility-boundary audit. The captured cold/later values do + // not establish ordinal semantics: native derives this flag + // from pending IO-fences at host descriptor +0x48 (8837268..74, + // 8837eb0). The TSQ ownership notes decode the producer. + put_u32(raw, 0x215c, u32::from(submission_ordinal == 0)); + // +0x2160 is the packed 3D-channel SKSM completion scratch value: + // the physical queue timestamp in bits [39:0] and the channel's + // payload byte in bits [47:40]. G17PUserRenderStorage stages it + // from the retained fragment producer immediately before publish. + put_u32(raw, 0x2170, submission_ordinal as u32); + put_u64(raw, 0x21ce, runtime_control); + // Native HWDSID mask occupies +0x220d..+0x222c inclusive; + // +0x222d is a separate mode flag. An unaligned u64 at +0x222c + // reads 0x100 from that following flag, not from HWDSID bit 8. + // Preserve the template's zero mask; aperture IDs alone do not + // supply the explicit MDSID-list and accounting contract. + } + } + Ok(()) +} + +/// Copy the four embedded 3D register-entry arrays into a diagnostic mirror. +/// +/// The hardware-visible arrays remain descriptor-local. Native's addresses at +/// descriptor +0x7a0/+0xec0/+0x15e0/+0x1d00 are the self pointers in each +/// embedded array's +0x700 trailer, not pointers to a second allocation. This +/// mirror intentionally contains entries only and must never be bound to KSM. +pub(crate) fn stage_g17p_fragment_rce_programs( + fragment: &mut [u8], + rce_gpu_va: u64, + rce: &mut [u8], +) -> Result<(), RenderBuildError> { + if fragment.len() < FRAGMENT_DESCRIPTOR_SIZE + || rce.len() < G17P_FRAGMENT_RCE_STORAGE_SIZE + { + return Err(RenderBuildError::BufferTooSmall); + } + + let programs = [ + (0x07a0usize, 0x00a0usize, FRAGMENT_REGISTER_COUNT), + (0x0ec0usize, 0x07c0usize, PARTIAL_STORE_REGISTER_COUNT), + (0x15e0usize, 0x0ee0usize, PARTIAL_RESUME_REGISTER_COUNT), + (0x1d00usize, 0x1600usize, PARTIAL_LOAD_REGISTER_COUNT), + ]; + + rce[..G17P_FRAGMENT_RCE_STORAGE_SIZE].fill(0); + for (index, (_pointer_offset, source_offset, register_count)) in + programs.into_iter().enumerate() + { + let bytes = register_count + .checked_mul(REGISTER_BYTES) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let target_offset = index + .checked_mul(G17P_FRAGMENT_RCE_PROGRAM_STRIDE as usize) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let source_end = source_offset + .checked_add(bytes) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let target_end = target_offset + .checked_add(bytes) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + rce[target_offset..target_end] + .copy_from_slice(&fragment[source_offset..source_end]); + } + let _ = rce_gpu_va; + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct AddressedBytes { + pub(crate) address: u64, + pub(crate) bytes: [u8; N], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueuePublicationInput { + pub(crate) queue: u64, + pub(crate) pointers: u64, + pub(crate) item_ring: u64, + pub(crate) channel_ring: u64, + pub(crate) channel_producer: u64, + pub(crate) items: [u64; 3], + pub(crate) grid_index: u16, + pub(crate) slot_index: u16, + pub(crate) current_write: u32, + pub(crate) group_number: u32, + pub(crate) first_submit: bool, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueuePublication { + pub(crate) item_entries: [AddressedBytes<8>; 3], + pub(crate) event: AddressedBytes, + pub(crate) queue_write: AddressedBytes<4>, + pub(crate) ring_slot: AddressedBytes, + pub(crate) producer: AddressedBytes<4>, + pub(crate) write_after: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PairedQueuePublication { + pub(crate) tiling: QueuePublication, + pub(crate) fragment: QueuePublication, +} + +pub(crate) fn build_queue_publication( + kind: RenderDescriptorKind, + input: QueuePublicationInput, +) -> Result { + let write_after = input + .current_write + .checked_add(3) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + if input.grid_index > 0xff { + return Err(RenderBuildError::QueueGridOutOfRange); + } + if input.slot_index > 0xff { + return Err(RenderBuildError::RingSlotOutOfRange); + } + if input.group_number > (u32::MAX >> 8) { + return Err(RenderBuildError::ArithmeticOverflow); + } + + let item_offset = (input.current_write as u64) + .checked_mul(8) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let first_item = input + .item_ring + .checked_add(item_offset) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let item_entries = [ + AddressedBytes { + address: first_item, + bytes: input.items[0].to_le_bytes(), + }, + AddressedBytes { + address: first_item + .checked_add(8) + .ok_or(RenderBuildError::ArithmeticOverflow)?, + bytes: input.items[1].to_le_bytes(), + }, + AddressedBytes { + address: first_item + .checked_add(16) + .ok_or(RenderBuildError::ArithmeticOverflow)?, + bytes: input.items[2].to_le_bytes(), + }, + ]; + + let mut event = [0u8; EVENT_RECORD_SIZE]; + let counter = input.group_number << 8; + put_u32(&mut event, 0x00, EVENT_SELECTOR); + put_u32( + &mut event, + 0x04, + EVENT_SUBTYPE_BASE | input.grid_index as u32, + ); + put_u32(&mut event, 0x08, counter); + put_u32( + &mut event, + 0x10, + match kind { + RenderDescriptorKind::Tiling => 0, + RenderDescriptorKind::Fragment => 0x100, + }, + ); + + let mut ring_slot = [0u8; RING_SLOT_SIZE]; + put_u64(&mut ring_slot, 0x08, input.queue); + put_u32( + &mut ring_slot, + 0x10, + match kind { + RenderDescriptorKind::Tiling => 0, + RenderDescriptorKind::Fragment => 1, + }, + ); + let flags = (write_after & 0xffff) + | ((input.grid_index as u32) << 16) + | if input.first_submit { 1 << 24 } else { 0 }; + put_u32(&mut ring_slot, 0x14, flags); + + let queue_write_address = input + .pointers + .checked_add(QUEUE_WRITE_OFFSET) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let slot_offset = (input.slot_index as u64) + .checked_mul(RING_SLOT_SIZE as u64) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let ring_slot_address = input + .channel_ring + .checked_add(slot_offset) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let producer = (input.slot_index as u32 + 1) & 0xff; + Ok(QueuePublication { + item_entries, + event: AddressedBytes { + address: input.items[2], + bytes: event, + }, + queue_write: AddressedBytes { + address: queue_write_address, + bytes: write_after.to_le_bytes(), + }, + ring_slot: AddressedBytes { + address: ring_slot_address, + bytes: ring_slot, + }, + producer: AddressedBytes { + address: input.channel_producer, + bytes: producer.to_le_bytes(), + }, + write_after, + }) +} + +pub(crate) fn build_paired_queue_publication( + tiling: QueuePublicationInput, + fragment: QueuePublicationInput, +) -> Result { + Ok(PairedQueuePublication { + tiling: build_queue_publication(RenderDescriptorKind::Tiling, tiling)?, + fragment: build_queue_publication(RenderDescriptorKind::Fragment, fragment)?, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mcache_mapping_matches_native_g17p_3d_records() { + assert_eq!( + encode_g17p_mcache_mapping(G17PMcacheMapping { + address: 0xeeee_0000, + size_units: 0x4d000, + hwsid: 0x18, + selector: 3, + range_shift: 2, + }) + .unwrap(), + [ + 0x00, 0xdc, 0xdd, 0x01, 0x00, 0x06, 0x8c, 0x00, 0x00, 0xac, 0xe2, 0x01, + 0x00, 0x12, 0x00, 0x00, + ] + ); + assert_eq!( + encode_g17p_mcache_mapping(G17PMcacheMapping { + address: 0x100_0e83_8000, + size_units: 0x26a80, + hwsid: 0x18, + selector: 3, + range_shift: 1, + }) + .unwrap(), + [ + 0x00, 0x07, 0x1d, 0x00, 0x02, 0x06, 0x8c, 0x00, 0x80, 0x71, 0x1f, 0x00, + 0x02, 0x0a, 0x00, 0x00, + ] + ); + } + + fn fnv1a64(parts: &[&[u8]]) -> u64 { + let mut value = 0xcbf2_9ce4_8422_2325u64; + for part in parts { + for byte in *part { + value ^= *byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + } + value + } + + fn encoded_register_hash(registers: &[RegisterWrite]) -> u64 { + let mut raw = [0u8; FRAGMENT_REGISTER_COUNT * REGISTER_BYTES]; + encode_registers(&mut raw, 0, registers); + fnv1a64(&[&raw[..registers.len() * REGISTER_BYTES]]) + } + + fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) + } + + fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) + } + + fn objects() -> RenderDescriptorObjects { + RenderDescriptorObjects { + record_a: 0xffff_fc20_0150_0000, + shared: 0xffff_fc20_0160_0000, + record_b: 0xffff_fc20_0158_0000, + zero: 0xffff_fc20_0168_0000, + } + } + + #[test] + fn source_profile_pins_rsrc8_and_native_aligned_programs() { + let parameters = G17pRenderParameters::source_rsrc8(); + assert_eq!(parameters.store_pipeline_bind, 0x8); + assert_eq!(parameters.load_pipeline_bind, 0x0007_8000_0000_0040); + assert_eq!(parameters.store_pipeline, 0x0000_0100_0199_0640); + assert_eq!(parameters.load_pipeline, 0x0000_0100_0199_0240); + } + + #[test] + fn render_lifecycles_use_native_gid_predecessor_and_consecutive_currents() { + // AtomicU32(1): reserve-group returns predecessor 1 and advances the + // counter to 0x01000001. Native OSAddAtomic64 returns the old value, + // hence 3D current 0x01000001 and TA current 0x01000002. + assert_eq!( + g17p_render_lifecycle_pair(1, 0x0100_0001), + Some((0x0000_0001_0100_0001, 0x0000_0001_0100_0002)) + ); + // Exact first render pair from the working Metal capture. + assert_eq!( + g17p_render_lifecycle_pair(0x0700_c271, 0x0800_c294), + Some((0x0700_c271_0800_c294, 0x0700_c271_0800_c295)) + ); + assert_eq!(g17p_render_lifecycle_pair(0, 1), None); + assert_eq!(g17p_render_lifecycle_pair(1, u32::MAX), None); + + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.lifecycle = 0x1111_2222_3333_4444; + parameters.fragment_lifecycle = 0x5555_6666_7777_8888; + let ta = build_ta_registers(¶meters).unwrap(); + let fragment = build_fragment_registers(¶meters).unwrap(); + assert!(ta.iter().filter(|write| write.value == parameters.lifecycle).count() >= 3); + assert_eq!( + fragment + .iter() + .filter(|write| write.value == parameters.fragment_lifecycle) + .count(), + 3 + ); + let mut ta_descriptor = [0u8; TA_DESCRIPTOR_SIZE]; + build_ta_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut ta_descriptor, + ) + .unwrap(); + assert_eq!(get_u32(&ta_descriptor, 0x086e), 0x1111_2222); + assert_eq!(get_u64(&ta_descriptor, 0x08ce), 0x3333_4444); + + let mut descriptor = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_fragment_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut descriptor, + ) + .unwrap(); + assert_eq!(get_u32(&descriptor, 0x2108), 0x5555_6666); + assert_eq!(get_u64(&descriptor, 0x2168), 0x7777_8888); + } + + #[test] + fn vdm_user_offset_matches_g17p_compact_window() { + assert_eq!(vdm_user_offset(0x10_0186_0420).unwrap(), 0x0186_0420); + assert_eq!( + vdm_user_offset(0x4f_fff5_4400).unwrap(), + 0x3f_fff5_4400 + ); + assert_eq!( + vdm_user_offset(0xbf_fff5_4400), + Err(RenderBuildError::VdmAddressOutOfRange) + ); + } + + #[test] + fn native_ta_addresses_use_split_resource_and_user_domains() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.context_base = 0x70_0000_0000; + parameters.tilemap = 0x70_011e_0a00; + parameters.layermeta = 0x70_011e_2a00; + parameters.heapmeta = 0x70_011e_2a00; + parameters.tpc = 0x70_0129_0000; + parameters.deflake_3 = 0x10_0009_ca60; + parameters.deflake_2 = 0x10_0009_ca80; + parameters.deflake_1 = 0x10_0009_cd00; + + let registers = build_ta_registers(¶meters).unwrap(); + let value = |number| { + registers + .iter() + .find(|write| write.number == number) + .unwrap() + .value + }; + assert_eq!(value(0x1c039), 0x011e_0a00); + assert_eq!(value(0x1c031), 0x8000_0000_011e_2a00); + assert_eq!(value(0x1c0a1), 0x0129_0000); + assert_eq!(value(0x10111), 0x0009_cd00); + assert_eq!(value(0x10119), 0x0009_ca80); + assert_eq!(value(0x1c950), 0x0004_0000_0009_ca60); + + let mut descriptor = [0u8; TA_DESCRIPTOR_SIZE]; + build_ta_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata { + context_id: 1, + submission_ordinal: 0, + command_buffer_state_word: 0, + ta_hardware_buffer_id: 0, + submit_sequence: 1, + }, + &mut descriptor, + ) + .unwrap(); + assert_eq!(get_u32(&descriptor, 0x07d6), 0x0009_cd00); + assert_eq!(get_u64(&descriptor, 0x0780), parameters.tpc); + + parameters.deflake_1 = 0x11_0000_0000; + assert_eq!( + build_ta_registers(¶meters), + Err(RenderBuildError::UserAddressOutOfRange) + ); + } + + #[test] + fn register_programs_match_m1n1_9165484() { + let parameters = G17pRenderParameters::source_rsrc8(); + let tiling = build_ta_registers(¶meters).unwrap(); + let fragment = build_fragment_registers(¶meters).unwrap(); + assert_eq!(tiling.len(), 73); + assert_eq!(fragment.len(), 89); + assert_eq!(encoded_register_hash(&tiling), 0x29dc_2725_a3c2_90c9); + assert_eq!(encoded_register_hash(&fragment), 0xdb21_2bfe_1bbf_85c8); + assert_eq!(fragment[2], RegisterWrite::new(0x15379, 0x8)); + assert_eq!( + fragment[3], + RegisterWrite::new(0x15381, G17P_SOURCE_EOT_PIPELINE) + ); + assert_eq!( + fragment[4], + RegisterWrite::new(0x15369, G17P_LOAD_RESOURCE_SPEC) + ); + assert_eq!( + fragment[5], + RegisterWrite::new(0x15371, G17P_SOURCE_BG_PIPELINE) + ); + } + + #[test] + fn reused_cpu_staging_produces_identical_wire_descriptors() { + let parameters = G17pRenderParameters::source_rsrc8(); + for dirty in [0x00, 0x55, 0xff] { + let mut staging = [dirty; 0x8000]; + staging.fill(0); + let mut tiling = [0; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0; FRAGMENT_DESCRIPTOR_SIZE]; + let ta_metadata = RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling); + let fragment_metadata = RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment); + build_ta_descriptor(¶meters, objects(), ta_metadata, &mut tiling).unwrap(); + build_fragment_descriptor(¶meters, objects(), fragment_metadata, &mut fragment).unwrap(); + let (ta, fr) = staging.split_at_mut(0x4000); + build_ta_descriptor(¶meters, objects(), ta_metadata, ta).unwrap(); + build_fragment_descriptor(¶meters, objects(), fragment_metadata, fr).unwrap(); + assert_eq!(&ta[..TA_DESCRIPTOR_SIZE], &tiling); + assert_eq!(&fr[..FRAGMENT_DESCRIPTOR_SIZE], &fragment); + assert!(ta[TA_DESCRIPTOR_SIZE..].iter().all(|v| *v == 0)); + assert!(fr[FRAGMENT_DESCRIPTOR_SIZE..].iter().all(|v| *v == 0)); + } + } + + #[test] + fn descriptors_match_m1n1_9165484_and_duplicate_pipeline_fields() { + let parameters = G17pRenderParameters::source_rsrc8(); + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_ta_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut tiling, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut fragment, + ) + .unwrap(); + + assert_eq!(fnv1a64(&[&tiling]), 0x8902_c437_9d74_1cc7); + assert_eq!(fnv1a64(&[&fragment]), 0xdd6e_bcc7_77e7_834d); + assert_eq!(get_u32(&fragment, 0xb8), 0x15379); + assert_eq!(get_u64(&fragment, 0xbc), 0x8); + assert_eq!(get_u64(&fragment, 0xc8), G17P_SOURCE_EOT_PIPELINE); + assert_eq!(get_u64(&fragment, 0x1e78), G17P_LOAD_RESOURCE_SPEC); + assert_eq!(get_u64(&fragment, 0x1e80), G17P_SOURCE_BG_PIPELINE); + assert_eq!(get_u32(&fragment, 0x1f78), 0x8); + assert_eq!(get_u64(&fragment, 0x1f7c), G17P_SOURCE_EOT_PIPELINE); + } + + #[test] + fn pool_a_record_one_changes_only_the_two_stage_pointer_fields() { + let parameters = G17pRenderParameters::source_rsrc8(); + let baseline_objects = objects(); + let record_one_objects = baseline_objects.with_pool_a_record_index(1).unwrap(); + let mut baseline_ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut record_one_ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut baseline_3d = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + let mut record_one_3d = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_ta_descriptor( + ¶meters, + baseline_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut baseline_ta, + ) + .unwrap(); + build_ta_descriptor( + ¶meters, + record_one_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut record_one_ta, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + baseline_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut baseline_3d, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + record_one_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut record_one_3d, + ) + .unwrap(); + + assert_eq!(record_one_objects.record_a, baseline_objects.record_a + 0x100); + assert_eq!(get_u64(&baseline_ta, 0x10), baseline_objects.record_a); + assert_eq!(get_u64(&record_one_ta, 0x10), record_one_objects.record_a); + assert_eq!(get_u64(&baseline_3d, 0x20), baseline_objects.record_a); + assert_eq!(get_u64(&record_one_3d, 0x20), record_one_objects.record_a); + for (offset, (before, after)) in baseline_ta.iter().zip(record_one_ta.iter()).enumerate() { + if !(0x10..0x18).contains(&offset) { + assert_eq!(before, after, "unexpected TA descriptor change at {offset:#x}"); + } + } + for (offset, (before, after)) in baseline_3d.iter().zip(record_one_3d.iter()).enumerate() { + if !(0x20..0x28).contains(&offset) { + assert_eq!(before, after, "unexpected 3D descriptor change at {offset:#x}"); + } + } + } + + #[test] + fn pool_a_record_index_checks_overflow() { + let objects = RenderDescriptorObjects { + record_a: u64::MAX - 0xff, + ..objects() + }; + assert_eq!( + objects.with_pool_a_record_index(1), + Err(RenderBuildError::ArithmeticOverflow) + ); + } + + #[test] + fn fragment_hwpb_split_changes_only_fragment_plus_0x28() { + let parameters = G17pRenderParameters::source_rsrc8(); + let baseline_objects = objects(); + let hwpb = 0xffff_fc20_c099_8000; + let fragment_objects = baseline_objects.with_shared(hwpb); + let mut baseline_ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut split_ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut baseline_3d = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + let mut split_3d = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + + build_ta_descriptor( + ¶meters, + baseline_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut baseline_ta, + ) + .unwrap(); + build_ta_descriptor( + ¶meters, + baseline_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut split_ta, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + baseline_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut baseline_3d, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + fragment_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut split_3d, + ) + .unwrap(); + + assert_eq!(baseline_ta, split_ta); + assert_eq!(get_u64(&baseline_3d, 0x20), baseline_objects.record_a); + assert_eq!(get_u64(&split_3d, 0x20), baseline_objects.record_a); + assert_eq!(get_u64(&baseline_3d, 0x28), baseline_objects.shared); + assert_eq!(get_u64(&split_3d, 0x28), hwpb); + assert_eq!(get_u64(&baseline_3d, 0x30), baseline_objects.record_b); + assert_eq!(get_u64(&split_3d, 0x30), baseline_objects.record_b); + assert_eq!(get_u64(&baseline_3d, 0x38), baseline_objects.zero); + assert_eq!(get_u64(&split_3d, 0x38), baseline_objects.zero); + for (offset, (before, after)) in baseline_3d.iter().zip(split_3d.iter()).enumerate() { + if !(0x28..0x30).contains(&offset) { + assert_eq!( + before, after, + "unexpected 3D descriptor change at {offset:#x}" + ); + } + } + } + + #[test] + fn memory_backed_render_policy_is_independent_of_tib_size() { + for tib_blocks in [4, 8, 9, 12, 16] { + for process_empty_tiles in [false, true] { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.tib_blocks = tib_blocks; + parameters.process_empty_tiles = process_empty_tiles; + parameters.tile_config = 0x280 + | if process_empty_tiles { 0x1_0000 } else { 0 }; + let mut fragment = [0xa5u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_cold_opening_fragment_descriptor( + ¶meters, objects(), 1, 1, &mut fragment, + ) + .unwrap(); + assert_eq!(fragment[0x2100], 0); + assert_eq!(get_u64(&fragment, 0x1f38), tib_blocks); + assert_eq!(get_u64(&fragment, 0x1f58), parameters.tile_config); + // Keep real memory-backed partial phase programs, not the + // native memoryless count-two reset-only phase variant. + for (at, count) in [(0xec8, 16), (0x15e8, 23), (0x1d08, 10)] { + assert_eq!(get_u32(&fragment, at) & 0xffff, count); + } + apply_g17p_retained_descriptor_fields( + RenderDescriptorKind::Fragment, 0x7000098000, 0, + 1, 1, 1, false, false, &mut fragment, + ) + .unwrap(); + assert_eq!(fragment[0x2100], 0); + } + } + } + + #[test] + fn fragment_sample_count_matches_tile_configuration_on_every_submission() { + for (sample_log2, samples) in [(0, 1), (1, 2), (2, 4)] { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.utile_config = 0xa000 | sample_log2; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + + build_fragment_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut fragment, + ) + .unwrap(); + assert_eq!(get_u32(&fragment, 0x50), samples, "retained {samples}x"); + assert_eq!(get_u32(&fragment, 0xac), 0x10009); + assert_eq!(get_u64(&fragment, 0xb0) & 3, sample_log2); + + build_cold_opening_fragment_descriptor( + ¶meters, objects(), 1, 1, &mut fragment, + ) + .unwrap(); + assert_eq!(get_u32(&fragment, 0x50), samples, "first {samples}x"); + assert_eq!(get_u64(&fragment, 0xb0) & 3, sample_log2); + } + } + + #[test] + fn partial_render_uses_phase_specific_zls_control() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.depth_buffer = 0x5be8_2a00_00; + parameters.stencil_buffer = 0x5bea_4240_00; + parameters.depth_flags = 0x000c_0000; + parameters.partial_depth_flags = 0x000c_c000; + + let main = build_fragment_registers(¶meters).unwrap(); + assert_eq!( + main.iter().find(|reg| reg.number == 0x15319).unwrap().value, + 0x000c_0000, + ); + let store = partial_store_registers(¶meters); + let resume = partial_resume_registers(¶meters); + let load = partial_load_registers(¶meters); + for registers in [store.as_slice(), resume.as_slice(), load.as_slice()] { + assert_eq!( + registers.iter().find(|reg| reg.number == 0x15319).unwrap().value, + 0x000c_c000, + ); + } + } + + #[test] + fn cold_opening_descriptors_apply_only_the_native_first_work_overrides() { + let parameters = G17pRenderParameters::source_rsrc8(); + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_cold_opening_ta_descriptor(¶meters, objects(), 1, 0, 0, &mut tiling).unwrap(); + build_cold_opening_fragment_descriptor(¶meters, objects(), 1, 1, &mut fragment) + .unwrap(); + + assert_eq!(&tiling[0x38..0x3e], &[0x47, 0, 0x49, 0, 0x49, 0]); + assert_eq!(tiling[0x789], 0x08); + assert_eq!(get_u32(&tiling, 0x892), 1); + assert_eq!(&tiling[0x93e..0x940], &[0xd0, 0x91]); + + assert_eq!(get_u32(&fragment, 0x50), 1); + assert_eq!(&fragment[0x80..0x86], &[0x56, 0, 0x57, 0, 0x57, 0]); + assert_eq!(&fragment[0x88..0x8a], &[0x59, 0]); + assert_eq!(get_u32(&fragment, 0x90), 0); // final channel payload is staged later + assert_eq!(get_u32(&fragment, 0x2154), 1); + assert_eq!(get_u32(&fragment, 0x215c), 1); + assert_eq!(fragment[0x2100], 0); + assert_eq!(&fragment[0x2114..0x2134], &[0; 0x20]); + assert_eq!(get_u64(&fragment, 0x21d7), 0); + assert_eq!(fragment[0x222d], 1); + + } + + #[test] + fn cold_opening_ta_couples_the_hardware_buffer_id_without_clobbering_registers() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.ta_hardware_buffer_id = 5; + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + build_cold_opening_ta_descriptor( + ¶meters, + objects(), + 1, + 0, + parameters.ta_hardware_buffer_id, + &mut tiling, + ) + .unwrap(); + + assert_eq!(get_u32(&tiling, 0x18), 5); + // +0x300 is register 0x1c830 and +0x304 is its value word. + assert_eq!(get_u32(&tiling, 0x300), 0x1c830); + assert_eq!(get_u64(&tiling, 0x304), 5); + } + + #[test] + fn shared_usc_flist_id_drives_both_stage_registers_independently_of_pm_id() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.ta_hardware_buffer_id = 7; + parameters.usc_flist_hardware_buffer_id = 5; + // Every diagnostic bit set must not recouple the retired bit-16 + // capture literals to these source-owned registers. + parameters.native_ta_registers = u32::MAX; + + let tiling = build_ta_registers(¶meters).unwrap(); + let fragment = build_fragment_registers(¶meters).unwrap(); + assert_eq!( + tiling.iter().find(|write| write.number == 0x1c830).unwrap().value, + 7 + ); + assert_eq!( + tiling.iter().find(|write| write.number == 0x14308).unwrap().value, + 5 + ); + assert_eq!( + fragment.iter().find(|write| write.number == 0x14048).unwrap().value, + 5 + ); + assert_eq!( + fragment.iter().find(|write| write.number == 0x1c838).unwrap().value, + 7 + ); + } + + #[test] + fn descriptor_context_and_state_word_drive_every_native_work_key() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.ta_hardware_buffer_id = 5; + parameters.gpc_perf_state_map = 7; + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + let tiling_metadata = RenderDescriptorMetadata { + context_id: 9, + submission_ordinal: 4, + command_buffer_state_word: 0x40, + ta_hardware_buffer_id: parameters.ta_hardware_buffer_id, + submit_sequence: 7, + }; + let fragment_metadata = RenderDescriptorMetadata { + command_buffer_state_word: 0x41, + ..tiling_metadata + }; + + build_ta_descriptor(¶meters, objects(), tiling_metadata, &mut tiling).unwrap(); + build_fragment_descriptor(¶meters, objects(), fragment_metadata, &mut fragment) + .unwrap(); + + for (raw, offset, register, value) in [ + (&tiling[..], 0x300, 0x1c830, 5u64), + (&tiling[..], 0x36c, 0x10209, 0x900), + (&tiling[..], 0x378, 0x1c9f0, 0x900), + (&tiling[..], 0x384, 0x14320, 0x900), + (&fragment[..], 0x454, 0x1c838, 5u64), + (&fragment[..], 0x46c, 0x10211, 0x901), + (&fragment[..], 0x478, 0x10420, 0x901), + ] { + assert_eq!(get_u32(raw, offset), register); + assert_eq!(get_u64(raw, offset + 4), value); + } + assert_eq!(get_u32(&tiling, 0x0c), 9); + assert_eq!(get_u32(&fragment, 0x0c), 9); + } + + #[test] + fn tpc_extent_survives_cold_and_retained_descriptor_construction() { + for size in [0x800, 0x2000, 0x20000, 0x1000000] { + let mut p = G17pRenderParameters::source_rsrc8(); + p.tpc_size = size; + let mut ta = [0xa5; TA_DESCRIPTOR_SIZE]; + build_cold_opening_ta_descriptor(&p, objects(), 7, 0, 0, &mut ta).unwrap(); + for cold in [true, false] { + if !cold { + build_ta_descriptor(&p, objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), &mut ta).unwrap(); + } + apply_g17p_retained_descriptor_fields(RenderDescriptorKind::Tiling, + 0x7000090000, 0xfffffc20c0828000, 11, 12, 32, + cold, true, &mut ta).unwrap(); + assert_eq!(get_u64(&ta, 0x780), p.tpc); + assert_eq!(get_u64(&ta, 0x788), size); + } + } + } + + #[test] + fn rce_boundaries_survive_retained_descriptor_rebuilds() { + let u16_at = |raw: &[u8], at: usize| u16::from_le_bytes([raw[at], raw[at + 1]]); + for dimension in [32, 512, 1024] { + let mut p = G17pRenderParameters::source_rsrc8(); + p.width = dimension; + p.height = dimension; + let mut raw = [0xa5; FRAGMENT_DESCRIPTOR_SIZE]; + let mut ta = [0xa5; TA_DESCRIPTOR_SIZE]; + for ordinal in [0, 1, 31, 32, 97, 232, 513] { + if ordinal == 0 { + build_cold_opening_fragment_descriptor(&p, objects(), 7, 1, &mut raw).unwrap(); + build_cold_opening_ta_descriptor(&p, objects(), 7, 0, 0, &mut ta).unwrap(); + } else { + build_fragment_descriptor(&p, objects(), RenderDescriptorMetadata { + context_id: 7, submission_ordinal: ordinal, + ..RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment) + }, &mut raw).unwrap(); + build_ta_descriptor(&p, objects(), RenderDescriptorMetadata { + context_id: 7, submission_ordinal: ordinal, + ..RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling) + }, &mut ta).unwrap(); + } + apply_g17p_retained_descriptor_fields(RenderDescriptorKind::Fragment, + 0x7000098000, 0xfffffc20c0828000, 12, 12, ordinal, + ordinal == 0, true, &mut raw).unwrap(); + apply_g17p_retained_descriptor_fields(RenderDescriptorKind::Tiling, + 0x7000090000, 0xfffffc20c0828000, 11, 12, ordinal, + ordinal == 0, true, &mut ta).unwrap(); + assert_eq!([u16_at(&ta, 0x38), u16_at(&ta, 0x3a), + u16_at(&ta, 0x3c)], [71, 73, 73]); + assert_eq!(get_u32(&ta, 0x768) & 0xffff, 73); + for (index, register) in [(70, 0x01740), (71, 0x1c880), (72, 0x1c898)] { + assert_eq!(get_u32(&ta, 0x60 + index * 12), register); + } + // Independent native publication oracle: all 160 full FRG + // images carry these boundaries, including pending overlap. + assert_eq!([u16_at(&raw, 0x80), u16_at(&raw, 0x82), + u16_at(&raw, 0x84), u16_at(&raw, 0x88)], [86, 87, 87, 89]); + assert_eq!(get_u32(&raw, 0x7a8) & 0xffff, 89); + // Check which actual encoded instructions bracket the indices, + // so a changed recipe cannot silently retain stale boundaries. + for (index, register, value) in [(85, 0x01731, 1), + (86, 0x16020, 1), (87, 0x16020, 0), (88, 0x16068, 0x40000)] { + let at = 0xa0 + index * 12; + assert_eq!(get_u32(&raw, at), register); + assert_eq!(get_u64(&raw, at + 4), value); + } + } + } + } + + #[test] + fn allocation_epoch_survives_cold_and_retained_encoders_and_wrap() { + for epoch in [0, 1, 2, 3, 127, 255] { + for diagnostic in [false, true] { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.context_generation = epoch; + parameters.native_pm_bytes = diagnostic; + for cold in [false, true] { + let mut ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + if cold { + build_cold_opening_ta_descriptor(¶meters, objects(), 7, 0, 0, &mut ta).unwrap(); + build_cold_opening_fragment_descriptor(¶meters, objects(), 7, 1, &mut fragment).unwrap(); + } else { + build_ta_descriptor(¶meters, objects(), RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), &mut ta).unwrap(); + build_fragment_descriptor(¶meters, objects(), RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), &mut fragment).unwrap(); + } + assert_eq!(ta[0x94d], epoch); + assert_eq!(fragment[0x21e7], epoch); + // Retained record restaging changes submission identity, + // but must preserve the immutable execution-context epoch. + apply_g17p_retained_descriptor_fields(RenderDescriptorKind::Tiling, + 0x7000098000, 0xfffffc20c0828000, 0, 1, 513, false, true, &mut ta).unwrap(); + apply_g17p_retained_descriptor_fields(RenderDescriptorKind::Fragment, + 0x7000098000, 0xfffffc20c0828000, 1, 1, 513, false, true, &mut fragment).unwrap(); + assert_eq!(ta[0x94d], epoch); + assert_eq!(fragment[0x21e7], epoch); + } + } + } + } + + #[test] + fn completion_generation_matches_render_tags_across_reuse_and_wrap() { + let parameters = G17pRenderParameters::source_rsrc8(); + // Include a second client's cold submission and generations beyond the + // six-bit hardware tag wrap. The slot ordinal is intentionally unrelated. + for generation in [0, 1, 2, 62, 64, 65, 128, 0x12345678, u32::MAX - 1] { + for context_id in [1, 7] { + let metadata = RenderDescriptorMetadata { + context_id, + submission_ordinal: 2, + command_buffer_state_word: generation, + ..RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment) + }; + let mut ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + let mut event_control = [0xa5; 0x100]; + build_ta_descriptor(¶meters, objects(), metadata, &mut ta).unwrap(); + build_fragment_descriptor(¶meters, objects(), metadata, &mut fragment) + .unwrap(); + set_render_event_control_generation(&mut event_control, generation).unwrap(); + assert_eq!(get_u32(&event_control, 8), generation); + assert_eq!(&event_control[..8], &[0xa5; 8]); + assert!(event_control[12..].iter().all(|&v| v == 0xa5)); + let maintenance_key = ((context_id as u64) << 8) + | (get_u32(&event_control, 8) & 63) as u64; + for (raw, offset) in [(&ta[..], 0x36c), (&ta[..], 0x378), + (&ta[..], 0x384), (&fragment[..], 0x46c), + (&fragment[..], 0x478)] { + assert_eq!(get_u64(raw, offset + 4), maintenance_key); + } + } + } + assert_eq!( + set_render_event_control_generation(&mut [0; 11], 0), + Err(RenderBuildError::BufferTooSmall) + ); + } + + #[test] + fn fragment_parameter_buffer_owner_is_independent_of_perf_map_and_tail() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.native_ta_registers = u32::MAX; + + for (control, low, map, id) in [(0, 7, 3, 0), (1, 3, 7, 5), (1, 0, 1, 3)] { + parameters.ta_hardware_buffer_id = id; + parameters.gpc_perf_state_control = control; + parameters.gpc_perf_state_map_low = low; + parameters.gpc_perf_state_map = map; + let fragment = build_fragment_registers(¶meters).unwrap(); + let write = fragment + .iter() + .find(|write| write.number == 0x1c838) + .unwrap(); + assert_eq!(write.value, id as u64); + + let mut descriptor = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_fragment_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut descriptor, + ) + .unwrap(); + assert_eq!(get_u32(&descriptor, 0x2104), control); + assert_eq!(get_u32(&descriptor, 0x210c), low); + assert_eq!(get_u32(&descriptor, 0x2110), u32::MAX); + assert_eq!(get_u32(&descriptor, 0x2124), 1); + assert_eq!(get_u32(&descriptor, 0x2128), 1); + assert_eq!(get_u32(&descriptor, 0x212c), 0); + assert_eq!(get_u32(&descriptor, 0x2130), 0); + } + } + + #[test] + fn priority_zero_ta_force_stamp_is_independent_of_physical_qid() { + for (queue_id, ordinal, cold, expected) in [ + (0, 0, true, 1), + (0, 1, false, 1), + (0, 2, false, 1), + (2, 0, true, 1), + (2, 1, false, 1), + (5, 0, true, 1), + (5, 1, false, 1), + ] { + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + apply_g17p_retained_descriptor_fields( + RenderDescriptorKind::Tiling, + 0x70_0001_8000, + 0xffff_fc20_c082_8000, + queue_id, + queue_id + 1, + ordinal, + cold, + true, + &mut tiling, + ) + .unwrap(); + assert_eq!(get_u32(&tiling, 0x8c2), expected); + assert_eq!(get_u32(&tiling, 0x8be), 0); + assert_eq!( + get_u64(&tiling, 0x8c6), + ((ordinal + 1) as u64) << 40 | (ordinal + 1) as u64 + ); + } + } + + #[test] + fn retained_descriptor_refresh_stages_native_external_rce_and_advances_item_two() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.lifecycle = 0x1122_3344_5566_7788; + parameters.fragment_lifecycle = 0x99aa_bbcc_ddee_ff00; + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + let tiling_va = 0x70_0001_8000; + let fragment_va = 0x70_0009_8000; + build_ta_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata { + context_id: 1, + submission_ordinal: 1, + command_buffer_state_word: 2, + ta_hardware_buffer_id: 0, + submit_sequence: 3, + }, + &mut tiling, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata { + context_id: 1, + submission_ordinal: 1, + command_buffer_state_word: 3, + ta_hardware_buffer_id: 0, + submit_sequence: 2, + }, + &mut fragment, + ) + .unwrap(); + apply_g17p_retained_descriptor_fields( + RenderDescriptorKind::Tiling, + tiling_va, + 0xffff_fc20_c082_8000, + 5, + 6, + 1, + false, + false, + &mut tiling, + ) + .unwrap(); + apply_g17p_retained_descriptor_fields( + RenderDescriptorKind::Fragment, + fragment_va, + 0xffff_fc20_c082_8000, + 6, + 6, + 1, + false, + false, + &mut fragment, + ) + .unwrap(); + let rce_va = 0x70_0028_cf20; + let mut rce = [0u8; G17P_FRAGMENT_RCE_STORAGE_SIZE]; + stage_g17p_fragment_rce_programs(&mut fragment, rce_va, &mut rce).unwrap(); + + assert_eq!(get_u64(&tiling, 0x760), tiling_va + 0x60); + assert_eq!(get_u32(&tiling, 0x79c), 6); + assert_eq!(get_u32(&tiling, 0x8ba), 5); + assert_eq!(get_u32(&tiling, 0x7a0), 0x200); + assert_eq!(get_u32(&tiling, 0x7a8), 2); + assert_eq!(get_u32(&tiling, 0x8d4), 0x1_0000); + assert_eq!(get_u64(&tiling, 0x934), 0xffff_fc20_c082_8000); + assert_eq!(get_u64(&fragment, 0x7a0), fragment_va + 0xa0); + assert_eq!(get_u64(&fragment, 0xec0), fragment_va + 0x7c0); + assert_eq!(get_u64(&fragment, 0x15e0), fragment_va + 0xee0); + assert_eq!(get_u64(&fragment, 0x1d00), fragment_va + 0x1600); + assert_eq!( + &rce[..FRAGMENT_REGISTER_COUNT * REGISTER_BYTES], + &fragment[0xa0..0xa0 + FRAGMENT_REGISTER_COUNT * REGISTER_BYTES], + ); + assert_eq!(get_u32(&tiling, 0x086e), 0x1122_3344); + assert_eq!(get_u64(&tiling, 0x08ce), 0x5566_7788); + assert_eq!(get_u32(&fragment, 0x2108), 0x99aa_bbcc); + assert_eq!(get_u64(&fragment, 0x2168), 0xddee_ff00); + assert_eq!( + &rce[0x720..0x720 + PARTIAL_STORE_REGISTER_COUNT * REGISTER_BYTES], + &fragment[0x7c0..0x7c0 + PARTIAL_STORE_REGISTER_COUNT * REGISTER_BYTES], + ); + assert_eq!(get_u32(&fragment, 0x2154), 6); + assert_eq!(get_u32(&fragment, 0x215c), 0); + assert_eq!(get_u32(&fragment, 0x2170), 1); + assert_eq!(get_u64(&fragment, 0x21ce), 0xffff_fc20_c082_8000); + } + + fn publication_input(kind: RenderDescriptorKind) -> QueuePublicationInput { + let fragment = matches!(kind, RenderDescriptorKind::Fragment); + let grid = fragment as u16; + QueuePublicationInput { + queue: 0xffff_fc20_c000_8000 + grid as u64 * 0xc0, + pointers: 0xffff_fc20_0001_0000 + grid as u64 * 0x80, + item_ring: 0xffff_fc20_c000_c000 + grid as u64 * 0x2870, + channel_ring: 0xffff_fc20_c010_0000 + grid as u64 * 0x4000, + channel_producer: 0xffff_fc20_0170_0008 + grid as u64 * 0x100, + items: [ + 0xffff_fc20_c020_0000 + grid as u64 * 0x1_0000, + 0xffff_fc20_c020_4000 + grid as u64 * 0x1_0000, + 0xffff_fc20_c020_8000 + grid as u64 * 0x1_0000, + ], + grid_index: grid, + slot_index: 0, + current_write: 0, + group_number: 1, + first_submit: true, + } + } + + fn publication_hash(publication: &QueuePublication) -> u64 { + let mut value = 0xcbf2_9ce4_8422_2325u64; + let mut add = |address: u64, bytes: &[u8]| { + for byte in address.to_le_bytes() { + value ^= byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + for byte in (bytes.len() as u32).to_le_bytes() { + value ^= byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + for byte in bytes { + value ^= *byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + }; + for entry in &publication.item_entries { + add(entry.address, &entry.bytes); + } + add(publication.event.address, &publication.event.bytes); + add( + publication.queue_write.address, + &publication.queue_write.bytes, + ); + add(publication.ring_slot.address, &publication.ring_slot.bytes); + add(publication.producer.address, &publication.producer.bytes); + value + } + + #[test] + fn paired_publication_matches_m1n1_9165484() { + let publication = build_paired_queue_publication( + publication_input(RenderDescriptorKind::Tiling), + publication_input(RenderDescriptorKind::Fragment), + ) + .unwrap(); + assert_eq!(publication.tiling.write_after, 3); + assert_eq!(publication.fragment.write_after, 3); + assert_eq!(publication_hash(&publication.tiling), 0x8d2e_1298_cc1d_39c7); + assert_eq!( + publication_hash(&publication.fragment), + 0x75e1_3fd1_c48e_51ae + ); + assert_eq!( + publication.tiling.ring_slot.bytes[0x14..0x18], + 0x0100_0003u32.to_le_bytes() + ); + assert_eq!( + publication.fragment.ring_slot.bytes[0x14..0x18], + 0x0101_0003u32.to_le_bytes() + ); + assert_eq!(publication.tiling.producer.bytes, 1u32.to_le_bytes()); + assert_eq!(publication.fragment.producer.bytes, 1u32.to_le_bytes()); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/render_passthrough.rs b/drivers/gpu/drm/asahi/g18/fw/render_passthrough.rs new file mode 100644 index 0000000000000..e219cee8d5dc6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/render_passthrough.rs @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Byte map from the native processRenderSetup slice, not a complete descriptor. +//! Source: the native render passthrough map, version 3. +//! Unknown bytes return None; callers must keep them open in the ledger. +//! Gears uses the common body but its parser envelope needs separate admission. +#![allow(dead_code)] +// (descriptor offset, segment offset, extent, byte mask); mask 0 means zero. +const FIELDS: &[(usize, usize, usize, u8)] = &[ + (0x440, 0x71c, 4, 255), + (0x4e0, 0x480, 8, 255), + (0x4e8, 0x478, 8, 255), + (0x4f0, 0x788, 8, 255), + (0x4f8, 0x8b0, 24, 255), + (0x510, 0x488, 8, 255), + (0x518, 0x398, 128, 255), + (0x598, 0x7f0, 128, 255), + (0x618, 0x418, 48, 255), + (0x648, 0x790, 48, 255), + (0x678, 0x490, 40, 255), + (0x6a0, 0x4d8, 8, 255), + (0x6a8, 0x870, 8, 255), + (0x6b0, 0x500, 8, 255), + (0x6b8, 0x528, 8, 255), + (0x6c0, 0x880, 8, 255), + (0x6c8, 0x4b8, 8, 255), + (0x6d0, 0x4e0, 8, 255), + (0x6d8, 0x890, 8, 255), + (0x6e0, 0x508, 8, 255), + (0x6e8, 0x530, 8, 255), + (0x6f0, 0x898, 8, 255), + (0x6f8, 0x4c0, 8, 255), + (0x700, 0x4e8, 8, 255), + (0x708, 0x510, 8, 255), + (0x710, 0x538, 8, 255), + (0x718, 0x4c8, 8, 255), + (0x720, 0x4f0, 8, 255), + (0x728, 0x878, 8, 255), + (0x730, 0x4d0, 8, 255), + (0x738, 0x4f8, 8, 255), + (0x740, 0x518, 8, 255), + (0x748, 0x540, 8, 255), + (0x750, 0x888, 8, 255), + (0x758, 0x520, 8, 255), + (0x760, 0x548, 8, 255), + (0x768, 0x570, 8, 255), + (0x770, 0x550, 32, 255), + (0x790, 0x458, 32, 255), + (0x7b0, 0x7d0, 16, 255), + (0x7c0, 0x448, 12, 255), + (0x7cc, 0x7c0, 12, 255), + (0x7d8, 0x700, 8, 255), + (0x7e8, 0x590, 8, 255), + (0x7f8, 0x598, 4, 255), + (0x818, 0x717, 1, 1), + (0x830, 0x578, 8, 255), + (0x838, 0x8d4, 1, 1), + (0x8a0, 0x8dc, 1, 1), + (0x8a4, 0x8d8, 4, 255), + (0x8a8, 0x8e0, 8, 255), + (0x8c0, 0x708, 4, 255), + (0x8c4, 0x70d, 1, 1), + (0x8c5, 0x8e8, 1, 1), + (0x8c6, 0x8a0, 5, 1), + (0x8cd, 0x710, 2, 1), + (0x8d0, 0x713, 1, 1), + (0x8d1, 0x716, 1, 1), + (0x8d2, 0x8a5, 2, 1), + (0x970, 0x8f8, 12, 255), + (0x981, 0x8ea, 1, 255), + (0x982, 0x714, 2, 1), + (0xac8, 0x8d0, 4, 255), + (0xaf0, 0x580, 4, 255), + (0xaf8, 0x588, 8, 255), + (0xb18, 0x738, 48, 255), + (0xb58, 0x768, 28, 255), + (0xfd8, 0x320, 4, 255), + (0x1058, 0xc8, 48, 255), + (0x1088, 0x2b8, 40, 255), + (0x10b0, 0x2e8, 20, 255), + (0x10c8, 0xf8, 112, 255), + (0x1138, 0x180, 12, 255), + (0x1158, 0x2ac, 12, 255), + (0x1178, 0x30e, 1, 1), + (0x1179, 0x0, 3, 0), + (0x1180, 0x168, 8, 255), + (0x11a1, 0x305, 3, 1), + (0x11a5, 0x309, 1, 1), + (0x11a6, 0x8e8, 1, 1), + (0x11a7, 0x30a, 2, 1), + (0x11aa, 0x311, 1, 1), + (0x12d4, 0x30c, 2, 1), + (0x1410, 0x300, 4, 255), + (0x1550, 0x170, 4, 255), + (0x1558, 0x178, 8, 255), + (0x1590, 0x348, 48, 255), + (0x15d0, 0x378, 28, 255), + (0x15f1, 0x313, 1, 1), + (0x1660, 0x312, 1, 1), +]; + +pub(crate) fn field(command: &[u8], descriptor: usize, width: usize) -> Option { + if !matches!(command.len(), 0xac0 | 0xad8) || width == 0 || width > 8 { return None; } + let mut value = 0u64; + for byte in 0..width { + let offset = descriptor.checked_add(byte)?; + let &(dest, source, count, mask) = FIELDS.iter().find(|&&(dest, _, count, _)| + offset >= dest && offset - dest < count)?; + let _ = count; + let v = if mask == 0 { 0 } else { *command.get(source + offset - dest)? & mask }; + value |= u64::from(v) << (byte * 8); + } + Some(value) +} + +/// Only the empirically mapped userspace bytes. Unknown prepare-time fields +/// are counted, returned as zero for dry-run output, and MUST block admission. +pub(crate) fn capture_fields(command: &[u8]) -> (crate::g18::fw::render_command::G18RenderCommandInputs, usize) { + use crate::g18::fw::render_command::*; + fn read(c: &[u8], offset: usize, bytes: usize, open: &mut usize) -> u64 { + match field(c, offset, bytes) { Some(v) => v, None => { *open += 1; 0 } } + } + let mut open = 0; + let mut inputs = G18RenderCommandInputs::default(); + inputs.ta.d12d0 = read(command, 0x12d0, 4, &mut open) as u32; + inputs.ta.d12c0 = read(command, 0x12c0, 4, &mut open) as u32; + inputs.ta.d12c8 = read(command, 0x12c8, 8, &mut open) as u64; + inputs.ta.d11b8 = read(command, 0x11b8, 8, &mut open) as u64; + inputs.ta.d11d8 = read(command, 0x11d8, 8, &mut open) as u64; + inputs.ta.d1068 = read(command, 0x1068, 8, &mut open) as u64; + inputs.ta.d1070 = read(command, 0x1070, 8, &mut open) as u64; + inputs.ta.d10f8 = read(command, 0x10f8, 8, &mut open) as u64; + inputs.ta.d08a0 = read(command, 0x8a0, 1, &mut open) as u8; + inputs.ta.d08a4 = read(command, 0x8a4, 4, &mut open) as u32; + inputs.ta.d08a8 = read(command, 0x8a8, 4, &mut open) as u32; + inputs.ta.d08ac = read(command, 0x8ac, 4, &mut open) as u32; + inputs.ta.d08b0 = read(command, 0x8b0, 8, &mut open) as u64; + inputs.ta.d1418 = core::array::from_fn(|i| read(command, 0x1418 + i * 8, 8, &mut open)); + inputs.ta.d1428 = read(command, 0x1428, 8, &mut open) as u64; + inputs.ta.d1430 = read(command, 0x1430, 8, &mut open) as u64; + inputs.ta.d1474 = read(command, 0x1474, 1, &mut open) as u8; + inputs.ta.d11a4 = read(command, 0x11a4, 1, &mut open) as u8; + inputs.ta.d11a7 = read(command, 0x11a7, 1, &mut open) as u8; + inputs.fragment.d158 = read(command, 0x158, 8, &mut open) as u64; + inputs.fragment.d150 = read(command, 0x150, 4, &mut open) as u32; + inputs.fragment.d980_is_2 = read(command, 0x980, 4, &mut open) == 2; + inputs.fragment.d08cf = read(command, 0x8cf, 1, &mut open) as u8; + inputs.fragment.dbbc = read(command, 0xbbc, 1, &mut open) as u8; + inputs.fragment.d08d8 = read(command, 0x8d8, 8, &mut open) as u64; + inputs.fragment.d08f0 = read(command, 0x8f0, 8, &mut open) as u64; + inputs.fragment.d08f8 = read(command, 0x8f8, 4, &mut open) as u32; + inputs.fragment.d0900 = read(command, 0x900, 4, &mut open) as u32; + inputs.fragment.d0908 = read(command, 0x908, 4, &mut open) as u32; + inputs.fragment.d0910 = core::array::from_fn(|i| read(command, 0x910 + i * 8, 8, &mut open)); + inputs.fragment.d0920 = read(command, 0x920, 8, &mut open) as u64; + inputs.fragment.d0930 = read(command, 0x930, 4, &mut open) as u32; + inputs.fragment.d04e0 = read(command, 0x4e0, 8, &mut open) as u64; + inputs.fragment.d04e8 = read(command, 0x4e8, 8, &mut open) as u64; + inputs.fragment.d04f0 = read(command, 0x4f0, 8, &mut open) as u64; + inputs.fragment.d0510 = read(command, 0x510, 8, &mut open) as u64; + inputs.fragment.d0518 = core::array::from_fn(|i| core::array::from_fn(|j| read(command, 0x518 + i * 16 + j * 8, 8, &mut open))); + inputs.fragment.d0690 = read(command, 0x690, 8, &mut open) as u64; + inputs.fragment.d0698 = read(command, 0x698, 8, &mut open) as u64; + inputs.fragment.d06a0 = core::array::from_fn(|i| read(command, 0x6a0 + i * 8, 8, &mut open)); + inputs.fragment.d06b0 = read(command, 0x6b0, 8, &mut open) as u64; + inputs.fragment.d06b8 = core::array::from_fn(|i| read(command, 0x6b8 + i * 8, 8, &mut open)); + inputs.fragment.d06d8 = read(command, 0x6d8, 8, &mut open) as u64; + inputs.fragment.d06f0 = read(command, 0x6f0, 8, &mut open) as u64; + inputs.fragment.d06f8 = read(command, 0x6f8, 8, &mut open) as u64; + inputs.fragment.d0708 = read(command, 0x708, 8, &mut open) as u64; + inputs.fragment.d0728 = read(command, 0x728, 8, &mut open) as u64; + inputs.fragment.d0730 = read(command, 0x730, 8, &mut open) as u64; + inputs.fragment.d0750 = read(command, 0x750, 8, &mut open) as u64; + inputs.fragment.d0758 = read(command, 0x758, 8, &mut open) as u64; + inputs.fragment.d0770 = core::array::from_fn(|i| read(command, 0x770 + i * 8, 8, &mut open)); + inputs.fragment.d0780 = core::array::from_fn(|i| read(command, 0x780 + i * 8, 8, &mut open)); + inputs.fragment.d0790 = core::array::from_fn(|i| read(command, 0x790 + i * 8, 8, &mut open)); + inputs.fragment.d07b0 = core::array::from_fn(|i| read(command, 0x7b0 + i * 8, 8, &mut open)); + inputs.fragment.d07c0 = read(command, 0x7c0, 8, &mut open) as u64; + inputs.fragment.d07c8 = read(command, 0x7c8, 4, &mut open) as u32; + inputs.fragment.d07cc = read(command, 0x7cc, 8, &mut open) as u64; + inputs.fragment.d07d4 = read(command, 0x7d4, 4, &mut open) as u32; + inputs.fragment.d07d8 = read(command, 0x7d8, 8, &mut open) as u64; + inputs.fragment.d07e8 = read(command, 0x7e8, 8, &mut open) as u64; + inputs.fragment.d07f8 = read(command, 0x7f8, 4, &mut open) as u32; + inputs.fragment.d04d0 = read(command, 0x4d0, 4, &mut open) as u32; + inputs.fragment.d08c4 = read(command, 0x8c4, 1, &mut open) as u8; + inputs.fragment.dc98 = read(command, 0xc98, 4, &mut open) as u32; + inputs.fragment.dc9c = read(command, 0xc9c, 1, &mut open) as u8; + inputs.fragment.d08a0 = read(command, 0x8a0, 1, &mut open) as u8; + inputs.fragment.d08a4 = read(command, 0x8a4, 4, &mut open) as u32; + inputs.fragment.d08a8 = read(command, 0x8a8, 4, &mut open) as u32; + inputs.fragment.d08ac = read(command, 0x8ac, 4, &mut open) as u32; + inputs.fragment.dad0 = core::array::from_fn(|i| read(command, 0xad0 + i * 8, 8, &mut open)); + inputs.fragment.dae0 = read(command, 0xae0, 8, &mut open) as u64; + inputs.fragment.dae8 = read(command, 0xae8, 8, &mut open) as u64; + inputs.fragment.d08b0 = core::array::from_fn(|i| read(command, 0x8b0 + i * 8, 8, &mut open)); + inputs.fragment.d08c5_flags = [0x8c5, 0x8c6, 0x8c7, 0x8cb].map(|o| read(command, o, 1, &mut open) as u8); + inputs.fragment.d08cd = read(command, 0x8cd, 1, &mut open) as u8; + (inputs, open) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/rtkit.rs b/drivers/gpu/drm/asahi/g18/fw/rtkit.rs new file mode 100644 index 0000000000000..e7fbe94188167 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/rtkit.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 RTKit endpoint numbers and the two firmware messages the live boot sends. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. + +pub(crate) const ENDPOINT_GFX_MESSAGES: u8 = 0x20; + +pub(crate) const ENDPOINT_GFX_INTERRUPTS: u8 = 0x21; + +const INITDATA_TAG: u64 = 0x0081_0000_0000_0000; + +const INITDATA_ADDRESS_MASK: u64 = 0x0000_0fff_ffff_ffff; + +const INITDATA_ADDRESS_PREFIX: u64 = 0xffff_f000_0000_0000; + +/// `AGXArmFirmware::powerTransitionEnd(AGFIFirmwareRole, bool)` closes the GPU +/// power-on transition by sending this note to EACH firmware role. +/// +/// KEXT_PROVEN, 25G83 @0xfffffe00088936a4. Both send sites are identical: +/// +/// ldr x0, [x19, #6616] / [x19, #6672] ; the two role objects +/// ldr x8, [obj + 0x8a8] ; sendAsyncNoteToFirmware +/// mov x1, #0x89000000000000 ; tag 0x89, payload 0 +/// mov w2, #0x1 ; gated=true +/// blraa x8, x16 +/// +/// MEASURED on hardware (m1n1 hypervisor capture): it is the LAST +/// EP-0x21 message of bring-up, sent once per role right after the initdata +/// handshake completes and 11.4 seconds before the first work doorbell: +/// +/// 57.95ms role0 -> ep=0x21 0x0089000000000000 +/// 59.31ms role1 -> ep=0x21 0x0089000000000000 +/// ... +/// 11425.12ms role0 -> ep=0x21 0x0083000000000000 <- first doorbell +/// +/// This driver has never sent it. Its EP-0x21 vocabulary was 0x81 initdata, +/// 0x83 doorbell, 0x84 device control and 0x86 recovery inspection only. +const POWER_TRANSITION_END_TAG: u64 = 0x0089_0000_0000_0000; + +/// Encode the role's power-transition-end note. The payload is zero in every +/// captured instance and the kext materialises the bare tag with a single +/// `movz x1, #0x89, lsl #48`. +pub(crate) const fn encode_power_transition_end() -> u64 { + POWER_TRANSITION_END_TAG +} + +/// Refusals made by the evidence-backed codec. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum WireError { + UnexpectedEndpointSet, + UnsupportedInterface, + AddressOutOfRange, + InvalidInitdataTag, + InvalidRuntimeType, + QueueIdOutOfRange, + UnmodeledBits, +} + +/// Encode the G17 init-data handoff exactly as the recovered host does. +/// +/// The host instruction truncates to 44 bits. Linux instead rejects an +/// address unless it is already in low-44-bit or firmware-canonical form. +pub(crate) const fn encode_initdata_doorbell(firmware_va: u64) -> Result { + let high = firmware_va & !INITDATA_ADDRESS_MASK; + if high != 0 && high != INITDATA_ADDRESS_PREFIX { + return Err(WireError::AddressOutOfRange); + } + + Ok(INITDATA_TAG | (firmware_va & INITDATA_ADDRESS_MASK)) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/scheduler.rs b/drivers/gpu/drm/asahi/g18/fw/scheduler.rs new file mode 100644 index 0000000000000..e4c793da78563 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/scheduler.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Command-specific G18 / 26A428 scheduler record layout. +//! +//! These are the 0x100-stride records referenced by compute command +0x10, +//! distinct from the queue's 0x200-stride scheduler entries. Native evidence: +//! AGXCommandBuffer::init 0x8be76b0..0x8be7a54 and +//! nextCommandBufferState 0x8be65bc..0x8be6604 / 0x8be668c..0x8be66f0. + +#![allow(dead_code)] + +pub(crate) const G18_SCHEDULER_RECORD_COUNT: usize = 36; +pub(crate) const G18_SCHEDULER_RECORD_STRIDE: usize = 0x100; +pub(crate) const G18_SCHEDULER_RECORD_EXTENT: usize = 0xc8; +pub(crate) const G18_SCHEDULER_RECORD_BYTES: usize = + G18_SCHEDULER_RECORD_COUNT * G18_SCHEDULER_RECORD_STRIDE; +pub(crate) const G18_SCHEDULER_COUNTER_BYTES: usize = G18_SCHEDULER_RECORD_COUNT * 4; +/// Native G18 channel initialization 0x8c45470/0x8c45488 sets device +0x6f8. +pub(crate) const G18_SCHEDULER_DEFAULT_THRESHOLD: u32 = 80; + +/// Initialize the full native-sized backing, before publishing any pointer. +/// The counter backing is a separately owned, zeroed allocation. +pub(crate) fn initialize_g18_scheduler_records( + out: &mut [u8; G18_SCHEDULER_RECORD_BYTES], + counter_base: u64, +) -> Option<()> { + if counter_base == 0 || counter_base & 3 != 0 { + return None; + } + counter_base.checked_add(G18_SCHEDULER_COUNTER_BYTES as u64)?; + out.fill(0); + for index in 0..G18_SCHEDULER_RECORD_COUNT { + let start = index * G18_SCHEDULER_RECORD_STRIDE; + out[start..start + 8].copy_from_slice(&(counter_base + index as u64 * 4).to_le_bytes()); + } + Some(()) +} + +/// Reset an unpublished record exactly as the native selected-slot reset. +/// Only the first 0xc8 bytes are reset; the native 0x100 stride tail survives. +/// The caller separately zeros the submitted counter. Never call this while +/// firmware can still own the record or its completion/list fields. +pub(crate) fn prepare_g18_scheduler_record( + out: &mut [u8; G18_SCHEDULER_RECORD_STRIDE], + submitted_counter: u64, + sequence: u32, + fallback_threshold: u32, + override_threshold: u32, + cap: u32, +) -> Option<()> { + if submitted_counter == 0 || submitted_counter & 3 != 0 { + return None; + } + submitted_counter.checked_add(4)?; + let threshold = if override_threshold == 0 { + fallback_threshold + } else { + override_threshold + }; + let threshold = if cap == 0 { + threshold + } else { + threshold.min(cap) + }; + out[..G18_SCHEDULER_RECORD_EXTENT].fill(0); + out[..8].copy_from_slice(&submitted_counter.to_le_bytes()); + out[8..12].copy_from_slice(&sequence.to_le_bytes()); + out[0x10..0x14].copy_from_slice(&threshold.to_le_bytes()); + Some(()) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/sksm.rs b/drivers/gpu/drm/asahi/g18/fw/sksm.rs new file mode 100644 index 0000000000000..d8c082230f433 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/sksm.rs @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8142/25G76 HAL300 SKSM entry and tag-14 serialization. +//! +//! The M5 host clears 0x200 bytes and writes two-bit dependency types at +//! +0x180. Neo's 0x180-byte entry cannot represent this data. These pure +//! builders do not register queues, publish producers or ring a doorbell. +//! Callers must retain every referenced object and implement the M5 lifetime +//! and completion protocol before any generated bytes reach firmware. + +pub(crate) const M5_SKSM_ENTRY_BYTES: usize = 0x200; +pub(crate) const M5_SKSM_ENTRY_COUNT: u32 = 256; + +/// Logical channel selector measured in native TA, 3D and CL tag-14 records. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum M5DataMaster { + Tiling = 0, + Fragment = 1, + Compute = 2, +} +const STAMP_MASK: u64 = (1 << 40) - 1; +const PACKED_ADDRESS_MASK: u64 = (1 << 38) - 1; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5SksmError { + QueueId, + Priority, + Stamp, + QueueFull, + Dependencies, + DependencyType, + Address, + Mcache, + Rce, + Qos, + AddCount, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmDependency { + pub(crate) queue_id: u8, + /// Already converted to the hardware's 32-bit generation / 8-bit index. + pub(crate) stamp: u64, + pub(crate) kind: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmRce { + pub(crate) address: u64, + pub(crate) tag: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmMcache { + pub(crate) address: u64, + pub(crate) index: u8, + pub(crate) count: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmQueue { + pub(crate) id: u8, + /// Host queue +0x1c, not the outer firmware channel's priority. + pub(crate) priority: u8, + pub(crate) current_stamp: u64, + pub(crate) submitted: u32, + pub(crate) previous_stamp: Option, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmOperands<'a> { + /// Explicit dependencies already resolved by their owning event machine. + pub(crate) barriers: &'a [M5SksmDependency], + /// Native convertEventMachineTSToHWTS(command) - 1, for first publication. + /// This must be grounded in the queue's initialized completion state. + pub(crate) first_predecessor: u64, + pub(crate) implicit_dependency_type: u8, + pub(crate) mcache: Option, + pub(crate) flag_bit1: bool, + pub(crate) flag_bit61: bool, + pub(crate) flag_bit63: bool, + pub(crate) payload: [u64; 2], + pub(crate) qos_byte_32: u8, + pub(crate) qos_class_40: u8, + pub(crate) qos_word_48: u16, + pub(crate) event_mask: [u64; 4], + pub(crate) rce_kind: u8, + pub(crate) rce: [M5SksmRce; 4], + /// Native encoder's scalar x5 argument, copied unmodified to +0x178. + pub(crate) auxiliary: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PreparedM5SksmEntry { + pub(crate) bytes: [u8; M5_SKSM_ENTRY_BYTES], + pub(crate) offset: u32, + /// May only replace host state after entry storage and a dmb ish. + pub(crate) queue_after: M5SksmQueue, +} + +fn check_queue(id: u8, priority: u8, stamp: u64) -> Result<(), M5SksmError> { + if id > 0x7f { + return Err(M5SksmError::QueueId); + } + if priority > 3 { + return Err(M5SksmError::Priority); + } + if stamp & !STAMP_MASK != 0 { + return Err(M5SksmError::Stamp); + } + Ok(()) +} + +fn packed_address(address: u64) -> Result { + if address & 31 != 0 || address >> 5 > PACKED_ADDRESS_MASK { + return Err(M5SksmError::Address); + } + Ok(address >> 5) +} + +fn put(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Recovered from M5 encodeKickEntry at 25G76 0xfffffe00089e750c and its +/// encodeDependencyList at 0xfffffe00089e781c. Geometry is deliberately fixed +/// to the independently captured 256 x 512-byte M5 queues. +pub(crate) fn prepare_m5_sksm_entry( + queue: M5SksmQueue, + operands: M5SksmOperands<'_>, +) -> Result { + check_queue(queue.id, queue.priority, queue.current_stamp)?; + if queue.submitted >= M5_SKSM_ENTRY_COUNT { + return Err(M5SksmError::QueueFull); + } + // Native always appends one implicit predecessor after explicit barriers. + if operands.barriers.len() >= 32 { + return Err(M5SksmError::Dependencies); + } + if operands.qos_class_40 > 31 { + return Err(M5SksmError::Qos); + } + if operands.rce_kind > 3 || operands.rce.iter().any(|r| r.tag > 0x7f) { + return Err(M5SksmError::Rce); + } + let mut bytes = [0; M5_SKSM_ENTRY_BYTES]; + let mut header = (queue.current_stamp << 2) + | ((queue.id as u64) << 42) + | ((queue.priority as u64) << 59) + | ((operands.flag_bit1 as u64) << 1) + | ((operands.flag_bit61 as u64) << 61) + | ((operands.flag_bit63 as u64) << 63); + if let Some(mcache) = operands.mcache { + if mcache.index > 63 || mcache.count == 0 || mcache.count > 64 { + return Err(M5SksmError::Mcache); + } + let packed = packed_address(mcache.address)?; + put( + &mut bytes, + 8, + (mcache.index as u64) | (packed << 6) | ((mcache.count as u64 - 1) << 54), + ); + header |= 1 << 58; + } + put(&mut bytes, 0, header); + put(&mut bytes, 0x10, operands.payload[0]); + put(&mut bytes, 0x18, operands.payload[1]); + let predecessor = M5SksmDependency { + queue_id: queue.id, + stamp: queue.previous_stamp.unwrap_or(operands.first_predecessor), + kind: operands.implicit_dependency_type, + }; + let mut kinds = 0; + for (i, dependency) in operands + .barriers + .iter() + .chain(core::iter::once(&predecessor)) + .enumerate() + { + check_queue(dependency.queue_id, 0, dependency.stamp)?; + if dependency.kind > 3 { + return Err(M5SksmError::DependencyType); + } + put( + &mut bytes, + 0x28 + i * 8, + dependency.stamp | ((dependency.queue_id as u64) << 40), + ); + kinds |= (dependency.kind as u64) << (i * 2); + } + let count = operands.barriers.len() + 1; + let valid = if count == 32 { + u32::MAX + } else { + (1u32 << count) - 1 + }; + put( + &mut bytes, + 0x20, + valid as u64 + | ((operands.qos_byte_32 as u64) << 32) + | ((operands.qos_class_40 as u64) << 40) + | ((operands.qos_word_48 as u64) << 48), + ); + put(&mut bytes, 0x180, kinds); + for (i, mask) in operands.event_mask.into_iter().enumerate() { + put(&mut bytes, 0x128 + i * 8, mask); + } + bytes[0x148] = operands.rce_kind; + for (i, rce) in operands.rce.into_iter().enumerate() { + put( + &mut bytes, + 0x150 + i * 8, + packed_address(rce.address)? | ((rce.tag as u64) << 43), + ); + } + put(&mut bytes, 0x178, operands.auxiliary); + Ok(PreparedM5SksmEntry { + bytes, + offset: (queue.current_stamp as u32 & 255) * M5_SKSM_ENTRY_BYTES as u32, + queue_after: M5SksmQueue { + current_stamp: (queue.current_stamp + 1) & STAMP_MASK, + submitted: queue.submitted + 1, + previous_stamp: Some(queue.current_stamp), + ..queue + }, + }) +} + +/// Native CL add-kicks record at the captured submitBuffer publication site. +/// This is a packed 21-byte command, unrelated to the outer 24-byte entry. +pub(crate) fn encode_m5_sksm_add_kicks( + queue_id: u8, + priority: u8, + first_stamp: u64, + count: u16, +) -> Result<[u8; 21], M5SksmError> { + encode_m5_sksm_add_kicks_for( + M5DataMaster::Compute, + queue_id, + priority, + first_stamp, + count, + ) +} + +/// Encode a retained queue's stage explicitly; graphics never uses CL's DM2. +pub(crate) fn encode_m5_sksm_add_kicks_for( + data_master: M5DataMaster, + queue_id: u8, + priority: u8, + first_stamp: u64, + count: u16, +) -> Result<[u8; 21], M5SksmError> { + check_queue(queue_id, priority, first_stamp)?; + if count == 0 || count > 256 { + return Err(M5SksmError::AddCount); + } + let mut bytes = [0; 21]; + bytes[..4].copy_from_slice(&14u32.to_le_bytes()); + bytes[4..6].copy_from_slice(&(queue_id as u16).to_le_bytes()); + bytes[6..8].copy_from_slice(&count.to_le_bytes()); + bytes[8] = priority; + bytes[9..17].copy_from_slice(&first_stamp.to_le_bytes()); + bytes[17..21].copy_from_slice(&(data_master as u32).to_le_bytes()); + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn native_graphics_add_kicks_records_match_both_stages() { + let ta = [ + 14, 0, 0, 0, 10, 0, 1, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + let fragment = [ + 14, 0, 0, 0, 11, 0, 1, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + ]; + assert_eq!( + encode_m5_sksm_add_kicks_for(M5DataMaster::Tiling, 10, 2, 3, 1).unwrap(), + ta + ); + assert_eq!( + encode_m5_sksm_add_kicks_for(M5DataMaster::Fragment, 11, 2, 3, 1).unwrap(), + fragment + ); + assert_eq!( + encode_m5_sksm_add_kicks(11, 2, 3, 1).unwrap(), + encode_m5_sksm_add_kicks_for(M5DataMaster::Compute, 11, 2, 3, 1).unwrap() + ); + } + + fn queue() -> M5SksmQueue { + M5SksmQueue { + id: 11, + priority: 2, + current_stamp: 0x10, + submitted: 0, + previous_stamp: Some(15), + } + } + + fn operands() -> M5SksmOperands<'static> { + M5SksmOperands { + barriers: &[], + first_predecessor: 0, + implicit_dependency_type: 1, + mcache: None, + flag_bit1: false, + flag_bit61: false, + flag_bit63: false, + payload: [0xffff_fc20_c040_0000, 0xffff_fc20_c000_0840], + qos_byte_32: 3, + qos_class_40: 8, + qos_word_48: 0xffff, + event_mask: [0, 0, 8, 0], + rce_kind: 0, + rce: [M5SksmRce { address: 0, tag: 0 }; 4], + auxiliary: 0x3f_ffff_ffff_ffff, + } + } + + fn word(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) + } + + #[test] + fn dependency_types_survive_beyond_the_neo_entry_extent() { + let barriers = [M5SksmDependency { + queue_id: 7, + stamp: 21, + kind: 2, + }; 31]; + let mut op = operands(); + op.barriers = &barriers; + op.implicit_dependency_type = 3; + let p = prepare_m5_sksm_entry(queue(), op).unwrap(); + assert_eq!(word(&p.bytes, 0x20) as u32, u32::MAX); + assert_eq!(word(&p.bytes, 0x180), 0xeaaa_aaaa_aaaa_aaaa); + assert_eq!(word(&p.bytes, 0x120), (11 << 40) | 15); + assert!(p.bytes[0x188..].iter().all(|b| *b == 0)); + } + + #[test] + fn queue_generation_wrap_and_full_queue_are_separate() { + let mut q = queue(); + q.current_stamp = STAMP_MASK; + q.submitted = 255; + let p = prepare_m5_sksm_entry(q, operands()).unwrap(); + assert_eq!(p.offset, 255 * 512); + assert_eq!(p.queue_after.current_stamp, 0); + assert_eq!(p.queue_after.previous_stamp, Some(STAMP_MASK)); + assert_eq!( + prepare_m5_sksm_entry(p.queue_after, operands()), + Err(M5SksmError::QueueFull) + ); + } + + #[test] + fn invalid_fields_cannot_alias_another_queue_or_address() { + let mut q = queue(); + q.id = 128; + assert_eq!( + prepare_m5_sksm_entry(q, operands()), + Err(M5SksmError::QueueId) + ); + let mut op = operands(); + op.rce[0].address = 1 << 43; + assert_eq!( + prepare_m5_sksm_entry(queue(), op), + Err(M5SksmError::Address) + ); + op.rce[0].address = 1; + assert_eq!( + prepare_m5_sksm_entry(queue(), op), + Err(M5SksmError::Address) + ); + assert_eq!( + encode_m5_sksm_add_kicks(11, 2, 0, 0), + Err(M5SksmError::AddCount) + ); + assert_eq!( + encode_m5_sksm_add_kicks(11, 2, 1 << 40, 1), + Err(M5SksmError::Stamp) + ); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/store.rs b/drivers/gpu/drm/asahi/g18/fw/store.rs new file mode 100644 index 0000000000000..9364be13b9931 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/store.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! v91 little-endian field stores for descriptor and record encoders. +//! +//! The encoders' `copy_from_slice` field writes compile to byte stores under +//! this target's strict alignment (one STRB per byte, measured in the v90 +//! module's `encode_registers`). Most fields land in write-combined +//! firmware-visible memory, where every store is a separate transaction for +//! the combining buffers. With `g17p_host_path` bit11 (WORD_STORES) a field +//! whose address is naturally aligned for a wider access is written with the +//! widest aligned stores that exactly cover it; the bytes written are +//! identical. Unaligned fields keep byte stores. Aligned word stores are +//! single-copy atomic, which byte stores are not, so no reader can observe a +//! state the byte sequence could not also have produced. +#[cfg(not(test))] +#[inline(always)] +fn word_stores() -> bool { + crate::g18::compat::host_path::enabled(crate::g18::compat::host_path::WORD_STORES) +} +#[cfg(test)] +pub(crate) static TEST_WORD_STORES: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(true); +#[cfg(test)] +fn word_stores() -> bool { TEST_WORD_STORES.load(core::sync::atomic::Ordering::Relaxed) } + +#[inline(always)] +fn store_u16(dst: &mut [u8], value: u16) { + let at = dst.as_mut_ptr(); + if at as usize & 1 == 0 { + // SAFETY: `dst` is exactly two in-bounds bytes and `at` is 2-aligned. + unsafe { at.cast::().write(value.to_le()) } + } else { + dst.copy_from_slice(&value.to_le_bytes()); + } +} + +#[inline(always)] +fn store_u32(dst: &mut [u8], value: u32) { + let at = dst.as_mut_ptr(); + if at as usize & 3 == 0 { + // SAFETY: `dst` is exactly four in-bounds bytes and `at` is 4-aligned. + unsafe { at.cast::().write(value.to_le()) } + } else if at as usize & 1 == 0 { + let (low, high) = dst.split_at_mut(2); + store_u16(low, value as u16); + store_u16(high, (value >> 16) as u16); + } else { + dst.copy_from_slice(&value.to_le_bytes()); + } +} + +/// Store a little-endian u16 at `raw[offset..offset + 2]` (panics out of +/// bounds, like the slice copy it replaces). +#[inline(always)] +pub(crate) fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + let dst = &mut raw[offset..offset + 2]; + if word_stores() { store_u16(dst, value) } else { dst.copy_from_slice(&value.to_le_bytes()) } +} + +/// Store a little-endian u32 at `raw[offset..offset + 4]`. +#[inline(always)] +pub(crate) fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + let dst = &mut raw[offset..offset + 4]; + if word_stores() { store_u32(dst, value) } else { dst.copy_from_slice(&value.to_le_bytes()) } +} + +/// Store a little-endian u64 at `raw[offset..offset + 8]`. +#[inline(always)] +pub(crate) fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + let dst = &mut raw[offset..offset + 8]; + if !word_stores() { + dst.copy_from_slice(&value.to_le_bytes()); + return; + } + let at = dst.as_mut_ptr(); + if at as usize & 7 == 0 { + // SAFETY: `dst` is exactly eight in-bounds bytes and `at` is 8-aligned. + unsafe { at.cast::().write(value.to_le()) } + } else { + let (low, high) = dst.split_at_mut(4); + store_u32(low, value as u32); + store_u32(high, (value >> 32) as u32); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/submission.rs b/drivers/gpu/drm/asahi/g18/fw/submission.rs new file mode 100644 index 0000000000000..3099ef306fadb --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/submission.rs @@ -0,0 +1,9012 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! Byte-confirmed subset of the G17 GPU submission-transport producer ABI (the +//! "SKSM producer"). +//! +//! This module is a pure codec for the driver->firmware submission path — the +//! wire form that hands a compiled command buffer to the firmware for +//! execution. It allocates no memory, starts no ASC, sends no mailbox message, +//! and touches no MMIO. It models the EP 0x21 work-submission doorbell, the +//! channel/work-queue ring producer indices, the SKSM kick word/record, and the +//! host queue-timestamp plus `addKicks` MMIO producer. It also models the exact +//! host-side reset images for `_AGFIChannelState` and +//! `AGXUncachedFWChannelMem`, the A18 Pro SKSM hardware-bridge programming +//! equation, and its four completion-queue descriptors. The provenance ledger +//! explicitly separates byte-grounded G17 fields from inherited producer +//! records whose G17 consumer offsets still need first-hand confirmation. The +//! native G17 channel-command dispatcher is also modeled for the exact A18 +//! Pro/G17P identity: it reads a little-endian u32 tag at command offset zero, +//! keeps unresolved and nested SKU meanings out of the top-level enum, and +//! separates the host's `Nop=17` name from the firmware's fatal handling of +//! that value. +//! +//! Grounding is first-hand from the on-disk M5 role macho `GFX.a010.macho` +//! (`g17s a0, SKSM Enabled`), all three pinned A18 Pro `g17p` firmware variants +//! (`a000`, `b000`, and `b100`), and the pinned G17P/G17G/G17X host kexts. The +//! native G17P channel-state ABI is admitted only for the A18 Pro identity; +//! the M5-family identities remain unsupported. +//! Firmware addresses are VAs (0xfffffc0000000000 + fileoff - 0x4000). See +//! the G17 submission notes. +//! +//! Two-signal discipline, mirroring `g17_rtkit`/`g17_boot`: +//! [`GROUNDED_SUBMISSION_PRODUCER_AVAILABLE`] states that the *codec* is real +//! and cited; [`EXECUTABLE_SUBMISSION_TRANSPORT_AVAILABLE`] states whether a +//! submission can be driven to completion offline. It is `false`: the SKSM +//! completion path is a hardware kick FIFO the firmware reads through indexed +//! MMIO scratch registers. The host publication form, runtime scratch geometry, +//! and resource-0 ordered writer are implemented. J700 has completed the +//! channel-doorbell round trip with exact compute output across a retained CL2 +//! queue, and the measured first-partial path has executed graphics through +//! the classic channel transport. Linux still needs that contract wired into +//! its runtime and completion IRQ ingress. The native kext `addKicks` path is +//! tracked independently. See [`SUBMISSION_LIVE_GAPS`]. + +#[cfg(not(test))] +use crate::g18::compat::g17p::G17SksmScratchGeometry; +#[cfg(test)] +#[path = "g17_manager.rs"] +mod g17_manager; +#[cfg(test)] +use g17_manager::G17SksmScratchGeometry; + +/// The submission producer subset is byte-grounded against the M5 firmware. +/// See [`SUBMISSION_ABI_LEDGER`] for the inherited, still-unconfirmed pieces. +pub(crate) const GROUNDED_SUBMISSION_PRODUCER_AVAILABLE: bool = true; + +/// Whether a submission can be driven to completion. `false`: see +/// [`SUBMISSION_LIVE_GAPS`]. This keeps `SUBMISSION_TRANSPORT` out of +/// `g17_boot::pre_handoff_gate` admission until completion is implemented. +pub(crate) const EXECUTABLE_SUBMISSION_TRANSPORT_AVAILABLE: bool = + SUBMISSION_LIVE_GAPS.all_proven(); + +// --------------------------------------------------------------------------- +// 1. EP 0x21 TX work-submission doorbell — byte-identical to AGX2. +// Grounded from the firmware doorbell decoder at GFX 0xfffffc00000251cc / +// the TX case 0xfffffc0000025278 (see the research note §1). +// --------------------------------------------------------------------------- + +/// GPU doorbell endpoint (gfx_ints). Matches `gpu.rs` `EP_DOORBELL`. +pub(crate) const EP_DOORBELL: u8 = 0x21; + +/// TX (channel-kick) doorbell tag, `0x83 << 48`. Matches `gpu.rs` +/// `MSG_TX_DOORBELL`. +const MSG_TX_DOORBELL: u64 = 0x83 << 48; + +/// The firmware reads a 6-bit doorbell type from bits [53:48] +/// (`ubfx x8, x1, #48, #6` @ 0x251dc). +const DOORBELL_TYPE_SHIFT: u32 = 48; +const DOORBELL_TYPE_BITS: u32 = 6; +const DOORBELL_TYPE_MASK: u64 = (1 << DOORBELL_TYPE_BITS) - 1; + +/// `0x83 & 0x3f = 3`: the channel-kick (TX) doorbell case (`b.eq 0x25278`). +pub(crate) const DOORBELL_TYPE_TX: u8 = 3; +/// `0x84 & 0x3f = 4`: the firmware-control doorbell case (`b.eq 0x25268`). +pub(crate) const DOORBELL_TYPE_FWCTL: u8 = 4; +/// `0x85 & 0x3f = 5`: the halt doorbell case. +pub(crate) const DOORBELL_TYPE_HALT: u8 = 5; + +/// TX pipe field is `msg & 0x3` (`and w8, w1, #0x3` @ 0x25278). +const DOORBELL_PIPE_MASK: u64 = 0x3; +/// TX index field is bits [4:2] (`tst x1, #0x1c` @ 0x2527c; `index << 2`). +const DOORBELL_INDEX_SHIFT: u32 = 2; +const DOORBELL_INDEX_MASK: u64 = 0x7; + +/// Well-known TX-doorbell payloads used for firmware wakeups, matching +/// `gpu.rs` `DOORBELL_KICKFW` / `DOORBELL_DEVCTRL`. +pub(crate) const DOORBELL_KICKFW: u8 = 0x10; +pub(crate) const DOORBELL_DEVCTRL: u8 = 0x11; + +/// The three GPU submission pipes. The firmware treats `pipe_type == 3` as invalid +/// (`cmp w8, #0x3 ; b.eq 0x252f4` — no such pipe). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum PipeType { + Vertex = 0, + Fragment = 1, + Compute = 2, +} + +/// Errors from the submission codec. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum SubmissionError { + RenderBarrierFuture, + PriorBarrierFuture, + RenderBarrierSequenceOverflow, + RenderBarrierMissingToken, + RenderBarrierPredecessorFailed, + /// A doorbell index outside the 3-bit [4:2] field. + DoorbellIndexOutOfRange, + /// A doorbell whose type field is not the TX (channel-kick) type. + NotATxDoorbell, + /// A doorbell that decodes to the reserved pipe value 3. + ReservedPipe, + /// A kick word with bits set outside the modeled 47-bit form. + KickWordUnmodeledBits, + /// A queue id outside the 7-bit [46:40] field / 128-slot table. + QueueIdOutOfRange, + /// A SKSM timestamp outside the 40-bit [39:0] field. + KickTimestampOutOfRange, + /// The runtime queue geometry must contain between 1 and 256 stamps. + KickStampCountOutOfRange, + /// No SKSM queue geometry is proven for this target/backend pair. + KickQueueGeometryUnconfirmed, + /// The queue stride/count/alignment/extent/seed tuple is inconsistent. + KickQueueGeometryInvalid, + /// The timestamp's low-byte slot does not exist in this queue geometry. + KickStampIndexOutOfRange, + /// A zero-byte entry stride would alias every queue entry. + KickEntryStrideZero, + /// Multiplying the slot by the runtime entry stride overflowed u32. + KickEntryOffsetOverflow, + /// The queue already has `num_stamps` entries awaiting completion. + KickQueueSubmittedFull, + /// A completion observed no submitted entries or exceeded their count. + KickCompletionUnderflow, + /// A provider-owned pre-B2 state object is smaller than one 16 KiB page. + KickB2PrimaryStateBufferTooSmall, + /// The retained queue already published its one-time pre-B2 state. + KickB2PrimaryStateAlreadyPublished, + /// A retained queue static object is smaller than its native extent. + KickRetainedStorageBufferTooSmall, + /// A CL entry has one implicit dependency and at most 31 explicit ones. + KickDependencyCountOutOfRange, + /// A CL dependency names a queue outside the seven-bit entry field. + KickDependencyQueueIdOutOfRange, + /// The MCache aperture index or count cannot be represented in its field. + KickMcacheFieldOutOfRange, + /// The MCache aperture address is unaligned or exceeds its 38-bit field. + KickMcacheAddressUnencodable, + /// The RCE kind is outside the two-bit entry field. + KickRceKindOutOfRange, + /// An RCE tag is outside its seven-bit entry field. + KickRceTagOutOfRange, + /// An RCE address is unaligned or exceeds its 38-bit entry field. + KickRceAddressUnencodable, + /// A compute descriptor-relative RCE address overflowed u64. + ComputeRceAddressOverflow, + /// A compute event counter overflowed its 64-bit entry field. + ComputeEventCounterOverflow, + /// Required first-bind command publication facts are still unresolved. + FirstBindCommandPublicationIncomplete, + /// G17P target-1 requires both retained CPU and booted RTKit owners. + RunningClientPowerTransitionUnavailable, + /// The middle QoS field is wider than the five bits copied by the host. + KickQosClassOutOfRange, + /// The runtime MMIO aperture selector or base offset overflowed u32. + KickMmioOffsetOverflow, + /// The A18 Pro SKSM queue data-master field is limited to TA/3D/CL. + KickQueueDataMasterOutOfRange, + /// The A18 Pro SKSM completion-queue selector is outside the host range. + KickQueueCompletionSelectorOutOfRange, + /// The A18 Pro SKSM entry address cannot be represented by the host mask. + KickQueueEntryAddressUnencodable, + /// The A18 Pro completion descriptor ordinal is outside its four slots. + CompletionQueueOrdinalOutOfRange, + /// A present A18 Pro completion queue cannot have zero entries. + CompletionQueueCapacityZero, + /// A ring with zero entries cannot advance a producer index. + RingSizeZero, + /// A producer index must identify an entry inside its ring. + RingIndexOutOfRange, + /// Advancing the producer would make it collide with the consumer. + RingFull, + /// G17 exposes four priority groups in its runtime channel table. + ChannelPriorityOutOfRange, + /// This native channel layout is not grounded for the selected identity. + ChannelAbiUnconfirmed, + /// The top-level channel-command tag ABI is not grounded for this target. + CommandAbiUnconfirmed, + /// A top-level channel command must expose its complete four-byte tag. + CommandHeaderTooShort, + /// A first-render status pointer overflowed while deriving the FWCtl VAs. + PartialOpeningAddressOverflow, + /// A first-render scheduler or index mapping is not exactly one 16 KiB page. + PartialOpeningPageSize, + /// A first-render outer-channel counter has bits outside the 8-bit ring index. + PartialOpeningChannelCounterOutOfRange, + /// A first-render queue head does not fit the outer slot's 16-bit field. + PartialOpeningQueueHeadOutOfRange, + /// A cold-opening support object does not match its exact encoded extent. + PartialOpeningGraphObjectSize, + /// A first-partial descriptor cannot be represented by its context locators. + PartialOpeningDescriptorLocatorUnencodable, + /// The host/firmware meaning of this in-range top-level tag is unresolved. + CommandTagUnresolved, + /// This value is not a grounded G17P top-level command tag. + CommandTagInvalid, + /// The host names this tag, but the matched firmware handles it fatally. + CommandTagInvalidForFirmware, + /// A tag-14 AddKicks command must publish at least one new kick entry. + KsmAddKicksCountZero, + /// The tag-14 AddKicks count does not fit the record's 16-bit field. + KsmAddKicksCountOutOfRange, +} + +/// Encode a work-submission doorbell exactly as `gpu.rs::run_job` does: +/// `MSG_TX_DOORBELL | pipe_type | (index << 2)`. +/// +/// `index` is the queue priority (0..=7 fits the firmware's [4:2] field). +pub(crate) const fn encode_tx_doorbell(pipe: PipeType, index: u8) -> Result { + if index as u64 > DOORBELL_INDEX_MASK { + return Err(SubmissionError::DoorbellIndexOutOfRange); + } + Ok(MSG_TX_DOORBELL | (pipe as u64) | ((index as u64) << DOORBELL_INDEX_SHIFT)) +} + +/// Host state which decides the two distinct EP 0x21 notifications around a +/// direct G17P CL submit. Activation is first-or-changed-channel state; the +/// direct kick has its own feature, suppression, and command-enable gates. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PDirectClNotificationState { + pub(crate) cached_channel_identity: u64, + pub(crate) submitted_channel_identity: u64, + pub(crate) firmware_ready: bool, + pub(crate) direct_feature: bool, + pub(crate) submission_suppressed: bool, + pub(crate) command_submission_enabled: bool, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PDirectClNotifications { + pub(crate) activation: Option, + pub(crate) direct_kick: Option, +} + +/// Submission backend identity. This keeps the T8140 G17P HAL200 hybrid +/// distinct from both classic channels and the unimplemented HAL300 family +/// without changing mature generic paths. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17ClSubmissionBackend { + Classic, + G17PHal200Hybrid, + Hal300, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClBackendConfig { + pub(crate) backend: G17ClSubmissionBackend, + pub(crate) direct_feature: bool, + pub(crate) submission_suppressed: bool, + pub(crate) command_submission_enabled: bool, +} + +pub(crate) const T8140_G17P_CL_BACKEND: G17PClBackendConfig = G17PClBackendConfig { + backend: G17ClSubmissionBackend::G17PHal200Hybrid, + direct_feature: true, + submission_suppressed: false, + command_submission_enabled: true, +}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PContractPredicateState { + Pending, + ProvenPresent, +} + +/// Dedicated FlushID classification is not sufficient for release: an absent +/// feature needs a proven represented fallback, while a present feature needs +/// its numeric allocator, lifetime, and command operands implemented. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PFlushIdImplementationState { + Pending, + ProvenAbsentFallbackUnverified, + ProvenAbsentFallbackRepresented, + ProvenPresentUnimplemented, + ProvenPresentImplemented, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PCorrelationRingImplementationState { + Pending, + ProvenInactive, + ProvenRequiredImplemented, +} + +/// Required first-bind commands and publication facts. FlushID uses represented +/// fallback zero, the command-pointer ring is implemented, and the taken +/// correlation branch is represented by a host-only nonzero job token. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PFirstBindCommandPlan { + pub(crate) config_update_required: bool, + pub(crate) entry_signal_required: bool, + pub(crate) logical_flush_id_required: bool, + pub(crate) fallback_logical_flush_id: u32, + pub(crate) flush_id_serialized_in_tag15: bool, + pub(crate) flush_id_completion_release_required: bool, + pub(crate) dedicated_flush_id: G17PFlushIdImplementationState, + pub(crate) native_command_pointer_ring: G17PContractPredicateState, + pub(crate) optional_correlation_ring: G17PCorrelationRingImplementationState, +} + +impl G17PFirstBindCommandPlan { + pub(crate) const fn hardware_release_ready(self) -> bool { + self.config_update_required + && self.entry_signal_required + && self.logical_flush_id_required + && self.fallback_logical_flush_id == 0 + && !self.flush_id_serialized_in_tag15 + && !self.flush_id_completion_release_required + && matches!( + self.dedicated_flush_id, + G17PFlushIdImplementationState::ProvenAbsentFallbackRepresented + | G17PFlushIdImplementationState::ProvenPresentImplemented + ) + && matches!( + self.native_command_pointer_ring, + G17PContractPredicateState::ProvenPresent + ) + && matches!( + self.optional_correlation_ring, + G17PCorrelationRingImplementationState::ProvenInactive + | G17PCorrelationRingImplementationState::ProvenRequiredImplemented + ) + } +} + +pub(crate) const G17P_FIRST_BIND_COMMAND_PLAN: G17PFirstBindCommandPlan = + G17PFirstBindCommandPlan { + config_update_required: true, + entry_signal_required: true, + logical_flush_id_required: true, + fallback_logical_flush_id: 0, + flush_id_serialized_in_tag15: false, + flush_id_completion_release_required: false, + dedicated_flush_id: + G17PFlushIdImplementationState::ProvenAbsentFallbackRepresented, + native_command_pointer_ring: G17PContractPredicateState::ProvenPresent, + optional_correlation_ring: + G17PCorrelationRingImplementationState::ProvenRequiredImplemented, + }; + +/// Compile-time release gate consumed by the live first-add3 boundary. Keep +/// true only because the host-only correlation token, tag-15/tag-16 pointer +/// publisher, and FlushID fallback are all represented for this profile. +pub(crate) const G17P_FIRST_BIND_DIRECT_HARDWARE_RELEASE_READY: bool = + G17P_FIRST_BIND_COMMAND_PLAN.hardware_release_ready(); + +pub(crate) const fn require_g17p_first_bind_direct_hardware_release( + plan: G17PFirstBindCommandPlan, +) -> Result<(), SubmissionError> { + if plan.hardware_release_ready() { + Ok(()) + } else { + Err(SubmissionError::FirstBindCommandPublicationIncomplete) + } +} + +/// Native target-bound active state required before TA/3D/CL submission. +/// +/// D93 role 1 is managed and non-standalone. Linux retains its started CPU, +/// mailbox, and booted RTKit owners for the complete session, which is the +/// native target-1 state. Submit performs no management power request. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PRunningClientPowerContract { + pub(crate) role_index: u8, + pub(crate) service_id: u8, + pub(crate) target: u8, + pub(crate) management_request_used: bool, + pub(crate) managed: bool, + pub(crate) standalone: bool, +} + +pub(crate) const G17P_RUNNING_CLIENT_POWER_CONTRACT: G17PRunningClientPowerContract = + G17PRunningClientPowerContract { + role_index: 1, + service_id: 0x20, + target: 1, + management_request_used: false, + managed: true, + standalone: false, + }; + +pub(crate) const fn require_g17p_running_client_power_transition( + contract: G17PRunningClientPowerContract, + retained_cpu_owner: bool, + retained_rtkit_owner: bool, +) -> Result<(), SubmissionError> { + if contract.role_index == 1 + && contract.service_id == 0x20 + && contract.target == 1 + && !contract.management_request_used + && contract.managed + && !contract.standalone + && retained_cpu_owner + && retained_rtkit_owner + { + Ok(()) + } else { + Err(SubmissionError::RunningClientPowerTransitionUnavailable) + } +} + +/// Required lifetime of the eventual target-1 active lease. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PComputePowerLeaseStep { + AcquireBeforeReadiness, + HoldThroughClaim, + HoldThroughCompletion, + ReleaseAfterCompletion, +} + +pub(crate) const G17P_COMPUTE_POWER_LEASE_ORDER: [G17PComputePowerLeaseStep; 4] = [ + G17PComputePowerLeaseStep::AcquireBeforeReadiness, + G17PComputePowerLeaseStep::HoldThroughClaim, + G17PComputePowerLeaseStep::HoldThroughCompletion, + G17PComputePowerLeaseStep::ReleaseAfterCompletion, +]; + +pub(crate) const fn g17p_first_bind_tag15_required(queue_config_dirty: bool) -> bool { + queue_config_dirty +} + +pub(crate) const fn g17p_first_bind_tag16_required( + backend: G17PClBackendConfig, + queue_config_dirty: bool, +) -> bool { + matches!(backend.backend, G17ClSubmissionBackend::G17PHal200Hybrid) + && backend.direct_feature + && queue_config_dirty +} + +/// Host-only first/repeat state for the direct CL submission contract. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PDirectClLifecycleState { + pub(crate) queue_config_dirty: bool, + pub(crate) cached_channel_identity: u64, + /// Nonzero host-only descriptor sequence used for job ownership/fencing. + pub(crate) next_correlation_token: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PDirectClLifecycle { + pub(crate) publish_config_update: bool, + pub(crate) publish_entry_signal: bool, + pub(crate) correlation_token: u64, + pub(crate) notifications: PreparedG17PDirectClNotifications, + pub(crate) next: G17PDirectClLifecycleState, +} + +/// Derive first-bind and repeat transitions without selecting publication +/// storage. First bind publishes tags 15/16; a clean repeat omits both and the +/// unchanged channel identity omits activation, while direct kick remains. +pub(crate) const fn prepare_g17p_direct_cl_lifecycle( + current: G17PDirectClLifecycleState, + submitted_channel_identity: u64, + firmware_ready: bool, + backend: G17PClBackendConfig, +) -> Result { + if current.next_correlation_token == 0 { + return Err(SubmissionError::ComputeEventCounterOverflow); + } + let next_correlation_token = match current.next_correlation_token.checked_add(1) { + Some(token) => token, + None => return Err(SubmissionError::ComputeEventCounterOverflow), + }; + let publish_config_update = g17p_first_bind_tag15_required(current.queue_config_dirty); + let publish_entry_signal = + g17p_first_bind_tag16_required(backend, current.queue_config_dirty); + let notifications = match prepare_g17p_direct_cl_notifications( + G17PDirectClNotificationState { + cached_channel_identity: current.cached_channel_identity, + submitted_channel_identity, + firmware_ready, + direct_feature: backend.direct_feature, + submission_suppressed: backend.submission_suppressed, + command_submission_enabled: backend.command_submission_enabled, + }, + ) { + Ok(notifications) => notifications, + Err(error) => return Err(error), + }; + Ok(PreparedG17PDirectClLifecycle { + publish_config_update, + publish_entry_signal, + correlation_token: current.next_correlation_token, + notifications, + next: G17PDirectClLifecycleState { + queue_config_dirty: false, + cached_channel_identity: submitted_channel_identity, + next_correlation_token, + }, + }) +} + +/// First/new compute-channel activation. Priority 2 encodes as `0x...0a`; +/// this is independent of SKSM QID 4 and outer work-channel index 10. +pub(crate) const fn encode_g17p_cl_activation(priority: u8) -> Result { + encode_tx_doorbell(PipeType::Compute, priority) +} + +/// Direct CL `kickFirmware(4, 3, false)`, byte-proven as EP 0x21 `0x...10`. +pub(crate) const fn encode_g17p_direct_cl_kick() -> u64 { + MSG_TX_DOORBELL | DOORBELL_KICKFW as u64 +} + +/// Exact J700 native EP21 lifecycle around firmware startup and the first real +/// userspace CL. Startup publishes no work or record-less device-control +/// announcement; the first changed priority-2 CL emits activation then the +/// direct kick. +pub(crate) const G17P_NATIVE_PRE_USER_EP21_MESSAGES: [u64; 0] = []; +pub(crate) const G17P_NATIVE_FIRST_USER_EP21_MESSAGES: [u64; 2] = [ + 0x0083_0000_0000_000a, + 0x0083_0000_0000_0010, +]; + +/// Pure predicate/encoding boundary. A repeat submit omits activation when the +/// cached identity already matches, but an enabled, direct, unsuppressed submit +/// still emits the separate direct kick. +pub(crate) const fn prepare_g17p_direct_cl_notifications( + state: G17PDirectClNotificationState, +) -> Result { + let activation = if state.cached_channel_identity != state.submitted_channel_identity + && state.firmware_ready + { + match encode_g17p_cl_activation(2) { + Ok(message) => Some(message), + Err(error) => return Err(error), + } + } else { + None + }; + let direct_kick = if state.direct_feature + && !state.submission_suppressed + && state.command_submission_enabled + { + Some(encode_g17p_direct_cl_kick()) + } else { + None + }; + Ok(PreparedG17PDirectClNotifications { + activation, + direct_kick, + }) +} + +/// Extract the firmware's 6-bit doorbell type from bits [53:48]. +pub(crate) const fn doorbell_type(message: u64) -> u8 { + ((message >> DOORBELL_TYPE_SHIFT) & DOORBELL_TYPE_MASK) as u8 +} + +/// Decode a TX doorbell to `(pipe_type, index)`, applying the firmware's exact +/// masks and its invalid `pipe == 3` outcome. +pub(crate) const fn decode_tx_doorbell(message: u64) -> Result<(PipeType, u8), SubmissionError> { + if doorbell_type(message) != DOORBELL_TYPE_TX { + return Err(SubmissionError::NotATxDoorbell); + } + let pipe = match message & DOORBELL_PIPE_MASK { + 0 => PipeType::Vertex, + 1 => PipeType::Fragment, + 2 => PipeType::Compute, + _ => return Err(SubmissionError::ReservedPipe), + }; + let index = ((message >> DOORBELL_INDEX_SHIFT) & DOORBELL_INDEX_MASK) as u8; + Ok((pipe, index)) +} + +/// The firmware's pending-pipe bit for a submitted pipe: `1 << pipe_type` +/// (`mov w9,#1 ; lsl w8,w9,w8` in the pinned J700 A000 type-3 handler). This +/// path is distinct from the type-4 device-control bit 17 at scheduler root +/// `0xfffffc0000177190`. +pub(crate) const fn pending_pipe_bit(pipe: PipeType) -> u32 { + 1u32 << (pipe as u32) +} + +// --------------------------------------------------------------------------- +// 2. Legacy AGX2 channel state. These constants are retained only to make the +// inherited producer explicit; they are not a native-G17 admission. +// --------------------------------------------------------------------------- + +/// AGX2 FW-owned consumer index. +pub(crate) const AGX2_CHANNEL_STATE_READ_PTR_OFFSET: u32 = 0x00; +/// AGX2 driver-owned producer index. +pub(crate) const AGX2_CHANNEL_STATE_WRITE_PTR_OFFSET: u32 = 0x20; +/// Total AGX2 `ChannelState` size. +pub(crate) const AGX2_CHANNEL_STATE_SIZE: u32 = 0x40; + +// --------------------------------------------------------------------------- +// 2a. G17 runtime channel descriptors -- native AGX3 layout. +// +// Grounded first-hand from the G17S firmware recovery scanner at GFX +// 0xfffffc000000ff8c-0xfffffc00000103dc. runtime_pointers+0x20 holds four +// priority groups of three descriptors (vertex, fragment, compute). Each +// descriptor is 0x20 bytes: the firmware dereferences cursor pointers at +// +0x00/+0x08/+0x10 and obtains the command-entry array from +0x18. It +// advances through that array in 0x18-byte records. The final descriptor +// ends at runtime_pointers+0x1a0, immediately before the independent cursor +// pair at +0x1a0/+0x1a8. +// --------------------------------------------------------------------------- + +/// First G17 channel descriptor in `runtime_pointers`. +pub(crate) const G17_CHANNEL_DESCRIPTOR_TABLE_OFFSET: u32 = 0x20; +/// Vertex, fragment, and compute descriptors per priority group. +pub(crate) const G17_CHANNEL_DESCRIPTORS_PER_PRIORITY: u32 = 3; +/// Priority groups scanned by the G17 firmware. +pub(crate) const G17_CHANNEL_PRIORITY_COUNT: u8 = 4; +/// Total descriptor count: four priorities times three submission pipes. +pub(crate) const G17_CHANNEL_DESCRIPTOR_COUNT: u32 = 12; +/// Native G17 descriptor stride. AGX2 `ChannelRing` is only 0x10 bytes. +pub(crate) const G17_CHANNEL_DESCRIPTOR_STRIDE: u32 = 0x20; +/// G17 command-entry stride used by all three pipe rings. +pub(crate) const G17_CHANNEL_ENTRY_STRIDE: u32 = 0x18; + +/// Pointer fields inside one native G17 channel descriptor. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17ChannelDescriptorLayout { + /// Pointer to the cursor advanced after an entry has been dispatched. + pub(crate) dispatch_cursor: u32, + /// Pointer to the cursor advanced after the entry range is made coherent. + pub(crate) coherency_cursor: u32, + /// Pointer to the producer cursor sampled before scanning the ring. + pub(crate) producer_cursor: u32, + /// GPU pointer to the first 0x18-byte command entry. + pub(crate) entries: u32, +} + +pub(crate) const G17_CHANNEL_DESCRIPTOR_LAYOUT: G17ChannelDescriptorLayout = + G17ChannelDescriptorLayout { + dispatch_cursor: 0x00, + coherency_cursor: 0x08, + producer_cursor: 0x10, + entries: 0x18, + }; + +/// Return the runtime-pointer byte offset of a native G17 pipe descriptor. +/// +/// The table is priority-major and pipe-minor. Refuse priority values outside +/// the four groups that the firmware scans instead of aliasing the independent +/// cursor pair at `runtime_pointers+0x1a0`. +pub(crate) const fn g17_channel_descriptor_offset( + priority: u8, + pipe: PipeType, +) -> Result { + if priority >= G17_CHANNEL_PRIORITY_COUNT { + return Err(SubmissionError::ChannelPriorityOutOfRange); + } + + let index = priority as u32 * G17_CHANNEL_DESCRIPTORS_PER_PRIORITY + pipe as u32; + Ok(G17_CHANNEL_DESCRIPTOR_TABLE_OFFSET + index * G17_CHANNEL_DESCRIPTOR_STRIDE) +} + +// --------------------------------------------------------------------------- +// 2b. G17 host channel-reset images -- native AGX3 layout. +// +// Grounded first-hand from byte-identical `AGXChannel::resetChannelState` +// implementations in the pinned G17P host at 0xfffffe00089f2a60, G17G at +// 0xfffffe0008b225ac, and G17X at 0xfffffe0008ae6a6c. The host clears all +// 0x80 bytes of `_AGFIChannelState`, then writes the fields below. Its +// separate `AGXUncachedFWChannelMem` reset touches only seven u32 fields +// through +0x63; bytes between them are deliberately preserved here. +// +// These are independent host-reset records. They do not prove that the +// inherited AGX2 ChannelState/RingState names in sections 2/3 describe the +// G17 firmware consumer, so those ledger entries remain unsupported. +// --------------------------------------------------------------------------- + +/// Full byte extent cleared by the G17 host for `_AGFIChannelState`. +pub(crate) const G17_HOST_CHANNEL_STATE_RESET_SIZE: usize = 0x80; +/// Last byte touched by the G17 host's uncached-channel reset stores. +pub(crate) const G17_UNCACHED_CHANNEL_RESET_WRITE_EXTENT: usize = 0x64; +/// The older pinned host builds share this reset layout. The live M5 25G76 +/// driver instead resets 0xc0 bytes; see `encode_m5_25g76_channel_state_reset`. +pub(crate) const G17_HOST_CHANNEL_RESET_TARGETS_CONFIRMED: bool = true; + +/// Offsets written after the host clears `_AGFIChannelState`. +/// +/// Names are intentionally structural until the firmware consumer supplies +/// first-hand semantics for every field. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17HostChannelStateResetLayout { + pub(crate) gpu_va_00: usize, + pub(crate) gpu_va_08: usize, + pub(crate) gpu_va_10: usize, + pub(crate) zero_18: usize, + pub(crate) zero_1c: usize, + pub(crate) zero_20: usize, + pub(crate) sentinel_24: usize, + pub(crate) fixed_4_28: usize, + pub(crate) sentinel_40: usize, + pub(crate) word_44: usize, + pub(crate) zero_58: usize, + pub(crate) gpu_va_70: usize, +} + +pub(crate) const G17_HOST_CHANNEL_STATE_RESET_LAYOUT: G17HostChannelStateResetLayout = + G17HostChannelStateResetLayout { + gpu_va_00: 0x00, + gpu_va_08: 0x08, + gpu_va_10: 0x10, + zero_18: 0x18, + zero_1c: 0x1c, + zero_20: 0x20, + sentinel_24: 0x24, + fixed_4_28: 0x28, + sentinel_40: 0x40, + word_44: 0x44, + zero_58: 0x58, + gpu_va_70: 0x70, + }; + +/// Runtime values copied by the host into the structural G17 reset image. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17HostChannelStateResetValues { + pub(crate) gpu_va_00: u64, + pub(crate) gpu_va_08: u64, + pub(crate) gpu_va_10: u64, + pub(crate) word_44: u32, + pub(crate) gpu_va_70: u64, +} + +/// Fields touched in `AGXUncachedFWChannelMem` during a G17 channel reset. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17UncachedChannelResetLayout { + pub(crate) zero_00: usize, + pub(crate) zero_10: usize, + pub(crate) zero_20: usize, + pub(crate) zero_30: usize, + pub(crate) zero_40: usize, + pub(crate) sentinel_50: usize, + pub(crate) ring_entry_count_60: usize, +} + +pub(crate) const G17_UNCACHED_CHANNEL_RESET_LAYOUT: G17UncachedChannelResetLayout = + G17UncachedChannelResetLayout { + zero_00: 0x00, + zero_10: 0x10, + zero_20: 0x20, + zero_30: 0x30, + zero_40: 0x40, + sentinel_50: 0x50, + ring_entry_count_60: 0x60, + }; + +fn put_u32_le(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u16_le(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64_le(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Build the exact 0x80-byte `_AGFIChannelState` reset image. +pub(crate) fn encode_g17_host_channel_state_reset( + values: G17HostChannelStateResetValues, +) -> [u8; G17_HOST_CHANNEL_STATE_RESET_SIZE] { + let mut raw = [0u8; G17_HOST_CHANNEL_STATE_RESET_SIZE]; + let layout = G17_HOST_CHANNEL_STATE_RESET_LAYOUT; + + put_u64_le(&mut raw, layout.gpu_va_00, values.gpu_va_00); + put_u64_le(&mut raw, layout.gpu_va_08, values.gpu_va_08); + put_u64_le(&mut raw, layout.gpu_va_10, values.gpu_va_10); + put_u32_le(&mut raw, layout.zero_18, 0); + put_u32_le(&mut raw, layout.zero_1c, 0); + put_u32_le(&mut raw, layout.zero_20, 0); + put_u32_le(&mut raw, layout.sentinel_24, u32::MAX); + put_u32_le(&mut raw, layout.fixed_4_28, 4); + put_u32_le(&mut raw, layout.sentinel_40, u32::MAX); + put_u32_le(&mut raw, layout.word_44, values.word_44); + put_u32_le(&mut raw, layout.zero_58, 0); + put_u64_le(&mut raw, layout.gpu_va_70, values.gpu_va_70); + raw +} + +/// M5 G17G / macOS 25G76 resets six pairs of 128-bit vectors, covering 0xc0 +/// bytes. Its firmware consumes through +0xa7. The common 0x80-byte reset +/// would omit the moved fields and the firmware-visible tail. +pub(crate) const M5_25G76_CHANNEL_STATE_RESET_SIZE: usize = 0xc0; +pub(crate) const M5_25G76_CHANNEL_FIRMWARE_READ_EXTENT: usize = 0xa8; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ChannelStateResetValues { + pub(crate) uncached_state: u64, + pub(crate) command_pointer_ring: u64, + pub(crate) gpu_va_10: u64, + pub(crate) word_48: u32, + /// Opaque host field +0xf8 copied to the unaligned firmware field +0x9c. + pub(crate) gpu_va_9c: u64, +} + +/// Exact reset stores from M5 AGXChannel::resetChannelState at +/// 0xfffffe000897d36c..0xfffffe000897d410 (AGXG17G 353.14, 25G76). +/// Uncached stores are separately shared with apply_g17_uncached_channel_reset. +/// This codec alone does not admit M5 submission or a Linux GPU runtime. +pub(crate) fn encode_m5_25g76_channel_state_reset( + values: M5ChannelStateResetValues, +) -> [u8; M5_25G76_CHANNEL_STATE_RESET_SIZE] { + let mut raw = [0u8; M5_25G76_CHANNEL_STATE_RESET_SIZE]; + put_u64_le(&mut raw, 0x00, values.uncached_state); + put_u64_le(&mut raw, 0x08, values.command_pointer_ring); + put_u64_le(&mut raw, 0x10, values.gpu_va_10); + put_u32_le(&mut raw, 0x24, u32::MAX); + put_u32_le(&mut raw, 0x28, 4); + put_u32_le(&mut raw, 0x44, u32::MAX); + put_u32_le(&mut raw, 0x48, values.word_48); + put_u64_le(&mut raw, 0x9c, values.gpu_va_9c); + raw +} + +/// Apply only the seven uncached reset stores made by the G17 host. +/// +/// The host does not clear the intervening bytes in `resetChannelState`; a +/// caller-provided image therefore stays intact outside the proven stores. +pub(crate) fn apply_g17_uncached_channel_reset( + raw: &mut [u8; G17_UNCACHED_CHANNEL_RESET_WRITE_EXTENT], + ring_entry_count: u32, +) { + let layout = G17_UNCACHED_CHANNEL_RESET_LAYOUT; + + put_u32_le(raw, layout.zero_00, 0); + put_u32_le(raw, layout.zero_10, 0); + put_u32_le(raw, layout.zero_20, 0); + put_u32_le(raw, layout.zero_30, 0); + put_u32_le(raw, layout.zero_40, 0); + put_u32_le(raw, layout.sentinel_50, u32::MAX); + put_u32_le(raw, layout.ring_entry_count_60, ring_entry_count); +} + +// --------------------------------------------------------------------------- +// 2c. A18 Pro / G17P native channel state. +// +// This is deliberately identity-scoped. The producer is the pinned G17P +// host `AGXChannel::writeChannelCommandPointer` at +// 0xfffffe00089f5a88-0xfffffe00089f5b58. The matching firmware consumers +// are `armfw_g17p.a000` 0xfffffc0000012388-0xfffffc00000124f4 and the +// resume path 0xfffffc0000012514-0xfffffc0000012690. The matching b000 +// and b100 channel blocks are byte-identical to each other and implement +// the same accesses at 0xfffffc00000122f4-0xfffffc0000012460 and +// 0xfffffc00000124a4-0xfffffc0000012664. +// +// `_AGFIChannelState` is the cached indirection record: +0x00 points to +// the uncached state, +0x08 points to the u64 command-pointer ring, and +// +0x1c holds the cached coherency cursor. `AGXUncachedFWChannelMem` holds +// u32 cursors at +0x00/+0x30/+0x40/+0x50 and a u32 ring entry count at +// +0x60. This is not AGX2's read@0/write@0x20 `ChannelState`. +// --------------------------------------------------------------------------- + +/// Silicon/host identity used to admit native G17 submission ABIs. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17ChannelAbiTarget { + /// A18 Pro: G17P host and `g17p` firmware. + A18ProG17P, + /// M5: G17G host. No native channel consumer is admitted by this pass. + M5G17G, + /// M5 Pro/Max: G17X host with `g17s` firmware. Kept separate from G17P. + M5ProMaxG17X, +} + +/// Cached `_AGFIChannelState` fields consumed by the A18 Pro firmware. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PCachedChannelStateLayout { + pub(crate) uncached_state_gpu_va: u32, + pub(crate) command_pointer_ring_gpu_va: u32, + pub(crate) coherency_cursor: u32, + pub(crate) record_size: u32, +} + +/// Uncached channel fields shared by the A18 Pro host and firmware. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PUncachedChannelStateLayout { + /// Firmware dispatch/consumer cursor. + pub(crate) consumer: u32, + /// Cursor published after the command-pointer range is made coherent. + pub(crate) coherency_shadow: u32, + /// Host producer cursor, published after a `dmb ish`. + pub(crate) producer: u32, + /// Recovery target selected instead of the accelerator-entry snapshot. + pub(crate) recovery_target: u32, + /// Runtime ring depth in 8-byte command-pointer entries. + pub(crate) ring_entry_count: u32, + pub(crate) cursor_width: u32, + pub(crate) write_extent: u32, +} + +/// Complete admitted A18 Pro channel layout. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PChannelAbiLayout { + pub(crate) cached: G17PCachedChannelStateLayout, + pub(crate) uncached: G17PUncachedChannelStateLayout, + pub(crate) command_pointer_entry_bytes: u32, + pub(crate) accelerator_target_width: u32, +} + +pub(crate) const G17P_CHANNEL_ABI: G17PChannelAbiLayout = G17PChannelAbiLayout { + cached: G17PCachedChannelStateLayout { + uncached_state_gpu_va: 0x00, + command_pointer_ring_gpu_va: 0x08, + coherency_cursor: 0x1c, + record_size: 0x80, + }, + uncached: G17PUncachedChannelStateLayout { + consumer: 0x00, + coherency_shadow: 0x30, + producer: 0x40, + recovery_target: 0x50, + ring_entry_count: 0x60, + cursor_width: 4, + write_extent: 0x64, + }, + command_pointer_entry_bytes: 8, + accelerator_target_width: 2, +}; + +/// The host executes `dmb ish` after storing the u64 command pointer and +/// before publishing the next u32 producer cursor. +pub(crate) const G17P_CHANNEL_PRODUCER_REQUIRES_DMB_ISH: bool = true; + +/// Return the native channel ABI only for the exact identity whose producer +/// and consumer were both inspected. M5-family targets remain unsupported. +pub(crate) const fn g17_channel_abi( + target: G17ChannelAbiTarget, +) -> Result { + match target { + G17ChannelAbiTarget::A18ProG17P => Ok(G17P_CHANNEL_ABI), + G17ChannelAbiTarget::M5G17G | G17ChannelAbiTarget::M5ProMaxG17X => { + Err(SubmissionError::ChannelAbiUnconfirmed) + } + } +} + +/// A host write plan for one G17P command-pointer ring publication. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PChannelWritePlan { + /// Byte offset of the u64 slot written before the barrier. + pub(crate) entry_byte_offset: u64, + /// u32 producer value published after the barrier. + pub(crate) next_producer: u32, +} + +/// Validate and calculate the write performed by the G17P host producer. +/// +/// This pure helper does not execute the required barrier; callers must store +/// the u64 command GPU VA, execute `dmb ish`, then store `next_producer` to +/// uncached offset +0x40. +pub(crate) const fn prepare_g17p_channel_write( + target: G17ChannelAbiTarget, + producer: u32, + consumer: u32, + ring_entry_count: u32, +) -> Result { + match g17_channel_abi(target) { + Ok(_) => {} + Err(error) => return Err(error), + } + + if consumer >= ring_entry_count { + return Err(if ring_entry_count == 0 { + SubmissionError::RingSizeZero + } else { + SubmissionError::RingIndexOutOfRange + }); + } + + let next_producer = match ring_next(producer, ring_entry_count) { + Ok(next) => next, + Err(error) => return Err(error), + }; + if next_producer == consumer { + return Err(SubmissionError::RingFull); + } + + Ok(G17PChannelWritePlan { + entry_byte_offset: producer as u64 * G17P_CHANNEL_ABI.command_pointer_entry_bytes as u64, + next_producer, + }) +} + +// --------------------------------------------------------------------------- +// 2d. Proven G17P first-partial graphics publication. +// +// The measured J700 first-partial source contract executes graphics through +// the native partial graph. It publishes a one-record scheduler page and +// an eight-group primary index, presents five primary status fields before +// control 0x84, then exposes 3D before TA and sends one EP 0x21/0x83 kick +// for transport pair 0 on channel 0. This is the classic channel path and +// remains separate from the kext's SKSM queue-entry plus `addKicks` path. +// --------------------------------------------------------------------------- + +pub(crate) const G17P_PARTIAL_OPENING_PAGE_SIZE: usize = 0x4000; +/// Firmware-high zero page consumed at +0xe00 after the two instances exchange +/// their first peer messages. It is not named by an initdata descriptor, but +/// its fixed mapping is part of the native first-partial topology. +pub(crate) const G17P_PARTIAL_OPENING_COMPUTED_GPU_VA: u64 = 0xffff_fc20_015d_8000; +pub(crate) const G17P_PARTIAL_OPENING_SCHEDULER_GPU_VA: u64 = 0xffff_fc20_015e_0000; +pub(crate) const G17P_PARTIAL_OPENING_SCHEDULER_WORD: u64 = 0x0000_0020_0001_9000; +/// Firmware-high owner of the four-page primary-index run in the clean +/// first-partial capture. The render TTB0 aliases this at +/// [`G17P_PARTIAL_OPENING_PRIMARY_INDEX_GPU_VA`]. +pub(crate) const G17P_PARTIAL_OPENING_PRIMARY_INDEX_FIRMWARE_GPU_VA: u64 = + 0xffff_fc20_c084_8000; +pub(crate) const G17P_PARTIAL_OPENING_PRIMARY_INDEX_GPU_VA: u64 = 0x0000_0010_0019_0000; +pub(crate) const G17P_PARTIAL_OPENING_PRIMARY_INDEX_SIZE: usize = + 4 * G17P_PARTIAL_OPENING_PAGE_SIZE; +pub(crate) const G17P_PARTIAL_OPENING_FWCTL_OFFSET: u64 = 0x001c_0000; +pub(crate) const G17P_PARTIAL_OPENING_CONTROL_MESSAGE_SIZE: u64 = 0x40; +/// Legacy Linux render work-submission doorbell channel. +/// +/// `encode_tx_doorbell` (this file) gives the layout as +/// `MSG_TX_DOORBELL | pipe_type | (index << 2)`, `index` being the queue +/// priority. The three actual GPU pipes are: +/// +/// TA = Vertex(0) +/// 3D = Fragment(1) +/// compute = Compute(2) +/// +/// Linux historically sent only Vertex/priority 0 after publishing both render +/// queues. A gate probe proved that message wakes the cores, not that it +/// dispatches both pipes. Mac17P `AGXArmFirmware::submit3DChannel` at +/// `0x8a43aa0` sends Fragment first (tail `0x8a43c6c`), then +/// `submitTAChannel` at `0x8a43cdc` sends Vertex (tail `0x8a43e10`). The +/// byte-complete boot trace confirms ordinary userspace queues use priority 2; +/// the earlier priority-0 pairs are a distinct system bring-up class. +pub(crate) const G17P_PARTIAL_OPENING_WORK_DOORBELL_CHANNEL: u8 = 0; +pub(crate) const G17P_PARTIAL_OPENING_PAIRED_DOORBELL: u64 = + MSG_TX_DOORBELL | G17P_PARTIAL_OPENING_WORK_DOORBELL_CHANNEL as u64; +// Default-off build diagnostic: KRUSTFLAGS="--cfg g17p_render_priority_probe". +// Native ordinary user queues captured priority/subpriority 2/2. This does +// not establish a Linux UAPI mapping or make the profile a production fix. +pub(crate) const G17P_NATIVE_RENDER_DOORBELL_PRIORITY: u8 = + if cfg!(g17p_render_priority_probe) { 2 } else { 0 }; +// HardwareBufferID is independently allocated by AGXHardwareBufferIDManager, +// not derived from priority (native captures have priority 2 with IDs 5/6). +// Preserve the existing Linux identity for this diagnostic. +pub(crate) const G17P_NATIVE_RENDER_QOS_HARDWARE_BUFFER_ID: u8 = 0; +pub(crate) const G17P_NATIVE_RENDER_POLICY: u16 = + if G17P_NATIVE_RENDER_DOORBELL_PRIORITY == 0 { 1 } else { 2 }; +/// Mac17P submits 3D first: Fragment(1) | (priority 2 << 2) = 0x09. +pub(crate) const G17P_NATIVE_3D_DOORBELL: u64 = MSG_TX_DOORBELL + | PipeType::Fragment as u64 + | ((G17P_NATIVE_RENDER_DOORBELL_PRIORITY as u64) << DOORBELL_INDEX_SHIFT); +/// Mac17P then submits TA: Vertex(0) | (priority 2 << 2) = 0x08. +pub(crate) const G17P_NATIVE_TA_DOORBELL: u64 = MSG_TX_DOORBELL + | PipeType::Vertex as u64 + | ((G17P_NATIVE_RENDER_DOORBELL_PRIORITY as u64) << DOORBELL_INDEX_SHIFT); +pub(crate) const G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE: usize = 0x18; +pub(crate) const G17P_PARTIAL_OPENING_OUTER_SLOT_COUNT: u32 = 0x100; +/// Work-channel table index is `priority * 3 + pipe`, with pipe Vertex(TA)=0, +/// Fragment(3D)=1, Compute(CL)=2. That is exactly the grouping +/// [`g17_initdata::WORK_STATE_GRID_OFFSETS`] documents: indices 0,1,2 are +/// TA_0/3D_0/CL_0 and indices 6,7,8 are TA_2/3D_2/CL_2. +/// +/// The render must publish into the rings for the priority it actually +/// configures and announces. This profile follows native's initial priority-0 +/// qid-0/1 pair consistently through table selection, queue records, tags, +/// QoS and the paired doorbell. Later priority-2 userspace queues use indices +/// 6/7 and must likewise remain internally coherent. +/// +/// Compute is the worked example of the rule: index 8 is CL at priority 2, it +/// agrees with its own priority-2 doorbell 0x0a, and it is validated by 2000 +/// consecutive passing computes. +const fn g17p_work_channel_table_index(priority: u8, pipe: PipeType) -> u8 { + priority * 3 + pipe as u8 +} +pub(crate) const G17P_PARTIAL_OPENING_TILING_CHANNEL_TABLE_INDEX: u8 = + g17p_work_channel_table_index(G17P_NATIVE_RENDER_DOORBELL_PRIORITY, PipeType::Vertex); +pub(crate) const G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX: u8 = + g17p_work_channel_table_index(G17P_NATIVE_RENDER_DOORBELL_PRIORITY, PipeType::Fragment); +pub(crate) const G17P_COMPUTE_CHANNEL_TABLE_INDEX: u8 = + g17p_work_channel_table_index(2, PipeType::Compute); +// The compute index is the validated anchor for the rule above: it must stay 8. +const _: () = assert!(G17P_COMPUTE_CHANNEL_TABLE_INDEX == 8); +const _: () = assert!(G17P_PARTIAL_OPENING_TILING_CHANNEL_TABLE_INDEX == + G17P_NATIVE_RENDER_DOORBELL_PRIORITY * 3); +const _: () = assert!(G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX == + G17P_NATIVE_RENDER_DOORBELL_PRIORITY * 3 + 1); +/// Compute work-submission doorbell channel. VALIDATED by 2000 consecutive +/// passing computes. Decodes under `encode_tx_doorbell` as +/// `pipe Compute(2) | (priority 2 << 2)` = 0x0a, which is why it is not simply +/// the channel-table index (8). +pub(crate) const G17P_COMPUTE_WORK_DOORBELL_CHANNEL: u8 = 0x0a; +pub(crate) const G17P_COMPUTE_WORK_DOORBELL: u64 = + MSG_TX_DOORBELL | G17P_COMPUTE_WORK_DOORBELL_CHANNEL as u64; +pub(crate) const G17P_COLD_OPENING_RECORD_POOL_A_COUNT: usize = 35; +pub(crate) const G17P_COLD_OPENING_RECORD_POOL_A_STRIDE: usize = 0x100; +pub(crate) const G17P_COLD_OPENING_RECORD_POOL_A_SIZE: usize = + G17P_COLD_OPENING_RECORD_POOL_A_COUNT * G17P_COLD_OPENING_RECORD_POOL_A_STRIDE; +pub(crate) const G17P_COLD_OPENING_RECORD_POOL_B_SIZE: usize = 79 * 0x80; +pub(crate) const G17P_COLD_OPENING_SHARED_OBJECT_SIZE: usize = 0x88; +pub(crate) const G17P_COLD_OPENING_SHARED_CONTROL_SIZE: usize = 0x70; +pub(crate) const G17P_COLD_OPENING_SHARED_CONTROL_INNER_SIZE: usize = 4; +pub(crate) const G17P_COLD_OPENING_CHANNEL_CONTROL_SIZE: usize = 0x40; +pub(crate) const G17P_COLD_OPENING_OPTIONAL_SIZE: usize = 0xc0; +pub(crate) const G17P_COLD_OPENING_EVENT_SIZE: usize = 0x40; +pub(crate) const G17P_COLD_OPENING_QUEUE_CONTEXT_SIZE: usize = 0x4000; +pub(crate) const G17P_COLD_OPENING_TA_DESCRIPTOR_BASE: u64 = 0xffff_fc20_c001_8000; +pub(crate) const G17P_COLD_OPENING_3D_DESCRIPTOR_BASE: u64 = 0xffff_fc20_c00b_0000; +pub(crate) const G17P_COLD_OPENING_QUEUE_RECORD_SIZE: usize = 0xc0; +pub(crate) const G17P_COLD_OPENING_POINTER_BLOCK_SIZE: usize = 0x80; +pub(crate) const G17P_COLD_OPENING_OPERAND_DIRECTORY_GPU_VA: u64 = 0x70_0000_0000; +pub(crate) const G17P_COLD_OPENING_OPERAND_DIRECTORY_SIZE: usize = 4 * 0x4000; +pub(crate) const G17P_COLD_OPENING_OPERAND_TABLE_GPU_VA: u64 = 0x70_0020_8000; +pub(crate) const G17P_COLD_OPENING_OPERAND_BUFFER_BASE: u64 = 0x70_0022_0000; +pub(crate) const G17P_COLD_OPENING_OPERAND_BUFFER_COUNT: usize = 28; +pub(crate) const G17P_COLD_OPENING_OPERAND_BUFFER_SIZE: u64 = 0x10_0000; +pub(crate) const G17P_COLD_OPENING_OPERAND_BUFFER_STRIDE: u64 = 0x10_8000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PColdOpeningLeaf { + PrimaryIndex, + SecondaryIndex, + PoolASlots, + PoolBSlots, + SharedSlots, + Flag, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PColdOpeningStage { + Tiling, + Fragment, +} + +/// Relocatable leaf addresses used by the cold-opening pool and shared graph. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PColdOpeningGraphAddresses { + pub(crate) primary_index: u64, + pub(crate) secondary_index: u64, + pub(crate) pool_a_slots: u64, + pub(crate) pool_b_slots: u64, + pub(crate) shared_slots: u64, + pub(crate) flag: u64, +} + +/// Relocatable pointers carried by one cold-opening optional record. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PColdOpeningOptionalAddresses { + pub(crate) context_scratch: u64, + pub(crate) firmware_scratch: u64, + pub(crate) usc_priv_mem_flist: u64, + /// HardwareBufferID of `usc_priv_mem_flist`, copied to tag-15 +0x46. + pub(crate) usc_freelist_hardware_buffer_id: u32, + pub(crate) channel_control: u64, + pub(crate) tiling_shared_object: u64, + /// HWPB owner lease token (HardwareBufferBase +0x68), not an ordinal. + /// Reacquiring the TA HardwareBufferID from zero references assigns it. + pub(crate) parameter_buffer_token: u64, + /// PID of the process that owns the paired TA/3D channels. Native stores + /// the same u32 in `_AGFIChannelState +0x44` and ConfigUpdate +0x5a. + pub(crate) owner_pid: u32, + /// TA HardwareBufferID owned by the command being published. + /// + /// Native copies one allocator result to five firmware-visible sinks: TA + /// command +0x18, register 0x1c830, ConfigUpdate +0x7e, PBState +0x0c, and + /// the PBDesc ring index. Keeping the ConfigUpdate copy beside its object + /// pointer prevents those fields from being staged from different command + /// lifetimes. + pub(crate) ta_hardware_buffer_id: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PColdOpeningQueueAddresses { + pub(crate) pointers: u64, + pub(crate) item_ring: u64, + pub(crate) job_list: u64, + pub(crate) channel_control: u64, + /// PID of the process that owns this channel state. + pub(crate) owner_pid: u32, +} + +/// Root, queue, and scheduler identities at the proven first-work boundary. +/// The successful run used the two-entry macOS context table: slot/tag 0 and +/// slot/tag 1 both retain their low roots while their high roots are replaced +/// with distinct empty pages. Linux owns those pages, so their measured +/// physical addresses are intentionally not part of this contract. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PPartialOpeningContextContract { + pub(crate) context0_root_slot: u8, + pub(crate) context0_context_id: u8, + pub(crate) render_root_slot: u8, + pub(crate) render_context_id: u8, + pub(crate) context0_empty_high_root_slot: u8, + pub(crate) render_empty_high_root_slot: u8, + pub(crate) transport_pair: u8, + pub(crate) descriptor_pair: u8, + pub(crate) shared_queue_pair_namespace: u8, + pub(crate) optional_identity_context_id: u8, + pub(crate) descriptor_context_id: u8, + pub(crate) scheduler_class: u8, + pub(crate) optional_grids: [u8; 2], + pub(crate) queue_identity: u8, + pub(crate) optional_submission_ordinal: u16, + pub(crate) event_counter: u32, + pub(crate) work_doorbell_channel: u8, + pub(crate) context0_low_root_present_at_first_work: bool, + pub(crate) render_low_root_present_at_first_work: bool, + pub(crate) source_firmware_high_root_table_resident: bool, +} + +pub(crate) const G17P_PARTIAL_OPENING_CONTEXT: G17PPartialOpeningContextContract = + G17PPartialOpeningContextContract { + context0_root_slot: 0, + context0_context_id: 0, + render_root_slot: 1, + render_context_id: 1, + context0_empty_high_root_slot: 0, + render_empty_high_root_slot: 1, + transport_pair: 0, + descriptor_pair: 0, + shared_queue_pair_namespace: 0, + optional_identity_context_id: 1, + descriptor_context_id: 1, + scheduler_class: 2, + optional_grids: [0, 1], + queue_identity: 0x15, + optional_submission_ordinal: 0, + event_counter: 0x102, + work_doorbell_channel: G17P_PARTIAL_OPENING_WORK_DOORBELL_CHANNEL, + context0_low_root_present_at_first_work: true, + render_low_root_present_at_first_work: true, + source_firmware_high_root_table_resident: false, + }; + +/// One exact 64-bit write into the primary status-B object before control 0x84. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PPartialOpeningStatusWrite { + pub(crate) offset: u32, + pub(crate) value: u64, +} + +/// State required to prepare one half of the paired outer-channel publish. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PPartialOpeningOuterChannelState { + pub(crate) consumers: [u32; 2], + pub(crate) producer: u32, + pub(crate) queue_gpu_va: u64, + pub(crate) queue_write_index: u32, +} + +/// One complete 0x18-byte outer slot followed by its producer publication. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PPartialOpeningOuterSlotPlan { + pub(crate) channel_table_index: u8, + pub(crate) slot_index: u8, + pub(crate) slot_byte_offset: u32, + pub(crate) slot: [u8; G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE], + pub(crate) next_producer: u32, +} + +/// The two producer writes and the single mailbox kick for first graphics. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PPartialOpeningWorkPair { + pub(crate) fragment: G17PPartialOpeningOuterSlotPlan, + pub(crate) tiling: G17PPartialOpeningOuterSlotPlan, + pub(crate) paired_doorbell: u64, +} + +/// One proven CL_2 outer-ring publication and its matching RTKit doorbell. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PComputeChannelWork { + pub(crate) outer: G17PPartialOpeningOuterSlotPlan, + pub(crate) doorbell: u64, +} + +/// Side-effect order measured at the successful first-partial boundary. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PPartialOpeningPublicationStep { + PrimarySchedulerPage, + PrimaryIndexPage, + PreControlStatus, + ControlDone0x84, + FragmentState, + EmptyHighRoots, + FragmentProducer, + LateTilingState, + SystemBarrierBeforeTilingProducer, + TilingProducer, + SystemBarrierBeforePairedDoorbell, + PairedWorkDoorbell0x83, +} + +pub(crate) const G17P_PARTIAL_OPENING_PUBLICATION_ORDER: [G17PPartialOpeningPublicationStep; 12] = [ + G17PPartialOpeningPublicationStep::PrimarySchedulerPage, + G17PPartialOpeningPublicationStep::PrimaryIndexPage, + G17PPartialOpeningPublicationStep::PreControlStatus, + G17PPartialOpeningPublicationStep::ControlDone0x84, + G17PPartialOpeningPublicationStep::FragmentState, + G17PPartialOpeningPublicationStep::EmptyHighRoots, + G17PPartialOpeningPublicationStep::FragmentProducer, + G17PPartialOpeningPublicationStep::LateTilingState, + G17PPartialOpeningPublicationStep::SystemBarrierBeforeTilingProducer, + G17PPartialOpeningPublicationStep::TilingProducer, + G17PPartialOpeningPublicationStep::SystemBarrierBeforePairedDoorbell, + G17PPartialOpeningPublicationStep::PairedWorkDoorbell0x83, +]; + +/// Build the computed one-record scheduler page that decided graphics entry in +/// the successful post-control graft. +pub(crate) fn apply_g17p_partial_opening_scheduler_page( + page: &mut [u8], +) -> Result<(), SubmissionError> { + if page.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningPageSize); + } + page.fill(0); + put_u64_le(page, 0, G17P_PARTIAL_OPENING_SCHEDULER_WORD); + Ok(()) +} + +/// Array-returning form retained for host tests. Kernel mappings should use +/// [`apply_g17p_partial_opening_scheduler_page`] to avoid a 16 KiB temporary. +pub(crate) fn build_g17p_partial_opening_scheduler_page() -> [u8; G17P_PARTIAL_OPENING_PAGE_SIZE] { + let mut page = [0u8; G17P_PARTIAL_OPENING_PAGE_SIZE]; + let _ = apply_g17p_partial_opening_scheduler_page(&mut page); + page +} + +/// Build the eight-group primary index used by the proven opening. +pub(crate) fn apply_g17p_partial_opening_primary_index_page( + page: &mut [u8], +) -> Result<(), SubmissionError> { + if page.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningPageSize); + } + page.fill(0); + let mut index = 0usize; + for (start, groups) in [(0x11u32, 6u32), (0x3cu32, 2u32)] { + for group in 0..groups { + let base = start + group * 5; + for member in 0..4u32 { + put_u32_le(page, index * 4, base + member); + index += 1; + } + } + } + Ok(()) +} + +/// Array-returning form retained for host tests. Kernel mappings should use +/// [`apply_g17p_partial_opening_primary_index_page`] directly. +pub(crate) fn build_g17p_partial_opening_primary_index_page() -> [u8; G17P_PARTIAL_OPENING_PAGE_SIZE] +{ + let mut page = [0u8; G17P_PARTIAL_OPENING_PAGE_SIZE]; + let _ = apply_g17p_partial_opening_primary_index_page(&mut page); + page +} + +/// Write one of the six source-built leaf pages used by the cold opening. +pub(crate) fn apply_g17p_cold_opening_leaf_page( + leaf: G17PColdOpeningLeaf, + page: &mut [u8], +) -> Result<(), SubmissionError> { + if page.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + page.fill(0); + match leaf { + G17PColdOpeningLeaf::PrimaryIndex => { + apply_g17p_partial_opening_primary_index_page(page)?; + } + G17PColdOpeningLeaf::SecondaryIndex => { + let mut index = 0usize; + for (start, groups) in [(0x11u32, 6u32), (0x3cu32, 2u32)] { + for group in 0..groups { + put_u64_le(page, index * 8, (start + group * 5) as u64); + index += 1; + } + } + } + G17PColdOpeningLeaf::PoolASlots => put_u32_le(page, 0x04, 2), + G17PColdOpeningLeaf::PoolBSlots => {} + G17PColdOpeningLeaf::SharedSlots => { + put_u32_le(page, 0x00, 8); + put_u32_le(page, 0x04, 8); + put_u32_le(page, 0x60, 1); + } + G17PColdOpeningLeaf::Flag => put_u32_le(page, 0x00, 1), + } + Ok(()) +} + +/// Select the Pool-A record used by one render pass. Native G17P moves the cold +/// first pass from record 0 to the captured record 1. Retained native +/// scenes can reuse an index across multiple sequence values, so the traces do +/// not justify replacing Linux's retained `ordinal * 2` policy globally. +pub(crate) const fn g17p_render_pool_a_record_index( + submission_ordinal: u64, + native_first_record: bool, +) -> usize { + if native_first_record && submission_ordinal == 0 { + 1 + } else { + ((submission_ordinal % G17P_COLD_OPENING_RECORD_POOL_A_COUNT as u64) as usize * 2) + % G17P_COLD_OPENING_RECORD_POOL_A_COUNT + } +} + +/// Initialize only the Pool-A slot corresponding to the selected first +/// record. The slot page has no other modeled contents. +pub(crate) fn apply_g17p_cold_opening_pool_a_start_slot( + record_index: usize, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + if record_index >= G17P_COLD_OPENING_RECORD_POOL_A_COUNT { + return Err(SubmissionError::PartialOpeningQueueHeadOutOfRange); + } + out.fill(0); + // Native numbers the slot words from the start of the page: cold record + // index 1 points at page +0x04. The old extra +4 coupled index 1 to +0x08 + // and made the host graph one slot ahead of the valid first-render graph. + put_u32_le(out, record_index * 4, 1); + Ok(()) +} + +/// Write the 35-record Pool-A array with relocatable slot pointers and select +/// the record whose first-pass state word is initialized to 0x50. +pub(crate) fn apply_g17p_cold_opening_record_pool_a_at( + addresses: G17PColdOpeningGraphAddresses, + active_record_index: usize, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if active_record_index >= G17P_COLD_OPENING_RECORD_POOL_A_COUNT { + return Err(SubmissionError::PartialOpeningQueueHeadOutOfRange); + } + if out.len() != G17P_COLD_OPENING_RECORD_POOL_A_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + let slot_base = addresses.pool_a_slots; + for index in 0..G17P_COLD_OPENING_RECORD_POOL_A_COUNT { + let slot = slot_base + .checked_add((index * 4) as u64) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?; + let base = index * G17P_COLD_OPENING_RECORD_POOL_A_STRIDE; + put_u64_le(out, base, slot); + if index == active_record_index { + put_u32_le(out, base + 0x10, 0x50); + } + } + Ok(()) +} + +/// Established record-zero wrapper retained for callers and fixture tests. +pub(crate) fn apply_g17p_cold_opening_record_pool_a( + addresses: G17PColdOpeningGraphAddresses, + out: &mut [u8], +) -> Result<(), SubmissionError> { + apply_g17p_cold_opening_record_pool_a_at(addresses, 0, out) +} + +/// Write the 79-record Pool-B array in the cold pair-zero namespace. +pub(crate) fn apply_g17p_cold_opening_record_pool_b( + addresses: G17PColdOpeningGraphAddresses, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_RECORD_POOL_B_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + let slot_base = addresses + .pool_b_slots + .checked_add(4) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?; + let shared = addresses + .shared_slots + .checked_add(0x40) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?; + for index in 0..79usize { + let base = index * 0x80; + put_u32_le(out, base, 0x80004 + index as u32 * 4); + put_u32_le(out, base + 0x04, 0x10); + put_u64_le( + out, + base + 0x08, + slot_base + .checked_add((index * 4) as u64) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?, + ); + let phase = index % 36; + let cycle = if phase == 35 { + 0x178000 + } else { + 0x178020 + phase as u32 * 0x20 + }; + put_u32_le(out, base + 0x28, cycle); + put_u64_le(out, base + 0x40, shared); + if index == 0 { + put_u32_le(out, base + 0x4c, 1); + } + } + Ok(()) +} + +/// Write the legacy packed HWPB-shaped object used by TA and by the +/// established Fragment path. Native Fragment can instead name the dedicated +/// PM HWPB state; TA keeps this distinct object. +pub(crate) fn apply_g17p_cold_opening_shared_object( + addresses: G17PColdOpeningGraphAddresses, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_SHARED_OBJECT_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + for (offset, address) in [ + (0x20, addresses.primary_index), + (0x44, addresses.secondary_index), + (0x4c, addresses.shared_slots), + (0x64, addresses.flag), + ] { + put_u64_le(out, offset, address); + } + for (offset, value) in [ + (0x28, 0x0019_0000), + (0x2c, 0x10), + (0x30, 0x0001_0000), + (0x34, 0x20), + (0x38, 0x0c18), + (0x3c, 8), + (0x54, 0x1f), + (0x58, 0x0002_0000), + (0x7c, 0x3060), + (0x80, 0x1020), + (0x84, 0x0018_0000), + ] { + put_u32_le(out, offset, value); + } + Ok(()) +} + +/// Write the 0x70-byte class-2 control object used before cold first work. +pub(crate) fn apply_g17p_cold_opening_shared_control( + operand_table: u64, + inner_state: u64, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_SHARED_CONTROL_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u32_le(out, 0x00, 1); + put_u32_le(out, 0x10, 2); + put_u64_le(out, 0x18, 0x0004_0000_0000_0070); + put_u64_le(out, 0x20, 0x0000_1900_0000_0000); + put_u64_le(out, 0x28, 0x0000_1900_0000_0000); + put_u64_le(out, 0x30, operand_table); + put_u32_le(out, 0x40, 4); + put_u32_le(out, 0x48, 0xe0); + put_u64_le(out, 0x4c, inner_state); + put_u32_le(out, 0x60, 3); + Ok(()) +} + +pub(crate) fn apply_g17p_cold_opening_shared_control_inner( + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_SHARED_CONTROL_INNER_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + put_u32_le(out, 0, 2); + Ok(()) +} + +/// Write the fresh channel-control record used by both cold queues. +pub(crate) fn apply_g17p_cold_opening_channel_control( + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_CHANNEL_CONTROL_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u64_le(out, 0x00, 0x0000_0100_0000_ffff); + put_u64_le(out, 0x20, 0x0002_0000_0000_0000); + put_u64_le(out, 0x30, 0x0000_0000_ff00_0000); + Ok(()) +} + +/// Write one exact grid-0/1 optional record from the cold partial opening. +pub(crate) fn apply_g17p_cold_opening_optional( + stage: G17PColdOpeningStage, + addresses: G17PColdOpeningOptionalAddresses, + queue_id: u16, + partner_queue_id: u16, + qos_hardware_buffer_id: u16, + install: u16, + context_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_OPTIONAL_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u32_le(out, 0, 0x0f); + for (offset, address) in [ + (0x08, addresses.context_scratch), + (0x10, addresses.firmware_scratch), + (0x36, addresses.usc_priv_mem_flist), + (0x4a, addresses.channel_control), + ] { + put_u64_le(out, offset, address); + } + let fields: &[(usize, u16)] = match stage { + // +0x56 is the AGXQOSQueue HardwareBufferID. Native gives each + // retained TA/3D pair one ID (both stages share it), while different + // pairs carry different IDs. It must match both the qid*8 QoS record + // and the KSM dependency-control qword. + // Byte-diffed against live ConfigUpdate records from two independent + // Metal render passes on the target (macOS 2026-09-02, + // `records-gfx.txt` and `everyobj-gfx.txt`): + // + // +0x66 = 1 in both. + // +0x46 is written below as a u32 from the allocated FList hardware + // buffer id; it reads 0 in one capture and 2 in the other, + // so it is dynamic, not a constant. + // +0x62 read 1 in one capture and 0 in the other, so it is dynamic, + // not a constant; 1 is what this driver always sent and what + // the majority of captured records carry. An earlier claim + // that native always has 0 came from a single record and is + // contradicted by the wider capture. + G17PColdOpeningStage::Tiling => &[ + (0x18, queue_id), + (0x1a, install), + (0x1e, G17P_NATIVE_RENDER_DOORBELL_PRIORITY as u16), + (0x26, 1), + (0x52, 1), + (0x56, qos_hardware_buffer_id), + (0x5e, G17P_NATIVE_RENDER_POLICY), + (0x62, 1), + (0x66, 1), + // +0x82 is the TA's FORWARD REFERENCE TO ITS 3D PARTNER, not a + // flag. MEASURED ON macOS 2026-09-02 (recs2-run1.txt): across two + // processes and six passes, a native TA ConfigUpdate carries the + // partner FRAGMENT queue's id here -- 0x10 while its own +0x18 is + // 0x0f -- and the matching 3D record fills +0x76..+0x85 with 0xff, + // the "no forward reference" sentinel this encoder already writes + // for the Fragment stage. The constant 1 published here before + // named queue 1, which is neither stage. + (0x82, partner_queue_id), + ], + G17PColdOpeningStage::Fragment => &[ + (0x18, queue_id), + (0x1a, install), + (0x1e, G17P_NATIVE_RENDER_DOORBELL_PRIORITY as u16), + (0x22, 1), + (0x26, 1), + (0x52, 1), + (0x56, qos_hardware_buffer_id), + (0x5e, G17P_NATIVE_RENDER_POLICY), + (0x62, 1), + (0x66, 1), + ], + }; + for (offset, value) in fields { + put_u16_le(out, *offset, *value); + } + // DIAGNOSTIC. +0x22 is the ConfigUpdate's data-master field, hardcoded to + // 1 for the fragment. The tiler (DM0) and compute (DM2) both dispatch and + // execute; the fragment is retired with KTrace 0x43 status 4 without ever + // running, with an identically shaped descriptor, a satisfied barrier and + // geometry already tiled. Overriding this isolates the data master itself. + let override_dm = match stage { + G17PColdOpeningStage::Fragment => crate::g18::fw::config::RENDER_3D_DM, + G17PColdOpeningStage::Tiling => crate::g18::fw::config::RENDER_TA_DM, + }; + if override_dm != 0xff { + put_u16_le(out, 0x22, override_dm as u16); + } + // AGXChannel::resetChannelState copies channel+0x4c (the owning process + // PID) to state+0x44. ConfigUpdate carries that same u32 at unaligned + // +0x5a; the working Metal capture has 0x625 in both locations. + put_u32_le(out, 0x5a, addresses.owner_pid); + // Native copies the HardwareBufferID allocated for the FList named at + // +0x36 into this tag-15 field. Keep the object pointer and its table slot + // sourced from the same retained owner. + put_u32_le(out, 0x46, addresses.usc_freelist_hardware_buffer_id); + // +0x32 is the parameter-buffer binding context when +0x6e is nonzero. + // It is a distinct kext allocation/source from TA command +0x0c even when + // both happen to be context 1 in our single-client bring-up path. + put_u16_le(out, 0x32, context_id); + match stage { + G17PColdOpeningStage::Tiling => { + put_u64_le(out, 0x6e, addresses.tiling_shared_object); + put_u64_le(out, 0x76, addresses.parameter_buffer_token); + // AGXTAChannelSKSM::submitBuffer copies descriptor +0x1248 here + // with a 32-bit store. The same source feeds TA command +0x18, + // register 0x1c830, PBState +0x0c, and the PBDesc ring; firmware + // uses it to index the 0x50-byte PB binding table. + put_u32_le(out, 0x7e, addresses.ta_hardware_buffer_id); + } + G17PColdOpeningStage::Fragment => out[0x76..0x86].fill(0xff), + } + Ok(()) +} + +/// Write one cold event record. Both halves use the measured counter 0x102. +pub(crate) fn apply_g17p_cold_opening_event( + stage: G17PColdOpeningStage, + queue_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_EVENT_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u32_le(out, 0x00, 0x0e); + // +0x04 is [u16 queue_id][u16 1]: the firmware's own 0x41 tag14-addkicks + // receipt reads the QID with `ldrh w21, [x19, #4]`, and the native capture + // carries 0x11/0x12 there for TA/3D. + put_u32_le(out, 0x04, 0x0001_0000 | queue_id as u32); + put_u32_le( + out, + 0x08, + 0x100 | u32::from(G17P_NATIVE_RENDER_DOORBELL_PRIORITY), + ); + if stage == G17PColdOpeningStage::Fragment { + put_u32_le(out, 0x10, 0x100); + } + Ok(()) +} + +/// Refresh the canonical optional record for a later logical submission. +/// The queue-context record itself remains in locator slot zero. +/// +/// Publishes the per-slot ordinal as the FList generation at `+0x3e`, which +/// is the same number as the device-global FList sequence for as long as one +/// client renders. `apply_g17p_retained_optional_with_flist_generation` takes +/// the device-global value explicitly. +pub(crate) fn apply_g17p_retained_optional( + stage: G17PColdOpeningStage, + addresses: G17PColdOpeningOptionalAddresses, + submission_ordinal: u64, + bias_generation: bool, + queue_id: u16, + partner_queue_id: u16, + qos_hardware_buffer_id: u16, + install: u16, + context_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + apply_g17p_retained_optional_with_flist_generation( + stage, + addresses, + submission_ordinal, + g17p_render_flist_generation(submission_ordinal), + bias_generation, + queue_id, + partner_queue_id, + qos_hardware_buffer_id, + install, + context_id, + out, + ) +} + +/// `apply_g17p_retained_optional` with the USC FList generation supplied by +/// the caller. +/// +/// The FList is one device-global pool, so its generation advances with every +/// render on the device, while `submission_ordinal` counts only this slot's +/// renders. With one client the two coincide; the moment a second slot +/// publishes, its first tag-15 would otherwise announce generation 0 against a +/// pool the firmware last saw at N. +pub(crate) fn apply_g17p_retained_optional_with_flist_generation( + stage: G17PColdOpeningStage, + addresses: G17PColdOpeningOptionalAddresses, + submission_ordinal: u64, + flist_generation: u64, + bias_generation: bool, + queue_id: u16, + partner_queue_id: u16, + qos_hardware_buffer_id: u16, + install: u16, + context_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + apply_g17p_retained_optional_with_flist_generation_and_context_update( + stage, + addresses, + submission_ordinal, + flist_generation, + bias_generation, + queue_id, + partner_queue_id, + qos_hardware_buffer_id, + install, + context_id, + submission_ordinal == 0, + out, + ) +} + +/// Retained ConfigUpdate with the GART-context branch supplied independently +/// from queue installation and QoS setup. +/// +/// Native derives +0x62 from the descriptor's GART allocation generation +/// versus the channel's cached context. A graph rebuilt for another VM must +/// therefore request the context update even when its physical QID is already +/// installed and its global submission ordinal is nonzero. Conversely, a +/// same-context restage must leave it clear. +pub(crate) fn apply_g17p_retained_optional_with_flist_generation_and_context_update( + stage: G17PColdOpeningStage, + addresses: G17PColdOpeningOptionalAddresses, + submission_ordinal: u64, + flist_generation: u64, + bias_generation: bool, + queue_id: u16, + partner_queue_id: u16, + qos_hardware_buffer_id: u16, + install: u16, + context_id: u16, + context_update: bool, + out: &mut [u8], +) -> Result<(), SubmissionError> { + // Payload bytes wrap independently of the full retained owner generation. + apply_g17p_cold_opening_optional( + stage, + addresses, + queue_id, + partner_queue_id, + qos_hardware_buffer_id, + install, + context_id, + out, + )?; + if submission_ordinal != 0 { + for offset in [0x1a, 0x52] { + put_u16_le(out, offset, 0); + } + } + put_u16_le(out, 0x62, u16::from(context_update)); + // +0x2a is read by the firmware as a whole u64 (`ldur x27, [x19, #42]` in + // the tag-15 ConfigUpdate handler) and stored into its per-queue table at + // `0xfffffc0000125df0 + qid * 8`. Every consumer of that table uses + // `lsr x9, x11, #40` + `uxtb` -- bits 47:40, which is byte 0x2f, the high + // byte of the u16 written at +0x2e, i.e. `ordinal & 0xff`. And each one + // compares it against a cached byte and SKIPS when they are equal: + // + // ldr x11, [table + qid*8] + // lsr x9, x11, #40 + // ldrb w10, [x19, #40] + // cmp w10, w9, uxtb + // b.eq ; unchanged -> do nothing + // strb w9, [x19, #40] + // + // It is a change-detected generation. The raw ordinal is 0 on the first + // render submit, which matches the firmware's initial cached 0, so the + // very first submission of a boot reads as "nothing changed" and the + // firmware declines to act -- with no record emitted either way, which is + // why no capture ever showed it. + // + // This is the same defect already fixed at +0x3e in this function, whose + // comment records that native never publishes zero there. Apply the same + // bias so the first submit publishes a generation the firmware sees as new. + let generation = if bias_generation { + (submission_ordinal as u16).wrapping_add(1) + } else { + submission_ordinal as u16 + }; + put_u16_le(out, 0x2a, generation); + put_u16_le(out, 0x2e, generation << 8); + // +0x3e is the AGXUSCPrivMemFList generation. Native copies FList+0xb0 + // here and copies the same value into opcode-0x20 +0x0c during + // AsyncGrowEngine::updateSubmitInfo. Keep both Linux producers on the + // shared helper below: changing this generation between registration and + // the tag-15 kick makes B1 release and re-register the just-published USC + // allocation before fragment engine service. + // The kext uses a qword store at 8a98c48; +0x46 begins the independent + // hardware-buffer ID and must remain untouched by this generation. + put_u64_le(out, 0x3e, flist_generation); + // Keep the pair-owned HardwareBufferID identical in ConfigUpdate, the + // QID-indexed QoS row, and the KSM dependency-control qword. + put_u16_le(out, 0x56, qos_hardware_buffer_id); + // TA +0x76 is not the submission ordinal. Native copies the u64 at + // AGXHardwareBufferBase +0x68 here. The cold encoder above copies the + // captured PB owner lease token; leave it intact when refreshing queue + // generations. In particular, do not overwrite +0x76/+0x7e with ordinal. + Ok(()) +} + +/// Host-owned equivalent of AGXUSCPrivMemFList +0xb0 for a retained render +/// queue. Valid native cold records start at zero; each later retained restage +/// advances the same generation once. +pub(crate) const fn g17p_render_flist_generation(submission_ordinal: u64) -> u64 { + submission_ordinal as u64 +} + +/// Refresh the canonical event record while retaining its physical address. +/// +/// Tag-14 +0x09 is an unaligned u64 containing the 40-bit KSM timestamp. +/// 25G83 A000 reads it with `ldur x3, [x19, #9]` at 0x11730. Small cold +/// captures do not establish a 24-bit field. +0x08 remains queue priority; +/// the independently owned data-master field starts at +0x11. +pub(crate) fn apply_g17p_retained_event_with_stamp( + stage: G17PColdOpeningStage, + entry_stamp: u64, + queue_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if entry_stamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + apply_g17p_cold_opening_event(stage, queue_id, out)?; + put_u64_le(out, 0x09, entry_stamp); + Ok(()) +} + +/// Refresh the canonical event record while retaining its physical address. +/// +/// Without the SKSM half there is no KSM producer to read a stamp from, so +/// the stamp is derived as `submission_ordinal + 1` -- which is exactly what +/// the producer would have yielded for the first `stamp_count` submissions, +/// since the geometry seeds `current_timestamp` at 1 and advances it by one +/// per kick. +pub(crate) fn apply_g17p_retained_event( + stage: G17PColdOpeningStage, + submission_ordinal: u64, + queue_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + let group_number = submission_ordinal + .checked_add(1) + .ok_or(SubmissionError::PartialOpeningQueueHeadOutOfRange)?; + apply_g17p_retained_event_with_stamp(stage, group_number & KICK_TS_MASK, queue_id, out) +} + +/// Write the cold class-2 queue record for the owning process. +pub(crate) fn apply_g17p_cold_opening_queue_record( + addresses: G17PColdOpeningQueueAddresses, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_QUEUE_RECORD_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u64_le(out, 0x00, addresses.pointers); + put_u64_le(out, 0x08, addresses.item_ring); + // +0x10. Native's per-queue ring records carry the SAME pointer here for + // both render stages (ringrec-gfx.txt: 0xfffffc2000000060 in the TA record + // at fwva 0xfffffc20c0000b40 and in the 3D record at 0xfffffc20c0000c00), + // and it is the a5 argument of every AGXChannel::init. That looked like a + // shared firmware constant we were failing to write. It is NOT. + // + // KEXT_PROVEN (25G83): cmdq+0x680 is an AGXTimeStampQueue whose +0x28 is + // its firmware VA; AGXCommandQueue::allocate3DWorkQueueInner passes that + // as a5 to the channel init, AGXChannel::init stores it at chan+0x98 and + // resetChannelState copies it to state+0x10. The 0x18-byte + // _AGFITimeStampQueue slots are host-allocated out of a firmware-shared + // stack, and 0x60 is simply slot 4. The value repeats across boots only + // because the Metal test app is always the fifth client. Writing that + // literal from Linux would have published a pointer into someone else's + // allocation. + // + // FW_PROVEN: nothing in the DM-ring consumption path (FUN_261a4 / + // FUN_12638) nor in the job builder FUN_23004 reads state+0x10 at all. It + // is a per-command-queue timestamp list head, used only by + // AGFAcceleratorProcessTimeStampQueue on the completion side. It cannot + // stop consumption or launch, so it is not our bug. Our per-graph value + // stays. + put_u64_le(out, 0x10, addresses.job_list); + put_u32_le(out, 0x24, u32::MAX); + // The live native render path reads state +0x28 through getPriority(), + // copies it to 3D setup +0xe5c, and validates it against KSM queue +0x1c. + // The captured ordinary Metal command carries 2. Channel +0x1c0 is only + // context-priority bookkeeping and is not the live-state value. + let priority = u32::from(G17P_NATIVE_RENDER_DOORBELL_PRIORITY); + put_u32_le(out, 0x28, priority); + put_u32_le(out, 0x2c, priority); + if priority == 0 { + out[0x32..0x38].fill(0xff); + put_u32_le(out, 0x38, 1); + } else { + out[0x36..0x38].fill(0xff); + put_u32_le(out, 0x38, 0); + } + // MEASURED ON macOS 2026-09-02 (`ringrec-gfx.txt`): the live per-queue + // record of a real Metal render pass, read through the kernel alias the + // AGXChannel keeps at +0x58 for the firmware VA at +0x80, is + // + // +0x24 0xffffffff +0x28 2 +0x2c 2 +0x36 0xffff + // +0x40 2 +0x44 0xffffffff +0x48 + // +0x9c + // + // This driver had that {2, 0xffffffff, pid} block four bytes early, at + // +0x3c/+0x40/+0x44, and the pointer at +0x70. The pointer had once been + // at +0x9c and was moved on the theory that the object is 0x80 bytes; the + // ring slots name records on an exact 0xc0 stride (base + qid * 0xc0), so + // 0x9c is inside the object and was right all along. + put_u32_le(out, 0x40, u32::from(G17P_NATIVE_RENDER_POLICY)); + put_u32_le(out, 0x44, u32::MAX); + // Native reset copies the channel owner's process PID here. The same + // value is published in ConfigUpdate +0x5a. + put_u32_le(out, 0x48, addresses.owner_pid); + put_u64_le(out, 0x9c, addresses.channel_control); + Ok(()) +} + +pub(crate) fn apply_g17p_cold_opening_pointer_block(out: &mut [u8]) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_POINTER_BLOCK_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u32_le(out, 0x50, u32::MAX); + put_u64_le(out, 0x60, 0x500); + Ok(()) +} + +/// Apply the measured first-publication bytes after descriptor construction. +pub(crate) fn apply_g17p_cold_opening_descriptor_overrides( + stage: G17PColdOpeningStage, + descriptor: &mut [u8], +) -> Result<(), SubmissionError> { + if descriptor.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + let fields: &[(usize, u8)] = match stage { + G17PColdOpeningStage::Tiling => &[ + (0x38, 0x47), + (0x3a, 0x49), + (0x3c, 0x49), + (0x0789, 0x08), + (0x093e, 0xd0), + (0x093f, 0x91), + ], + G17PColdOpeningStage::Fragment => &[ + (0x50, 0x01), + (0x80, 0x56), + (0x82, 0x57), + (0x84, 0x57), + (0x88, 0x59), + (0x90, 0x01), + (0x215c, 0x00), + (0x222d, 0x00), + ], + }; + for (offset, value) in fields { + descriptor[*offset] = *value; + } + put_u32_le(descriptor, 0x0c, 1); + Ok(()) +} + +/// Keep the fixed operand table zero at the proven cold first-work boundary. +pub(crate) fn apply_g17p_cold_opening_operand_table(out: &mut [u8]) -> Result<(), SubmissionError> { + if out.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + Ok(()) +} + +/// Write the optional 28-entry table used by a later populated control state. +pub(crate) fn apply_g17p_populated_operand_table(out: &mut [u8]) -> Result<(), SubmissionError> { + apply_g17p_cold_opening_operand_table(out)?; + for index in 0..G17P_COLD_OPENING_OPERAND_BUFFER_COUNT { + let address = G17P_COLD_OPENING_OPERAND_BUFFER_BASE + .checked_add(index as u64 * G17P_COLD_OPENING_OPERAND_BUFFER_STRIDE) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?; + put_u64_le(out, index * 0x40, address | (1u64 << 60)); + } + Ok(()) +} + +/// Keep the four-page operand directory zero through cold first work. +pub(crate) fn apply_g17p_cold_opening_operand_directory( + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_OPERAND_DIRECTORY_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + Ok(()) +} + +/// Write the optional populated directory for the 28 operand buffers. +pub(crate) fn apply_g17p_populated_operand_directory( + out: &mut [u8], +) -> Result<(), SubmissionError> { + apply_g17p_cold_opening_operand_directory(out)?; + let pages_per_buffer = G17P_COLD_OPENING_OPERAND_BUFFER_SIZE / 0x1000; + let mut qword = 0usize; + for buffer in 0..G17P_COLD_OPENING_OPERAND_BUFFER_COUNT { + let base = G17P_COLD_OPENING_OPERAND_BUFFER_BASE + .checked_add(buffer as u64 * G17P_COLD_OPENING_OPERAND_BUFFER_STRIDE) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?; + for page in 0..pages_per_buffer { + put_u64_le(out, qword * 8, base + page * 0x1000); + qword += 1; + } + } + Ok(()) +} + +/// Write the queue-context page for the clean first-partial TA/3D pair. +/// +/// This is the new local grid-0/1 generation selected by +/// `G17P_PARTIAL_OPENING_GRAPH=1`. Its namespace tag and fragment locators are +/// distinct from the untagged full-opening pair-zero context. +pub(crate) fn apply_g17p_cold_opening_queue_context( + stage: G17PColdOpeningStage, + descriptor: u64, + queue: u64, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_QUEUE_CONTEXT_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + let (descriptor_base, words, locators): (u64, &[(usize, u64)], &[(usize, u64)]) = match stage { + G17PColdOpeningStage::Tiling => ( + G17P_COLD_OPENING_TA_DESCRIPTOR_BASE, + &[ + (0x200, 0x1000_0000_0000_0004), + (0x220, 0xffff_0c00_0000_0001), + (0x378, 0x003f_ffff_ffff_ffff), + ], + &[(0x350, 0x0002_3803_8000_0003)], + ), + G17PColdOpeningStage::Fragment => ( + G17P_COLD_OPENING_3D_DESCRIPTOR_BASE, + &[ + (0x200, 0x1000_0400_0000_0004), + (0x220, 0xffff_1800_0000_0003), + (0x228, 0x0000_0000_0000_0001), + (0x230, 0x0000_0100_0000_0000), + (0x378, 0x003f_ffff_ffff_ffff), + ], + &[ + (0x350, 0x0002_b003_8000_4c05), + (0x358, 0x0000_8003_8000_4c3e), + (0x360, 0x0000_b803_8000_4c77), + (0x368, 0x0000_5003_8000_4cb0), + ], + ), + }; + let descriptor_delta = descriptor + .checked_sub(descriptor_base) + .ok_or(SubmissionError::PartialOpeningDescriptorLocatorUnencodable)?; + if descriptor_delta & 0x1f != 0 { + return Err(SubmissionError::PartialOpeningDescriptorLocatorUnencodable); + } + let locator_delta = descriptor_delta / 0x20; + for (offset, value) in words { + put_u64_le(out, *offset, *value); + } + for (offset, value) in locators { + put_u64_le( + out, + *offset, + value + .checked_add(locator_delta) + .ok_or(SubmissionError::PartialOpeningDescriptorLocatorUnencodable)?, + ); + } + put_u64_le(out, 0x210, descriptor); + put_u64_le(out, 0x218, queue); + Ok(()) +} + +/// Derive the five stable status-B writes presented before control 0x84. +pub(crate) const fn prepare_g17p_partial_opening_pre_0x84_status( + kernel_va_base: u64, +) -> Result<[G17PPartialOpeningStatusWrite; 5], SubmissionError> { + let fwctl_va = match kernel_va_base.checked_add(G17P_PARTIAL_OPENING_FWCTL_OFFSET) { + Some(value) => value, + None => return Err(SubmissionError::PartialOpeningAddressOverflow), + }; + let fwctl_next = match fwctl_va.checked_add(G17P_PARTIAL_OPENING_CONTROL_MESSAGE_SIZE) { + Some(value) => value, + None => return Err(SubmissionError::PartialOpeningAddressOverflow), + }; + Ok([ + G17PPartialOpeningStatusWrite { + offset: 0x4018, + value: 0x0005_0601_0000_0000, + }, + G17PPartialOpeningStatusWrite { + offset: 0x4020, + value: 61, + }, + G17PPartialOpeningStatusWrite { + offset: 0x40b0, + value: 5, + }, + G17PPartialOpeningStatusWrite { + offset: 0x48e0, + value: fwctl_va, + }, + G17PPartialOpeningStatusWrite { + offset: 0x48e8, + value: fwctl_next, + }, + ]) +} + +fn prepare_g17p_partial_opening_outer_slot( + state: G17PPartialOpeningOuterChannelState, + channel_table_index: u8, + queue_grid_index: u8, // the QID this slot dispatches to + kind: u32, + channel_started: bool, +) -> Result { + if state.producer >= G17P_PARTIAL_OPENING_OUTER_SLOT_COUNT + || state + .consumers + .iter() + .any(|consumer| *consumer >= G17P_PARTIAL_OPENING_OUTER_SLOT_COUNT) + { + return Err(SubmissionError::PartialOpeningChannelCounterOutOfRange); + } + if state.queue_write_index > u16::MAX as u32 { + return Err(SubmissionError::PartialOpeningQueueHeadOutOfRange); + } + + let next_producer = (state.producer + 1) & 0xff; + if state.consumers.contains(&next_producer) { + return Err(SubmissionError::RingFull); + } + + let mut slot = [0u8; G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE]; + put_u64_le(&mut slot, 0x08, state.queue_gpu_va); + put_u32_le(&mut slot, 0x10, kind); + // +0x14 is [u16 write_index][u8 queue_id][u8 1]. The middle byte is the + // QUEUE ID, not a separate "grid index" -- macOS's own + // AGFIAcceleratorDataMasterEntry slot carries 0x11 (TA) and 0x12 (3D) + // there, matching the QIDs its tag-15 publishes, and the rest of the + // 0x18-byte entry matches this encoder field for field: + // + // +0x00 zero +0x00 zero + // +0x08 queue record VA +0x08 state.queue_gpu_va + // +0x10 data master (TA 0 / 3D 1) +0x10 kind (PipeType Vertex/Fragment) + // +0x14 0x01_11_0003 / 0x01_12_0003 write_index 3, qid, bit24 set + // + // This byte must agree with the QID published in tag-15, tag-14 and the + // SKSM registration. It follows the retained queue object even when a + // recovered compute session advances past its first QID. + // + // +0x17 is NOT a constant. `AGXArmFirmware::encodeAcceleratorRingCommand` + // fills this entry as `+0x08 chanstate fwva, +0x10 type, +0x14 info[0], + // +0x16 qid, +0x17 !chan[0x3c]` -- the last byte is the logical NOT of a + // channel state word, so it is 0 once that word is set. MEASURED on macOS + // 2026-09-02: a live Metal render's data-master ring entries carry 0x00 + // there for BOTH stages (`00 3c 00 10 .. 39 01 08 00` for 3D and + // `.. 43 01 07 00` for TA), across two independent captures with different + // queue ids. This driver hardcoded bit 24, i.e. it told firmware the + // channel state word was clear on every single submission. + put_u32_le( + &mut slot, + 0x14, + state.queue_write_index + | ((queue_grid_index as u32) << 16) + | (if channel_started { 0 } else { 1 << 24 }), + ); + + Ok(G17PPartialOpeningOuterSlotPlan { + channel_table_index, + slot_index: state.producer as u8, + slot_byte_offset: state.producer * G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE as u32, + slot, + next_producer, + }) +} + +pub(crate) fn prepare_g17p_compute_channel_work( + state: G17PPartialOpeningOuterChannelState, + queue_id: u8, + channel_started: bool, +) -> Result { + // Match AGXArmFirmware::encodeAcceleratorRingCommand for every pipe: + // +0x17 is !AGXChannel[0x3c], so it is one only for the first publication + // of a physical channel and zero after that channel has started. + let outer = prepare_g17p_partial_opening_outer_slot( + state, + G17P_COMPUTE_CHANNEL_TABLE_INDEX, + queue_id, + PipeType::Compute as u32, + channel_started, + )?; + Ok(PreparedG17PComputeChannelWork { + outer, + doorbell: G17P_COMPUTE_WORK_DOORBELL, + }) +} + +/// Couple the fragment-first/tiling-second outer slots to their one channel-0 +/// 0x83 notification. These are the initdata work rings at table indices 1 and +/// 0, not the kext's separate 8-byte command-pointer rings. The caller performs +/// the barriers in [`G17P_PARTIAL_OPENING_PUBLICATION_ORDER`]. +pub(crate) fn prepare_g17p_partial_opening_work_pair( + fragment: G17PPartialOpeningOuterChannelState, + tiling: G17PPartialOpeningOuterChannelState, + fragment_queue_id: u8, + tiling_queue_id: u8, + first_submit: bool, +) -> Result { + // `+0x17` is `!AGXChannel[0x3c]`: one on the first publication of a + // physical queue and zero after it has started. Every valid native queue + // follows 1 -> 0; a module-wide constant encoded one side incorrectly. + let render_channel_started = !first_submit; + let fragment = match prepare_g17p_partial_opening_outer_slot( + fragment, + G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX, + fragment_queue_id, + 1, + render_channel_started, + ) { + Ok(value) => value, + Err(error) => return Err(error), + }; + let tiling = match prepare_g17p_partial_opening_outer_slot( + tiling, + G17P_PARTIAL_OPENING_TILING_CHANNEL_TABLE_INDEX, + tiling_queue_id, + 0, + render_channel_started, + ) { + Ok(value) => value, + Err(error) => return Err(error), + }; + Ok(PreparedG17PPartialOpeningWorkPair { + fragment, + tiling, + paired_doorbell: G17P_PARTIAL_OPENING_PAIRED_DOORBELL, + }) +} + +/// Layout-only forms used to make accidental field drift a compile-time +/// error. The intervening words remain unnamed because their semantics are +/// outside the admitted producer/consumer subset. +#[allow(dead_code)] +#[repr(C)] +struct G17PCachedChannelStateLayoutAssert { + uncached_state_gpu_va: u64, + command_pointer_ring_gpu_va: u64, + _unknown_10: u64, + _unknown_18: u32, + coherency_cursor: u32, + _unknown_20: [u8; 0x60], +} + +#[allow(dead_code)] +#[repr(C)] +struct G17PUncachedChannelStateLayoutAssert { + consumer: u32, + _unknown_04: [u8; 0x0c], + _unknown_10: u32, + _unknown_14: [u8; 0x0c], + _unknown_20: u32, + _unknown_24: [u8; 0x0c], + coherency_shadow: u32, + _unknown_34: [u8; 0x0c], + producer: u32, + _unknown_44: [u8; 0x0c], + recovery_target: u32, + _unknown_54: [u8; 0x0c], + ring_entry_count: u32, +} + +const _: () = { + assert!(core::mem::size_of::() == 0x80); + assert!( + core::mem::offset_of!(G17PCachedChannelStateLayoutAssert, uncached_state_gpu_va) == 0x00 + ); + assert!( + core::mem::offset_of!( + G17PCachedChannelStateLayoutAssert, + command_pointer_ring_gpu_va + ) == 0x08 + ); + assert!(core::mem::offset_of!(G17PCachedChannelStateLayoutAssert, coherency_cursor) == 0x1c); + + assert!(core::mem::size_of::() == 0x64); + assert!(core::mem::offset_of!(G17PUncachedChannelStateLayoutAssert, consumer) == 0x00); + assert!(core::mem::offset_of!(G17PUncachedChannelStateLayoutAssert, coherency_shadow) == 0x30); + assert!(core::mem::offset_of!(G17PUncachedChannelStateLayoutAssert, producer) == 0x40); + assert!(core::mem::offset_of!(G17PUncachedChannelStateLayoutAssert, recovery_target) == 0x50); + assert!(core::mem::offset_of!(G17PUncachedChannelStateLayoutAssert, ring_entry_count) == 0x60); +}; + +// --------------------------------------------------------------------------- +// 3. Legacy AGX2 work-queue RingState. The A18 Pro ARM firmware directly +// confirms only done@0; the SKSM hardware consumer keeps the remaining +// rptr/wptr/size offsets unconfirmed, so this record is not admitted as a +// native G17P layout. +// --------------------------------------------------------------------------- + +pub(crate) const AGX2_RING_STATE_GPU_DONEPTR_OFFSET: u32 = 0x00; +pub(crate) const AGX2_RING_STATE_GPU_RPTR_OFFSET: u32 = 0x30; +pub(crate) const AGX2_RING_STATE_CPU_WPTR_OFFSET: u32 = 0x40; +pub(crate) const AGX2_RING_STATE_RB_SIZE_OFFSET: u32 = 0x50; + +/// AGX2 work-queue entries are u64 command GPU virtual addresses. +pub(crate) const AGX2_RING_ENTRY_BYTES: u32 = 8; + +/// The shipping producer's maximum outstanding event slots +/// (`workqueue.rs::MAX_JOB_SLOTS`). This is not the command-ring depth: the +/// current Linux ring has `WQ_SIZE == 0x500`. Its numerical equality with the +/// 7-bit QID maximum is only a coincidence and carries no ABI meaning. +pub(crate) const HOST_MAX_OUTSTANDING_JOB_SLOTS: u32 = 127; + +/// Advance a producer/consumer index within a ring of `size` entries. +/// +/// Return an error for a zero-sized ring or a producer outside the ring. A +/// valid `index` is always below `u32::MAX`, so `index + 1` cannot overflow +/// after these checks. +pub(crate) const fn ring_next(index: u32, size: u32) -> Result { + if size == 0 { + return Err(SubmissionError::RingSizeZero); + } + if index >= size { + return Err(SubmissionError::RingIndexOutOfRange); + } + Ok((index + 1) % size) +} + +// --------------------------------------------------------------------------- +// 4. SKSM kick word + per-QID kick-record table (AGX3-specific). +// Grounded from GFX 0x10c88-0x10d2c. The canonical kick-word *encoder* lives +// in `g17_rtkit::KickWord`; these constants are the same firmware bytes, +// re-cited so this module stands alone. `g17_boot` cross-checks them against +// `g17_rtkit` to prove no drift. +// --------------------------------------------------------------------------- + +/// TS occupies bits [39:0] (`and x0,x5,#0xffffffffff` at 25G83 A000 0xfd18). 40-bit, per +/// the firmware print `"TS=0x%010llx"`. +pub(crate) const KICK_TS_BITS: u32 = 40; +pub(crate) const KICK_TS_MASK: u64 = (1u64 << KICK_TS_BITS) - 1; +/// QID occupies bits [46:40] (`ubfx x1,x5,#40,#7` @ 0x10c90). +pub(crate) const KICK_QID_SHIFT: u32 = 40; +pub(crate) const KICK_QID_BITS: u32 = 7; +pub(crate) const KICK_QID_MASK: u8 = (1u8 << KICK_QID_BITS) - 1; +/// Bit 46 also selects one of two 64-bit valid-queue mask words +/// (`ubfx x6,x5,#46,#1` @ 0x10c9c). +pub(crate) const KICK_SELECTOR_BIT: u32 = 46; +/// Bits [63:47] are unmodeled and must be zero. +pub(crate) const KICK_RESERVED_MASK: u64 = !((1u64 << (KICK_SELECTOR_BIT + 1)) - 1); + +/// Per-QID kick-record table base offset from the valid-mask base +/// (`add x2,x22,#0x10` @ 0x10c94). +pub(crate) const KICK_RECORD_TABLE_BASE: u32 = 0x10; +/// Per-QID record stride (`add x2,x2,x1,lsl#5` @ 0x10c98). +pub(crate) const KICK_RECORD_STRIDE: u32 = 0x20; +/// 128 queue slots: 7-bit QID, two 64-bit valid-mask words selected by bit 46. +pub(crate) const KICK_RECORD_COUNT: u32 = 128; + +/// Fields of one 0x20-byte per-QID kick record (written across 0x10cb0-0x10d2c). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct KickRecordLayout { + /// `+0x00` u64: first/min TS seen for this QID. + pub(crate) ts_first: u32, + /// `+0x08` u64: last/max TS seen for this QID. + pub(crate) ts_last: u32, + /// `+0x10` u64: pending kick-slot bitmask. + pub(crate) slot_mask: u32, + /// `+0x18` u32: valid/first flag. + pub(crate) valid_flag: u32, + /// `+0x1c` u32: winning kick-slot index. + pub(crate) slot_index: u32, +} + +/// The exact offsets, so a change to the firmware layout fails a test. +pub(crate) const KICK_RECORD_LAYOUT: KickRecordLayout = KickRecordLayout { + ts_first: 0x00, + ts_last: 0x08, + slot_mask: 0x10, + valid_flag: 0x18, + slot_index: 0x1c, +}; + +/// Byte offset of one firmware kick record: `0x10 + QID * 0x20`. Mirrors +/// `g17_rtkit::kick_queue_record_offset`. +pub(crate) const fn kick_record_offset(queue_id: u8) -> Result { + if queue_id > KICK_QID_MASK { + Err(SubmissionError::QueueIdOutOfRange) + } else { + Ok(KICK_RECORD_TABLE_BASE + queue_id as u32 * KICK_RECORD_STRIDE) + } +} + +/// The firmware-visible fields of a decoded SKSM kick word. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SksmKickWord { + pub(crate) timestamp: u64, + pub(crate) queue_id: u8, + pub(crate) mask_selector: u8, +} + +/// Encode the exact raw SKSM kick fields, leaving bits [63:47] clear. +pub(crate) const fn encode_kick_word(timestamp: u64, queue_id: u8) -> Result { + if timestamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + if queue_id > KICK_QID_MASK { + return Err(SubmissionError::QueueIdOutOfRange); + } + Ok(timestamp | ((queue_id as u64) << KICK_QID_SHIFT)) +} + +/// Decode a kick word with the firmware's exact bitfields, refusing the +/// unmodeled upper bits. Kept parallel to `g17_rtkit::KickWord::decode_strict`. +pub(crate) const fn decode_kick_word(word: u64) -> Result { + if word & KICK_RESERVED_MASK != 0 { + return Err(SubmissionError::KickWordUnmodeledBits); + } + Ok(SksmKickWord { + timestamp: word & KICK_TS_MASK, + queue_id: ((word >> KICK_QID_SHIFT) as u8) & KICK_QID_MASK, + mask_selector: ((word >> KICK_SELECTOR_BIT) & 1) as u8, + }) +} + +// --------------------------------------------------------------------------- +// 5. Host SKSM kick-queue producer transaction (AGX3-specific). +// +// Grounded from G17S KSMKickQueue::encodeKickEntry at host text +// 0xfffffe0008b51f74-0xfffffe0008b521c4. The current timestamp at queue +// +0x48 selects `entry_base + low8(timestamp) * entry_stride`; the current +// 40-bit timestamp is returned for the doorbell, then the low-byte slot is +// advanced modulo runtime `num_stamps`. Wrapping the slot to zero increments +// the 32-bit stamp in bits [39:8]. This coupling is load-bearing: choosing a +// different entry slot from the one encoded in the doorbell desynchronizes +// the queue and can reach the firmware's out-of-order fatal path. +// --------------------------------------------------------------------------- + +/// Stamp sequence in bits [39:8] of a host SKSM timestamp. +pub(crate) const KICK_STAMP_SHIFT: u32 = 8; +pub(crate) const KICK_STAMP_MASK: u64 = u32::MAX as u64; +/// Entry/stamp slot in bits [7:0] of a host SKSM timestamp. +pub(crate) const KICK_STAMP_INDEX_MASK: u64 = 0xff; +/// The low-byte slot representation permits at most 256 runtime stamps. +pub(crate) const KICK_NUM_STAMPS_MAX: u16 = 256; + +/// Decoded host-side timestamp state before a kick is published. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct KickTimestamp { + pub(crate) stamp: u32, + pub(crate) stamp_index: u8, +} + +/// Pure result of preparing one host SKSM kick-queue producer transaction. +/// +/// The caller must still encode the complete target-specific entry, execute +/// the recovered `dmb ish`, update the submitted counter, and perform the live +/// target-specific `addKicks` MMIO publication. This structure only makes the +/// entry selection, queue identity, and next producer timestamp indivisible. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedKickSubmission { + /// Slot selected from the current timestamp's low byte. + pub(crate) entry_index: u8, + /// Byte offset from the runtime queue entry base. + pub(crate) entry_offset: u32, + /// Current 40-bit timestamp consumed by this entry. + pub(crate) timestamp: u64, + /// Queue whose target-specific `addKicks` tag must be published later. + pub(crate) queue_id: u8, + /// Timestamp stored at queue +0x48 for the following submission. + pub(crate) next_timestamp: u64, +} + +/// Decode the exact host timestamp split: stamp[31:0] followed by slot[7:0]. +pub(crate) const fn decode_kick_timestamp( + timestamp: u64, +) -> Result { + if timestamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + + Ok(KickTimestamp { + stamp: ((timestamp >> KICK_STAMP_SHIFT) & KICK_STAMP_MASK) as u32, + stamp_index: (timestamp & KICK_STAMP_INDEX_MASK) as u8, + }) +} + +/// Encode the host timestamp layout without a queue ID. +pub(crate) const fn encode_kick_timestamp(timestamp: KickTimestamp) -> u64 { + ((timestamp.stamp as u64) << KICK_STAMP_SHIFT) | timestamp.stamp_index as u64 +} + +/// Low-byte payload of the retained render channel's next submission. +/// +/// In the inspected 25G83 kext, FRG submit increments channel +0x204 +/// (8837ae0..e8); getter 88b4ee8 returns its low byte. The counter follows +/// channel-object lifetime, independently of its physical timestamp base. +/// Linux retains one ordinal per physical pair across its storage handoffs, +/// advancing once per publication; outstanding submissions need not retire +/// before this counter advances. The byte wraps independently of the 40-bit +/// SKSM timestamp and is neither a QID nor an address-space tag. +pub(crate) const fn g17p_serialized_render_scratch_payload(submission_ordinal: u64) -> u8 { + submission_ordinal.wrapping_add(1) as u8 +} + +/// Pack `AGX3DCommandDescriptor::getSKSMCompletionScratchRegValue()`. +/// +/// The kext stores this value in both its host descriptor and the emitted GPU +/// command at +0x2160. Bits [39:0] are the command's SKSM queue timestamp and +/// bits [47:40] are the 3D-channel scratch payload. The native full-command +/// witness `0x0000010000000f76` therefore decodes as payload one and timestamp +/// `0xf76`; it is categorically not a USC FList pointer. +pub(crate) const fn encode_g17p_3d_completion_scratch( + timestamp: u64, + payload: u8, +) -> Result { + if timestamp & !KICK_TS_MASK != 0 { + Err(SubmissionError::KickTimestampOutOfRange) + } else { + Ok(timestamp | ((payload as u64) << KICK_QID_SHIFT)) + } +} + +/// ConfigUpdate +0x26 gates the u64 completion seed at +0x2a (B1 +/// 11434..11470). Busy channels preserve firmware's evolving scratch value. +/// Native emits both fields with u32/u64 stores; validate before mutation. +pub(crate) fn apply_g17p_idle_scratch_seed( + out: &mut [u8], seed: Option, +) -> Result<(), SubmissionError> { + if out.len() < 0x32 { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + put_u32_le(out, 0x26, u32::from(seed.is_some())); + put_u64_le(out, 0x2a, seed.unwrap_or(0)); + Ok(()) +} + +/// Prepare the exact entry-slot/doorbell/next-timestamp tuple for one kick. +/// +/// `num_stamps` and `entry_stride` are intentionally runtime inputs: Apple's +/// host loads both from the per-queue geometry rather than hard-coding them. +/// Invalid geometry returns an error before an entry address or doorbell is +/// produced. No shared memory, counter, barrier, MMIO, or endpoint state is +/// mutated here. +pub(crate) const fn prepare_kick_submission( + current_timestamp: u64, + queue_id: u8, + num_stamps: u16, + entry_stride: u32, +) -> Result { + let current = match decode_kick_timestamp(current_timestamp) { + Ok(timestamp) => timestamp, + Err(err) => return Err(err), + }; + + if num_stamps == 0 || num_stamps > KICK_NUM_STAMPS_MAX { + return Err(SubmissionError::KickStampCountOutOfRange); + } + if current.stamp_index as u16 >= num_stamps { + return Err(SubmissionError::KickStampIndexOutOfRange); + } + if entry_stride == 0 { + return Err(SubmissionError::KickEntryStrideZero); + } + + let index = current.stamp_index as u32; + if index != 0 && entry_stride > u32::MAX / index { + return Err(SubmissionError::KickEntryOffsetOverflow); + } + let entry_offset = index * entry_stride; + + if queue_id > KICK_QID_MASK as u8 { + return Err(SubmissionError::QueueIdOutOfRange); + } + + let next_index = ((current.stamp_index as u16 + 1) % num_stamps) as u8; + let next_stamp = if next_index == 0 { + current.stamp.wrapping_add(1) + } else { + current.stamp + }; + let next_timestamp = encode_kick_timestamp(KickTimestamp { + stamp: next_stamp, + stamp_index: next_index, + }); + + Ok(PreparedKickSubmission { + entry_index: current.stamp_index, + entry_offset, + timestamp: current_timestamp, + queue_id, + next_timestamp, + }) +} + +// --------------------------------------------------------------------------- +// 5a. A18 Pro / G17P CL kick-entry body and host queue-state transition. +// +// `AGXKSMKickQueueG16::encodeCLKickEntry` at +// 0xfffffe0008a56848-0xfffffe0008a56a64 gathers the CL descriptor inputs. +// Its shared encoder at 0xfffffe0008a56a68-0xfffffe0008a56eb4 writes the +// complete 0x180-byte body described below. The recovered G17S encoder at +// 0xfffffe0008b51f74-0xfffffe0008b521c4 independently confirms the slot, +// timestamp, barrier, and counter tail; scratch tags remain target-specific. +// +// The body is the first 0x180 bytes of each 0x200-spaced queue-context +// record captured by m1n1. The host zeroes the runtime stride before these +// stores, so every byte not named here stays zero. +// --------------------------------------------------------------------------- + +pub(crate) const G17P_CL_KICK_ENTRY_SIZE: usize = 0x180; +pub(crate) const G17P_CL_DEPENDENCIES_MAX: usize = 32; +pub(crate) const G17P_CL_EXPLICIT_DEPENDENCIES_MAX: usize = G17P_CL_DEPENDENCIES_MAX - 1; +const G17P_CL_PACKED_ADDRESS_MASK: u64 = (1u64 << 38) - 1; + +/// Proven allocation and timestamp geometry for one target/backend pair. +/// +/// Only A18 Pro's G17P HAL200 hybrid has an admitted tuple. Classic and +/// HAL300 remain deliberately absent until their own host contracts are +/// recovered. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17SksmQueueGeometry { + target: G17ChannelAbiTarget, + backend: G17ClSubmissionBackend, + allocation_alignment: u64, + entry_stride: u32, + stamp_count: u16, + timestamp_seed: u64, + backing_size: usize, +} + +impl G17SksmQueueGeometry { + pub(crate) const fn validate(self) -> Result<(), SubmissionError> { + if !matches!( + (self.target, self.backend), + ( + G17ChannelAbiTarget::A18ProG17P, + G17ClSubmissionBackend::G17PHal200Hybrid + ) + ) { + return Err(SubmissionError::KickQueueGeometryUnconfirmed); + } + if self.allocation_alignment == 0 + || self.allocation_alignment & (self.allocation_alignment - 1) != 0 + || self.entry_stride < G17P_CL_KICK_ENTRY_SIZE as u32 + || self.stamp_count == 0 + || self.stamp_count > KICK_NUM_STAMPS_MAX + || self.timestamp_seed & !KICK_TS_MASK != 0 + || (self.timestamp_seed & KICK_STAMP_INDEX_MASK) >= self.stamp_count as u64 + || self.backing_size as u64 + != self.entry_stride as u64 * self.stamp_count as u64 + || self.backing_size as u64 % self.allocation_alignment != 0 + { + return Err(SubmissionError::KickQueueGeometryInvalid); + } + Ok(()) + } + + pub(crate) const fn allocation_alignment(self) -> u64 { + self.allocation_alignment + } + + pub(crate) const fn mapping_alignment(self, minimum: u64) -> u64 { + if self.allocation_alignment > minimum { + self.allocation_alignment + } else { + minimum + } + } + + pub(crate) const fn entry_stride(self) -> u32 { + self.entry_stride + } + + pub(crate) const fn stamp_count(self) -> u16 { + self.stamp_count + } + + pub(crate) const fn timestamp_seed(self) -> u64 { + self.timestamp_seed + } + + pub(crate) const fn backing_size(self) -> usize { + self.backing_size + } +} + +pub(crate) const T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY: G17SksmQueueGeometry = + G17SksmQueueGeometry { + target: G17ChannelAbiTarget::A18ProG17P, + backend: G17ClSubmissionBackend::G17PHal200Hybrid, + allocation_alignment: 0x100, + entry_stride: 0x200, + stamp_count: 0x100, + timestamp_seed: 1, + backing_size: 0x20000, + }; + +pub(crate) const fn g17_sksm_queue_geometry( + target: G17ChannelAbiTarget, + backend: G17ClSubmissionBackend, +) -> Option { + match (target, backend) { + (G17ChannelAbiTarget::A18ProG17P, G17ClSubmissionBackend::G17PHal200Hybrid) => { + Some(T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY) + } + _ => None, + } +} + +const _: () = assert!(T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.entry_stride as usize + * T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.stamp_count as usize + == T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.backing_size); +const _: () = assert!(T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.entry_stride as usize + >= G17P_CL_KICK_ENTRY_SIZE); +const _: () = assert!(T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.allocation_alignment == 0x100); +const _: () = assert!(T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.timestamp_seed == 1); + +/// Host-object fields used while selecting, publishing, and retiring one CL +/// kick entry. These are CPU-object offsets, not firmware queue-record offsets. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClKickQueueStateLayout { + pub(crate) queue_id: usize, + pub(crate) num_stamps: usize, + pub(crate) entry_stride: usize, + pub(crate) completion_selector: usize, + pub(crate) submitted: usize, + pub(crate) current_timestamp: usize, + pub(crate) last_add_kicks_timestamp: usize, + pub(crate) entry_cpu_base: usize, + pub(crate) entry_gpu_base: usize, + pub(crate) base_timestamp: usize, + pub(crate) previous_valid: usize, + pub(crate) previous_timestamp: usize, +} + +pub(crate) const G17P_CL_KICK_QUEUE_STATE_LAYOUT: G17PClKickQueueStateLayout = + G17PClKickQueueStateLayout { + queue_id: 0x0c, + num_stamps: 0x10, + entry_stride: 0x14, + completion_selector: 0x1c, + submitted: 0x44, + current_timestamp: 0x48, + last_add_kicks_timestamp: 0x50, + entry_cpu_base: 0x78, + entry_gpu_base: 0x80, + base_timestamp: 0x90, + previous_valid: 0x98, + previous_timestamp: 0xa0, + }; + +/// Fixed byte positions inside the 0x180-byte G17P CL entry body. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClKickEntryLayout { + pub(crate) header: usize, + pub(crate) mcache: usize, + pub(crate) payload: [usize; 2], + pub(crate) dependency_control: usize, + pub(crate) dependencies: usize, + pub(crate) event_mask: [usize; 4], + pub(crate) rce_kind: usize, + pub(crate) rce_bindings: [usize; 4], + pub(crate) zero_170: usize, + pub(crate) auxiliary: usize, +} + +pub(crate) const G17P_CL_KICK_ENTRY_LAYOUT: G17PClKickEntryLayout = G17PClKickEntryLayout { + header: 0x00, + mcache: 0x08, + payload: [0x10, 0x18], + dependency_control: 0x20, + dependencies: 0x28, + event_mask: [0x128, 0x130, 0x138, 0x140], + rce_kind: 0x148, + rce_bindings: [0x150, 0x158, 0x160, 0x168], + zero_170: 0x170, + auxiliary: 0x178, +}; + +/// One explicit barrier dependency after its descriptor-selected shared stamp +/// has been loaded. `event_word` is barrier-record word zero; record word one +/// selects the shared-stamp source and has no independent wire field. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClBarrierDependency { + /// Already converted native predecessor stamp. Never reinterpret it using + /// the consumer queue's base timestamp or ring geometry. + pub(crate) native_timestamp: Option, + pub(crate) event_word: u32, + pub(crate) shared_stamp: u64, + pub(crate) queue_id: u8, +} + +/// Represent a render dependency using the native event word plus epoch. +/// The event value advances by 0x100 and wraps as on M1/M2. Its high timestamp +/// bits are carried separately; dropping them makes a dependency at 2^24 +/// name the wrong kick even if AddKicks itself preserves all forty bits. +pub(crate) fn g17p_render_barrier_from_timestamp( + timestamp: u64, + queue_id: u8, +) -> Result { + if timestamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + if queue_id > KICK_QID_MASK { + return Err(SubmissionError::KickDependencyQueueIdOutOfRange); + } + Ok(G17PClBarrierDependency { + native_timestamp: None, + event_word: (timestamp as u32).wrapping_shl(8), + shared_stamp: timestamp >> 24, + queue_id, + }) +} + +/// Logical render prefix, resolved at ioctl admission before asynchronous work. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] +pub(crate) struct G17PRenderOrder { + pub(crate) sequence: u64, + pub(crate) wait_through: Option, +} + +/// Zero snapshots all preceding ioctls; N adds the first N renders of this +/// ioctl. NONE remains independent. Sequence zero is reserved for internal work. +pub(crate) fn g17p_render_order( + ioctl_base: u64, preceding: u64, barrier: u16, +) -> Result { + let sequence = ioctl_base.checked_add(preceding) + .and_then(|n| n.checked_add(1)) + .ok_or(SubmissionError::RenderBarrierSequenceOverflow)?; + let wait_through = if barrier == u16::MAX { + None + } else { + if u64::from(barrier) > preceding { + return Err(SubmissionError::RenderBarrierFuture); + } + Some(ioctl_base.checked_add(u64::from(barrier)) + .ok_or(SubmissionError::RenderBarrierSequenceOverflow)?) + }; + Ok(G17PRenderOrder { sequence, wait_through }) +} + +/// Add an exact historical prefix without broadening it to the ioctl base. +/// The base is frozen for the whole ioctl, before any of its jobs publish; +/// even an earlier command in this ioctl cannot satisfy a historical ordinal. +/// Zero is absent, including when the legacy header is NONE. +pub(crate) fn g17p_render_order_with_prior( + ioctl_base: u64, preceding: u64, barrier: u16, prior: u64, +) -> Result { + if prior > ioctl_base { + return Err(SubmissionError::PriorBarrierFuture); + } + let mut order = g17p_render_order(ioctl_base, preceding, barrier)?; + if prior != 0 { + order.wait_through = Some(order.wait_through.map_or(prior, |p| p.max(prior))); + } + Ok(order) +} + +/// Resolve a still-owned predecessor. A successful retirement needs no GPU +/// dependency; errors and an unassigned token must never satisfy the prefix. +/// The caller filters owner identity and supplies every live predecessor in +/// the prefix under the same runtime lock as publication and retirement. +pub(crate) fn g17p_live_render_dependency( + order: G17PRenderOrder, predecessor_sequence: u64, + predecessor_status: i32, native_word: u64, +) -> Result, SubmissionError> { + if !g17p_render_prefix_contains(order.wait_through, predecessor_sequence) { + return Ok(None); + } + if predecessor_status < 0 { + return Err(SubmissionError::RenderBarrierPredecessorFailed); + } + if predecessor_status > 0 { return Ok(None); } + if native_word == u64::MAX { + return Err(SubmissionError::RenderBarrierMissingToken); + } + // Bit 46 is QID bit 6: the firmware takes the QID as bits [46:40] (ubfx + // x1,x5,#40,#7) and the same bit as its valid-queue mask-word selector + // (ubfx x6,x5,#46,#1). A predecessor on QID 64..127 sets it; its token is + // fully modeled and the dependency encoder takes all seven QID bits. + // decode_kick_word still refuses bits [63:47]. Refusing the selector made + // every barrier on a 64..127 fragment QID fail the dependent render with + // EIO (the dead-pair DeviceLost). + let token = decode_kick_word(native_word)?; + let mut dependency = g17p_render_barrier_from_timestamp(token.timestamp, token.queue_id)?; + dependency.native_timestamp = Some(token.timestamp); + Ok(Some(dependency)) +} + +pub(crate) fn g17p_render_prefix_contains(prefix: Option, sequence: u64) -> bool { + sequence != 0 && prefix.is_some_and(|limit| sequence <= limit) +} + +/// Collapse a validated logical prefix to its newest kick per hardware QID. +/// Each QID's implicit previous-kick parent supplies the transitive order. +/// Use logical sequence, not wrapped firmware timestamps or scan order, to +/// choose the newest. Callers must check every covered predecessor for errors +/// before inserting it, including predecessors this frontier will subsume. +pub(crate) fn g17p_insert_prefix_dependency( + dependencies: &mut [G17PClBarrierDependency], + sequences: &mut [u64], count: &mut usize, + sequence: u64, dependency: G17PClBarrierDependency, +) -> Result<(), SubmissionError> { + if *count > dependencies.len() || *count > sequences.len() { + return Err(SubmissionError::KickDependencyCountOutOfRange); + } + if let Some(index) = dependencies[..*count].iter() + .position(|prior| prior.queue_id == dependency.queue_id) + { + if sequence > sequences[index] { + dependencies[index] = dependency; + sequences[index] = sequence; + } + return Ok(()); + } + if *count == dependencies.len() || *count == sequences.len() { + return Err(SubmissionError::KickDependencyCountOutOfRange); + } + dependencies[*count] = dependency; + sequences[*count] = sequence; + *count += 1; + Ok(()) +} + +/// Optional MCache aperture descriptor copied into entry +0x08. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClMcacheAperture { + pub(crate) address: u64, + pub(crate) index: u8, + pub(crate) count: u8, +} + +/// One of the four address/tag bindings written at +0x150..+0x168. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClRceBinding { + pub(crate) address: u64, + pub(crate) tag: u8, +} + +/// Command-relative CL fields witnessed on consecutive output-positive +/// compute entries. The descriptor slot stride remains a storage decision; +/// both RCE addresses are derived from the current slot's low alias. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PComputeClCommandState { + pub(crate) event_mask: [u64; 4], + pub(crate) rce_bindings: [G17PClRceBinding; 4], +} + +/// Host-written prefix of the conditional tag-14 fallback command. +pub(crate) const G17P_COMPUTE_PRIMARY_EVENT_SIZE: usize = 0x15; +/// Host-written prefix of the descriptor-selected tag-16 command. +pub(crate) const G17P_COMPUTE_OPTIONAL_EVENT_SIZE: usize = 0x12; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PComputePrimaryEvent { + pub(crate) queue_id: u16, + pub(crate) descriptor_generation: u16, + pub(crate) data_master: u8, + pub(crate) old_timestamp: u64, +} + +/// Descriptor accounting and bytes prepared only for the tag-14 fallback. +/// Direct-feature G17P skips this transport command. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PComputePrimaryEvent { + pub(crate) descriptor_generation: u32, + pub(crate) bytes: [u8; G17P_COMPUTE_PRIMARY_EVENT_SIZE], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PComputeOptionalEvent { + pub(crate) queue_id: u16, + pub(crate) old_timestamp: u64, +} + +/// Encode the tag-14 fallback command used when the direct feature is absent. +pub(crate) fn encode_g17p_compute_primary_event( + event: G17PComputePrimaryEvent, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() < G17P_COMPUTE_PRIMARY_EVENT_SIZE { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + out[..G17P_COMPUTE_PRIMARY_EVENT_SIZE].fill(0); + put_u32_le(out, 0x00, 0x0e); + put_u16_le(out, 0x04, event.queue_id); + put_u16_le(out, 0x06, event.descriptor_generation); + out[0x08] = event.data_master; + put_u64_le(out, 0x09, event.old_timestamp); + put_u32_le(out, 0x11, 2); + Ok(()) +} + +/// Advance the host descriptor's submitted-entry count for one CL entry. +pub(crate) fn advance_g17p_compute_descriptor_generation( + generation_before: u32, + nop_count: u32, +) -> Result { + generation_before + .checked_add(nop_count) + .and_then(|generation| generation.checked_add(1)) + .ok_or(SubmissionError::ComputeEventCounterOverflow) +} + +/// Prepare the tag-14 fallback bytes. Required NOPs are accounted before the +/// one CL entry; the direct-feature path advances the same generation but +/// does not call this fallback encoder or publish its pointer. +pub(crate) fn prepare_g17p_compute_primary_event( + generation_before: u32, + nop_count: u32, + queue_id: u8, + data_master: u8, + old_timestamp: u64, +) -> Result { + let descriptor_generation = + advance_g17p_compute_descriptor_generation(generation_before, nop_count)?; + let event_generation = u16::try_from(descriptor_generation) + .map_err(|_| SubmissionError::ComputeEventCounterOverflow)?; + let mut bytes = [0u8; G17P_COMPUTE_PRIMARY_EVENT_SIZE]; + encode_g17p_compute_primary_event( + G17PComputePrimaryEvent { + queue_id: u16::from(queue_id), + descriptor_generation: event_generation, + data_master, + old_timestamp, + }, + &mut bytes, + )?; + Ok(PreparedG17PComputePrimaryEvent { + descriptor_generation, + bytes, + }) +} + +/// Host-written extent of the tag-14 `KsmKickQueueAddKicks` channel command. +/// +/// `AGXCLChannelSKSM::submitBuffer` writes exactly these 0x15 bytes inline at +/// mac17p `0xfffffe00089fa39c-0xfffffe00089fa3cc`, then publishes the record's +/// GPU address through `AGXChannel::writeChannelCommandPointer`. +pub(crate) const G17P_KSM_ADD_KICKS_COMMAND_SIZE: usize = 0x15; +/// Bytes the matched J700 A000 handler cache-invalidates before reading the +/// record (`mov w1, #0x18` at `0xfffffc0000011714`). The three bytes past the +/// host-written extent are never loaded, but must not be stale. +pub(crate) const G17P_KSM_ADD_KICKS_COMMAND_FETCH_SIZE: usize = 0x18; + +/// Fields the native host copies into one tag-14 AddKicks record. +/// +/// Native sources, all from `submitBuffer`: +/// * `queue_id` = `AGXKSMKickQueue[0x0c]`, the QID. +/// * `add_count` = `AGXCLCommandDescriptor[0x2e4]`, the number of kick +/// entries this submission published (NOPs plus the one CL entry). It is the +/// same value the suppressed MMIO variant passes to +/// `AGXKSMKickQueueG16::addKicks`. +/// * `queue_priority`= `AGXKSMKickQueue[0x1c]`, i.e. `AGXChannel::getPriority()`. +/// This queue's value is the retained completion selector. +/// * `entry_stamp` = the queue's producer stamp **before** the CL entry +/// advanced it, returned by `AGXKSMKickQueueG16::encodeKickEntry` +/// (`ldr x0, [x19, #0x48]` at `0xfffffe0008a56e00`). Its low byte is the ring +/// entry index; A000 reads it only for its trace event. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmAddKicksCommand { + pub(crate) queue_id: u8, + pub(crate) add_count: u32, + pub(crate) queue_priority: u8, + pub(crate) entry_stamp: u64, +} + +/// Bytes for one tag-14 record, ready to copy into retained queue storage. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PKsmAddKicks { + pub(crate) bytes: [u8; G17P_KSM_ADD_KICKS_COMMAND_SIZE], +} + +/// Build the tag-14 `KsmKickQueueAddKicks` channel command. +/// +/// The record replaces the host SKSM-port `addKicks` pair on J700, where every +/// host access to `sgx 0x0..0x2ffff` takes an asynchronous SError. The matched +/// A000 firmware dispatches type 14 to `0xfffffc0000011710`, which writes the +/// QID to `sgx+0x103a8` and this record's `add_count` to `sgx+0x21018` through +/// the firmware's own mapping of the block. +/// +/// The A000 handler loads only `+0x04`, `+0x06` and `+0x09`; `+0x08` and `+0x11` +/// are written for host parity. +pub(crate) fn prepare_g17p_ksm_add_kicks_command( + command: G17PKsmAddKicksCommand, +) -> Result { + if command.queue_id > KICK_QID_MASK { + return Err(SubmissionError::QueueIdOutOfRange); + } + if command.queue_priority > 3 { + return Err(SubmissionError::KickQueueCompletionSelectorOutOfRange); + } + if command.entry_stamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + if command.add_count == 0 { + return Err(SubmissionError::KsmAddKicksCountZero); + } + let add_count = match u16::try_from(command.add_count) { + Ok(add_count) => add_count, + Err(_) => return Err(SubmissionError::KsmAddKicksCountOutOfRange), + }; + + let mut bytes = [0u8; G17P_KSM_ADD_KICKS_COMMAND_SIZE]; + put_u32_le( + &mut bytes, + 0x00, + G17PTopLevelCommandType::KsmKickQueueAddKicks as u32, + ); + put_u16_le(&mut bytes, 0x04, u16::from(command.queue_id)); + put_u16_le(&mut bytes, 0x06, add_count); + bytes[0x08] = command.queue_priority; + put_u64_le(&mut bytes, 0x09, command.entry_stamp); + put_u32_le(&mut bytes, 0x11, G17P_COMPUTE_DATA_MASTER); + Ok(PreparedG17PKsmAddKicks { bytes }) +} + +/// Encode tag 16 only when the host descriptor carries its optional pointer. +/// The simple direct first add3 profile does not publish this record. +pub(crate) fn encode_g17p_compute_optional_event( + event: G17PComputeOptionalEvent, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() < G17P_COMPUTE_OPTIONAL_EVENT_SIZE { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + out[..G17P_COMPUTE_OPTIONAL_EVENT_SIZE].fill(0); + put_u32_le(out, 0x00, 0x10); + put_u16_le(out, 0x04, event.queue_id); + put_u64_le(out, 0x06, event.old_timestamp); + put_u32_le(out, 0x0e, 0xfe); + Ok(()) +} + +pub(crate) fn prepare_g17p_compute_cl_command_state( + descriptor_low: u64, + submission_ordinal: u64, + primary_checkpoint: u8, +) -> Result { + let next = submission_ordinal + .checked_add(1) + .ok_or(SubmissionError::ComputeEventCounterOverflow)?; + let event_previous = u64::from(submission_ordinal) + .checked_mul(2) + .ok_or(SubmissionError::ComputeEventCounterOverflow)?; + let event_current = u64::from(next) + .checked_mul(4) + .ok_or(SubmissionError::ComputeEventCounterOverflow)?; + let rce_a = descriptor_low + .checked_add(0x40) + .ok_or(SubmissionError::ComputeRceAddressOverflow)?; + let rce_b = descriptor_low + .checked_add(0x760) + .ok_or(SubmissionError::ComputeRceAddressOverflow)?; + let empty = G17PClRceBinding { address: 0, tag: 0 }; + Ok(G17PComputeClCommandState { + event_mask: [0, event_previous, event_current, 0], + rce_bindings: [ + G17PClRceBinding { + address: rce_a, + tag: primary_checkpoint, + }, + G17PClRceBinding { + address: rce_b, + tag: 0x04, + }, + empty, + empty, + ], + }) +} + +/// Select the descriptor-embedded RegisterArrays consumed by a native render +/// kick. 3D's four complete arrays start at command +0xa0 and follow at the +/// 0x720-byte stride present in every working capture. +pub(crate) fn prepare_g17p_render_cl_command_state( + descriptor_low: u64, + tiling: bool, +) -> Result { + const FRAGMENT_RCE_PROGRAM_STRIDE: u64 = 0x720; + const FRAGMENT_RCE_PROGRAM_OFFSET: u64 = 0xa0; + let binding = |offset: u64, tag: u8| { + descriptor_low + .checked_add(offset) + .map(|address| G17PClRceBinding { address, tag }) + .ok_or(SubmissionError::ComputeRceAddressOverflow) + }; + let empty = G17PClRceBinding { address: 0, tag: 0 }; + + let rce_bindings = if tiling { + [binding(0x60, 0x47)?, empty, empty, empty] + } else { + let fragment_rce = descriptor_low + .checked_add(FRAGMENT_RCE_PROGRAM_OFFSET) + .ok_or(SubmissionError::ComputeRceAddressOverflow)?; + let external_binding = |index: u64, tag: u8| { + fragment_rce + .checked_add(index * FRAGMENT_RCE_PROGRAM_STRIDE) + .map(|address| G17PClRceBinding { address, tag }) + .ok_or(SubmissionError::ComputeRceAddressOverflow) + }; + let full = [ + external_binding(0, 0x56)?, + external_binding(1, 0x10)?, + external_binding(2, 0x17)?, + external_binding(3, 0x0a)?, + ]; + match crate::g18::fw::config::RENDER_3D_RCE { + 0 => [empty, empty, empty, empty], + 1 => [full[0], empty, empty, empty], + 2 => [full[0], full[1], empty, empty], + 3 => [full[0], full[1], full[2], empty], + _ => full, + } + }; + + Ok(G17PComputeClCommandState { + event_mask: [0; 4], + rce_bindings, + }) +} + +/// Physical-queue QoS fields packed above the 32-bit dependency mask at entry +/// +0x20. These remain with the registered SKSM queue when later descriptors +/// publish new per-command event masks. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClQosConfig { + pub(crate) queue_byte_32: u8, + pub(crate) class_40: u8, + pub(crate) word_48: u16, +} + +/// Physical queue configuration and mutable producer state read before one CL +/// entry. The QoS fields belong to this retained queue, while the event mask in +/// [`G17PClKickEntryOperands`] belongs to the command being submitted. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClKickProducerState { + pub(crate) queue_id: u8, + pub(crate) geometry: G17SksmQueueGeometry, + pub(crate) completion_selector: u8, + pub(crate) qos: G17PClQosConfig, + pub(crate) submitted: u32, + pub(crate) current_timestamp: u64, + /// Last timestamp reconciled through `submitCLChannel` addKicks. + pub(crate) last_add_kicks_timestamp: u64, + pub(crate) base_timestamp: u64, + pub(crate) previous_valid: bool, + pub(crate) previous_timestamp: u64, +} + +impl G17SksmQueueGeometry { + pub(crate) const fn initial_producer_state( + self, + queue_id: u8, + completion_selector: u8, + qos: G17PClQosConfig, + ) -> Result { + match self.validate() { + Ok(()) => {} + Err(error) => return Err(error), + } + if queue_id > KICK_QID_MASK { + return Err(SubmissionError::QueueIdOutOfRange); + } + if completion_selector > 3 { + return Err(SubmissionError::KickQueueCompletionSelectorOutOfRange); + } + Ok(G17PClKickProducerState { + queue_id, + geometry: self, + completion_selector, + qos, + submitted: 0, + current_timestamp: self.timestamp_seed, + last_add_kicks_timestamp: self.timestamp_seed, + base_timestamp: self.timestamp_seed, + previous_valid: false, + previous_timestamp: 0, + }) + } +} + +/// Descriptor and call-site operands consumed by `encodeCLKickEntry`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClKickEntryOperands<'a> { + /// Descriptor +0x4c bit 0; true sets entry-header bit 1. + pub(crate) descriptor_flag_4c: bool, + /// Descriptor +0x5c8 bit 0; true sets entry-header bit 61. + pub(crate) descriptor_flag_5c8: bool, + /// Result of `convertEventMachineTSToHWTS` for a queue with no predecessor. + pub(crate) converted_command_timestamp: u64, + pub(crate) barriers: &'a [G17PClBarrierDependency], + pub(crate) mcache: Option, + /// The two qwords reached through `AGXKSMKickQueueEntryPayload`. + pub(crate) payload: [u64; 2], + /// Per-command mask copied independently to +0x128..+0x140. + pub(crate) event_mask: [u64; 4], + pub(crate) rce_kind: u8, + pub(crate) rce_bindings: [G17PClRceBinding; 4], + /// Descriptor +0x178 when nonzero, otherwise the descriptor-indexed host + /// table value selected by +0x61c. + pub(crate) auxiliary: u64, +} + +/// Pure result of one entry encode and the host state values published after +/// the entry barrier. No shared memory or MMIO is touched by this function. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PClKickEntry { + /// Bytes the live publisher must clear before copying `entry`. + pub(crate) zero_length: u32, + pub(crate) entry: [u8; G17P_CL_KICK_ENTRY_SIZE], + pub(crate) entry_index: u8, + pub(crate) entry_offset: u32, + pub(crate) current_timestamp: u64, + pub(crate) next_timestamp: u64, + pub(crate) submitted_before: u32, + pub(crate) submitted_after: u32, + pub(crate) previous_timestamp_after: u64, + /// Producer state to use for the next entry on this physical queue. + pub(crate) producer_after: G17PClKickProducerState, +} + +/// Ordering at the side-effect boundary of the host encoder. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PClKickPublicationStep { + ZeroEntryStride, + EntryBody, + InnerShareableBarrier, + NextTimestamp, + SubmittedAtomicAdd, + PreviousValid, + PreviousTimestamp, +} + +pub(crate) const G17P_CL_KICK_PUBLICATION_ORDER: [G17PClKickPublicationStep; 7] = [ + G17PClKickPublicationStep::ZeroEntryStride, + G17PClKickPublicationStep::EntryBody, + G17PClKickPublicationStep::InnerShareableBarrier, + G17PClKickPublicationStep::NextTimestamp, + G17PClKickPublicationStep::SubmittedAtomicAdd, + G17PClKickPublicationStep::PreviousValid, + G17PClKickPublicationStep::PreviousTimestamp, +]; + +/// G17P direct first-submit side-effect ledger. The tag/QID/SKSM/channel and +/// mailbox ordering is binary-verified against the native submit path. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PFirstSubmitPublicationStep { + AcquireGfx1RunningPower, + PublishB2, + PublishConfigUpdatePointer, + ConfigureQid4Hardware, + ZeroSksmStride, + WriteSksmEntry, + EntryBarrier, + PublishSksmProducer, + AdvanceDescriptorGeneration, + RecordHostCorrelationToken, + PublishEntrySignalPointer, + AddKicks, + PublishCl2, + SendActivation, + DirectKickBarrier, + SendDirectKick, +} + +pub(crate) const G17P_FIRST_SUBMIT_PUBLICATION_ORDER: [G17PFirstSubmitPublicationStep; 16] = [ + G17PFirstSubmitPublicationStep::AcquireGfx1RunningPower, + G17PFirstSubmitPublicationStep::PublishB2, + G17PFirstSubmitPublicationStep::PublishConfigUpdatePointer, + G17PFirstSubmitPublicationStep::ConfigureQid4Hardware, + G17PFirstSubmitPublicationStep::ZeroSksmStride, + G17PFirstSubmitPublicationStep::WriteSksmEntry, + G17PFirstSubmitPublicationStep::EntryBarrier, + G17PFirstSubmitPublicationStep::PublishSksmProducer, + G17PFirstSubmitPublicationStep::AdvanceDescriptorGeneration, + G17PFirstSubmitPublicationStep::RecordHostCorrelationToken, + G17PFirstSubmitPublicationStep::PublishEntrySignalPointer, + G17PFirstSubmitPublicationStep::AddKicks, + G17PFirstSubmitPublicationStep::PublishCl2, + G17PFirstSubmitPublicationStep::SendActivation, + G17PFirstSubmitPublicationStep::DirectKickBarrier, + G17PFirstSubmitPublicationStep::SendDirectKick, +]; + +fn encode_g17p_cl_dependency(timestamp: u64, queue_id: u8) -> Result { + if timestamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + if queue_id > KICK_QID_MASK as u8 { + return Err(SubmissionError::KickDependencyQueueIdOutOfRange); + } + Ok(timestamp | ((queue_id as u64) << KICK_QID_SHIFT)) +} + +fn encode_g17p_cl_barrier_dependency( + queue: G17PClKickProducerState, + barrier: G17PClBarrierDependency, +) -> Result { + if barrier.queue_id > KICK_QID_MASK as u8 { + return Err(SubmissionError::KickDependencyQueueIdOutOfRange); + } + + if let Some(timestamp) = barrier.native_timestamp { + return encode_g17p_cl_dependency(timestamp, barrier.queue_id); + } + + let stamp_count = queue.geometry.stamp_count() as u64; + let event_timestamp = (((barrier.event_word as u64) >> 8) + | barrier.shared_stamp.wrapping_shl(24)) + .wrapping_sub(1); + let event_stamp = event_timestamp / stamp_count; + let event_index = event_timestamp % stamp_count; + let index_sum = (queue.base_timestamp & KICK_STAMP_INDEX_MASK) + event_index; + let index_carry = index_sum / stamp_count; + let stamp = ((queue.base_timestamp >> KICK_STAMP_SHIFT) as u32) + .wrapping_add(event_stamp as u32) + .wrapping_add(index_carry as u32); + let index = (index_sum % stamp_count) as u8; + + encode_g17p_cl_dependency( + encode_kick_timestamp(KickTimestamp { + stamp, + stamp_index: index, + }), + barrier.queue_id, + ) +} + +fn encode_g17p_cl_mcache(aperture: G17PClMcacheAperture) -> Result { + if aperture.index > 0x3f || aperture.count == 0 || aperture.count > 0x40 { + return Err(SubmissionError::KickMcacheFieldOutOfRange); + } + if aperture.address & 0x1f != 0 || aperture.address >> 5 > G17P_CL_PACKED_ADDRESS_MASK { + return Err(SubmissionError::KickMcacheAddressUnencodable); + } + + Ok((aperture.index as u64) + | ((aperture.address >> 5) << 6) + | (((aperture.count as u64 - 1) & 0x3f) << 54)) +} + +fn encode_g17p_cl_rce(binding: G17PClRceBinding) -> Result { + if binding.tag > KICK_QID_MASK as u8 { + return Err(SubmissionError::KickRceTagOutOfRange); + } + if binding.address & 0x1f != 0 || binding.address >> 5 > G17P_CL_PACKED_ADDRESS_MASK { + return Err(SubmissionError::KickRceAddressUnencodable); + } + + Ok((binding.address >> 5) | ((binding.tag as u64) << 43)) +} + +/// Encode the complete G17P CL entry body and calculate the exact host state +/// transition that follows the recovered `dmb ish`. +pub(crate) fn prepare_g17p_cl_kick_entry( + queue: G17PClKickProducerState, + operands: G17PClKickEntryOperands<'_>, +) -> Result { + queue.geometry.validate()?; + if queue.completion_selector > 3 { + return Err(SubmissionError::KickQueueCompletionSelectorOutOfRange); + } + if queue.base_timestamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + if operands.barriers.len() > G17P_CL_EXPLICIT_DEPENDENCIES_MAX { + return Err(SubmissionError::KickDependencyCountOutOfRange); + } + if queue.qos.class_40 > 0x1f { + return Err(SubmissionError::KickQosClassOutOfRange); + } + // Native writes `(RCEInfo + 0x00) & 3` here. RCEInfo +0x04 is the active + // RegisterArray count (1 for TA, 4 for 3D), not this field. + if operands.rce_kind > 3 { + return Err(SubmissionError::KickRceKindOutOfRange); + } + + let prepared = prepare_kick_submission( + queue.current_timestamp, + queue.queue_id, + queue.geometry.stamp_count(), + queue.geometry.entry_stride(), + )?; + if queue.submitted >= queue.geometry.stamp_count() as u32 { + return Err(SubmissionError::KickQueueSubmittedFull); + } + + let mut entry = [0u8; G17P_CL_KICK_ENTRY_SIZE]; + let layout = G17P_CL_KICK_ENTRY_LAYOUT; + + let mut header = queue.current_timestamp << 2; + header |= (queue.queue_id as u64) << 42; + header |= (queue.completion_selector as u64) << 59; + if operands.descriptor_flag_4c { + header |= 1 << 1; + } + if let Some(aperture) = operands.mcache { + header |= 1 << 58; + put_u64_le(&mut entry, layout.mcache, encode_g17p_cl_mcache(aperture)?); + } + if operands.descriptor_flag_5c8 { + header |= 1 << 61; + } + put_u64_le(&mut entry, layout.header, header); + + for (offset, value) in layout.payload.into_iter().zip(operands.payload) { + put_u64_le(&mut entry, offset, value); + } + + for (index, barrier) in operands.barriers.iter().copied().enumerate() { + put_u64_le( + &mut entry, + layout.dependencies + index * 8, + encode_g17p_cl_barrier_dependency(queue, barrier)?, + ); + } + + let implicit_timestamp = if queue.previous_valid { + queue.previous_timestamp + } else { + operands.converted_command_timestamp.wrapping_sub(1) + }; + let implicit_index = operands.barriers.len(); + // DIAGNOSTIC. On a first submit this declares a self-dependency on stamp 0 + // (`converted_command_timestamp` is 1, minus one). Measured on hardware at + // the stall: the TA entry's dependency word is 0x0000050000000000, i.e. + // {qid 5, stamp 0}, while the hardware's completed register for that queue, + // `0x21028[5]`, reads 0x862cde4910 -- not zero, and not the firmware's + // "never completed = base-1 = 0" convention. Firmware assumes one value and + // the hardware register holds another, so this dependency may be + // unsatisfiable from cold. Omitting it isolates whether the SKSM front-end + // is waiting on it. + let omit_implicit = !queue.previous_valid + && crate::g18::fw::config::NO_IMPLICIT_DEP != 0; + if !omit_implicit { + put_u64_le( + &mut entry, + layout.dependencies + implicit_index * 8, + encode_g17p_cl_dependency(implicit_timestamp, queue.queue_id)?, + ); + } + + let dependency_count = implicit_index + usize::from(!omit_implicit); + let dependency_mask = if dependency_count == G17P_CL_DEPENDENCIES_MAX { + u32::MAX + } else if dependency_count == 0 { + 0 + } else { + (1u32 << dependency_count) - 1 + }; + let dependency_control = dependency_mask as u64 + | ((queue.qos.queue_byte_32 as u64) << 32) + | ((queue.qos.class_40 as u64) << 40) + | ((queue.qos.word_48 as u64) << 48); + put_u64_le(&mut entry, layout.dependency_control, dependency_control); + + for (offset, value) in layout.event_mask.into_iter().zip(operands.event_mask) { + put_u64_le(&mut entry, offset, value); + } + entry[layout.rce_kind] = operands.rce_kind; + for (offset, binding) in layout.rce_bindings.into_iter().zip(operands.rce_bindings) { + put_u64_le(&mut entry, offset, encode_g17p_cl_rce(binding)?); + } + put_u64_le(&mut entry, layout.auxiliary, operands.auxiliary); + + let mut producer_after = queue; + producer_after.current_timestamp = prepared.next_timestamp; + producer_after.submitted = queue.submitted.wrapping_add(1); + producer_after.previous_valid = true; + producer_after.previous_timestamp = prepared.timestamp; + + Ok(PreparedG17PClKickEntry { + zero_length: queue.geometry.entry_stride(), + entry, + entry_index: prepared.entry_index, + entry_offset: prepared.entry_offset, + current_timestamp: prepared.timestamp, + next_timestamp: prepared.next_timestamp, + submitted_before: queue.submitted, + submitted_after: producer_after.submitted, + previous_timestamp_after: prepared.timestamp, + producer_after, + }) +} + +/// Apply the completed-kick count used by `AGXCLCommandDescriptor::complete`. +/// The live callback performs an acquire-release atomic subtraction at queue +/// +0x44. This pure form keeps every other producer field unchanged. +pub(crate) fn complete_g17p_cl_kicks( + mut producer: G17PClKickProducerState, + completed_count: u32, +) -> Result { + if completed_count == 0 { + return Ok(producer); + } + if producer.submitted == 0 || completed_count > producer.submitted { + return Err(SubmissionError::KickCompletionUnderflow); + } + + producer.submitted -= completed_count; + Ok(producer) +} + +/// One range inside provider-owned queue storage. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClStorageRange { + pub(crate) offset: usize, + pub(crate) size: usize, +} + +/// The scheduler record at +0x200 aliases the registered queue's 16 KiB +/// shared-support page. A second full-page support write would erase it. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClRetainedStorageLayout { + pub(crate) shared_support: G17PClStorageRange, + pub(crate) scheduler: G17PClStorageRange, +} + +pub(crate) const G17P_CL_RETAINED_STORAGE_LAYOUT: G17PClRetainedStorageLayout = + G17PClRetainedStorageLayout { + shared_support: G17PClStorageRange { + offset: 0, + size: 0x4000, + }, + scheduler: G17PClStorageRange { + offset: 0x200, + size: 0x100, + }, + }; + +/// Required provider-owned storage for the measured pre-B2 state writes. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClB2PrimaryStateLayout { + pub(crate) object_size: usize, + pub(crate) computed_write_extent: usize, + pub(crate) region_1_write_extent: usize, + pub(crate) region_2_write_extent: usize, +} + +pub(crate) const G17P_CL_B2_PRIMARY_STATE_LAYOUT: G17PClB2PrimaryStateLayout = + G17PClB2PrimaryStateLayout { + object_size: 0x4000, + computed_write_extent: 0xe30, + region_1_write_extent: 0x20, + region_2_write_extent: 0x18, + }; + +pub(crate) const fn g17p_cl_ranges_overlap( + left: G17PClStorageRange, + right: G17PClStorageRange, +) -> bool { + let left_end = match left.offset.checked_add(left.size) { + Some(end) => end, + None => return false, + }; + let right_end = match right.offset.checked_add(right.size) { + Some(end) => end, + None => return false, + }; + left.offset < right_end && right.offset < left_end +} + +pub(crate) const fn g17p_cl_range_contains( + outer: G17PClStorageRange, + inner: G17PClStorageRange, +) -> bool { + let outer_end = match outer.offset.checked_add(outer.size) { + Some(end) => end, + None => return false, + }; + let inner_end = match inner.offset.checked_add(inner.size) { + Some(end) => end, + None => return false, + }; + outer.offset <= inner.offset && inner_end <= outer_end +} + +/// A whole-page support initializer is valid only before queue registration. +pub(crate) const fn g17p_cl_full_support_initialization_allowed(registered: bool) -> bool { + !registered +} + +/// The one compute private-memory pool, distinct from render's pool ID zero. +/// B1 indexes the same registration table with tag15 +0x46 and FList +0x08. +pub(crate) const G17P_COMPUTE_FLIST_ID: u16 = crate::g18::fw::config::COMPUTE_FLIST_ID; + +/// Initialize pool-owned state before its first firmware registration. +pub(crate) fn apply_g17p_cl_flist_static_state( + shared_support: &mut [u8], + support_state: &mut [u8], + page_list_gpu_va: u64, + run_table_gpu_va: u64, + support_state_gpu_va: u64, + pool_id: u16, +) -> Result<(), SubmissionError> { + if shared_support.len() < 0x4000 || support_state.len() < 0x4000 { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + shared_support.fill(0); + support_state.fill(0); + put_u64_le(shared_support, 0x00, 2); + put_u64_le(shared_support, 0x08, u64::from(pool_id)); + put_u64_le(shared_support, 0x10, 2); + put_u64_le(shared_support, 0x14, page_list_gpu_va); + put_u32_le(shared_support, 0x1c, 0x4_0000); + put_u64_le(shared_support, 0x20, 0x0000_1500_0000_0000); + put_u64_le(shared_support, 0x28, 0x0000_1500_0000_0000); + put_u64_le(shared_support, 0x30, run_table_gpu_va); + put_u64_le(shared_support, 0x40, 4); + put_u32_le(shared_support, 0x48, 0xa8); + put_u64_le(shared_support, 0x4c, support_state_gpu_va); + // B1 compares *FList[+0x4c] (submitted) with FList[+0x54] + // (completed). Both are pool-wide u64 counters, initially zero. + put_u64_le(shared_support, 0x54, 0); + put_u32_le(shared_support, 0x5c, 1); + put_u32_le(shared_support, 0x60, 2); + Ok(()) +} + +/// Historical captured per-queue fixture initializer. Production pool creation +/// uses `apply_g17p_cl_flist_static_state`; never call this on its shared pages. +pub(crate) fn apply_g17p_cl_queue_static_state( + shared_support: &mut [u8], + support_state: &mut [u8], + scheduler_state: &mut [u8], + page_list_gpu_va: u64, + run_table_gpu_va: u64, + support_state_gpu_va: u64, + scheduler_state_gpu_va: u64, +) -> Result<(), SubmissionError> { + if shared_support.len() < 0x4000 + || support_state.len() < 0x4000 + || scheduler_state.len() < 0x4000 + { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + + shared_support.fill(0); + support_state.fill(0); + scheduler_state.fill(0); + + put_u64_le(shared_support, 0x00, 2); + // AGXUSCPrivMemFList+0x08 is the pool ID copied into the firmware's + // internal allocator entry and ultimately written to gUPM's pool-select + // operation. All compute queues share this pool and UMA slot; QoS + // HardwareBufferIDs remain per queue. + put_u64_le(shared_support, 0x08, 1); + put_u64_le(shared_support, 0x10, 2); + // _AGFIUMAPagePoolState stores the full VA at unaligned +0x14. + // Copying the native +0x18 qword leaves compute naming render's pool. + put_u64_le(shared_support, 0x14, page_list_gpu_va); + put_u32_le(shared_support, 0x1c, 0x4_0000); + put_u64_le(shared_support, 0x20, 0x0000_1500_0000_0000); + put_u64_le(shared_support, 0x28, 0x0000_1500_0000_0000); + // Every queue names the run table that describes the shared backing. + // A private empty table here is incoherent with the shared pool and leaves + // overlapping spill allocation without the pool's initialized run map. + put_u64_le(shared_support, 0x30, run_table_gpu_va); + put_u64_le(shared_support, 0x40, 4); + put_u32_le(shared_support, 0x48, 0xa8); + put_u64_le(shared_support, 0x4c, support_state_gpu_va); + put_u32_le(shared_support, 0x54, 0); + put_u32_le(shared_support, 0x5c, 1); + put_u32_le(shared_support, 0x60, 2); + + let scheduler = G17P_CL_RETAINED_STORAGE_LAYOUT.scheduler.offset; + put_u64_le(shared_support, scheduler, scheduler_state_gpu_va + 8); + put_u32_le(shared_support, scheduler + 0x08, 1); + put_u32_le(shared_support, scheduler + 0x0c, 0); + put_u32_le(shared_support, scheduler + 0x10, 0x50); + put_u32_le(support_state, 0, 1); + put_u32_le(scheduler_state, 8, 1); + + Ok(()) +} + +/// Initialize the one channel-control page retained by physical CL_2. +pub(crate) fn apply_g17p_cl_channel_control_static_state( + channel_control: &mut [u8], +) -> Result<(), SubmissionError> { + if channel_control.len() < 0x40 { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + channel_control.fill(0); + + put_u64_le(channel_control, 0x00, 0x0000_0100_0000_ffff); + put_u64_le(channel_control, 0x20, 0x0002_0000_0000_0000); + put_u64_le(channel_control, 0x30, 0x0000_0000_ff00_0000); + Ok(()) +} + +/// Compatibility helper for byte-layout tests covering both ownership +/// domains. Production initializes the queue pages and CL_2 page separately. +pub(crate) fn apply_g17p_cl_retained_static_state( + shared_support: &mut [u8], + support_state: &mut [u8], + scheduler_state: &mut [u8], + channel_control: &mut [u8], + page_list_gpu_va: u64, + run_table_gpu_va: u64, + support_state_gpu_va: u64, + scheduler_state_gpu_va: u64, +) -> Result<(), SubmissionError> { + apply_g17p_cl_queue_static_state( + shared_support, + support_state, + scheduler_state, + page_list_gpu_va, + run_table_gpu_va, + support_state_gpu_va, + scheduler_state_gpu_va, + )?; + apply_g17p_cl_channel_control_static_state(channel_control) +} + +/// Apply the exact host-owned pre-B2 values to three provider-owned pages. +/// Bytes outside the listed fields remain unchanged. `channel_control_gpu_va` +/// comes from the retained Linux mapping, not the captured macOS address. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PClB2ActivationState { + Pending, + Published, +} + +pub(crate) fn apply_g17p_cl_b2_primary_state( + activation: G17PClB2ActivationState, + computed: &mut [u8], + region_1: &mut [u8], + region_2: &mut [u8], + channel_control_gpu_va: u64, +) -> Result { + if activation == G17PClB2ActivationState::Published { + return Err(SubmissionError::KickB2PrimaryStateAlreadyPublished); + } + let layout = G17P_CL_B2_PRIMARY_STATE_LAYOUT; + if computed.len() < layout.object_size + || region_1.len() < layout.object_size + || region_2.len() < layout.object_size + { + return Err(SubmissionError::KickB2PrimaryStateBufferTooSmall); + } + + let activation = apply_g17p_cl_b2_computed_state( + activation, + computed, + channel_control_gpu_va, + )?; + apply_g17p_cl_b2_descriptor_tables(region_1, region_2)?; + Ok(activation) +} + +/// Apply the session-wide CL_2 half of B2 activation. The pointer is pinned to +/// a dedicated bootstrap owner; logical queues have separate scheduler pages. +/// The exact consumer of this firmware-private slot remains unresolved. Never +/// retarget it during ordinary queue switching. PBDesc/UMA are separate state. +pub(crate) fn apply_g17p_cl_b2_computed_state( + activation: G17PClB2ActivationState, + computed: &mut [u8], + channel_control_gpu_va: u64, +) -> Result { + if activation == G17PClB2ActivationState::Published { + return Err(SubmissionError::KickB2PrimaryStateAlreadyPublished); + } + let layout = G17P_CL_B2_PRIMARY_STATE_LAYOUT; + if computed.len() < layout.object_size { + return Err(SubmissionError::KickB2PrimaryStateBufferTooSmall); + } + + computed[0x000] = 0xff; + computed[0x008] = 0xff; + computed[0x010..0x012].copy_from_slice(&0x0f01u16.to_le_bytes()); + computed[0x018..0x01a].copy_from_slice(&0x0f01u16.to_le_bytes()); + put_u32_le(computed, 0x408, 2); + put_u32_le(computed, 0x40c, 2); + put_u32_le(computed, 0x604, 4); + put_u64_le(computed, 0x808, channel_control_gpu_va); + put_u32_le(computed, 0xc04, 0x1000); + put_u64_le(computed, 0xe00, 7); + put_u64_le(computed, 0xe08, 0x0000_0001_0e22_618a); + put_u64_le(computed, 0xe20, 4); + put_u64_le(computed, 0xe28, 4); + + Ok(G17PClB2ActivationState::Published) +} + +/// Apply the device-global descriptor-table half of B2 initialization. +/// +/// Native performs this before client halLoadPBDesc/halLoadUMADesc updates. +/// Reapplying these captured cold entries for every later compute queue would +/// overwrite whichever live render owner currently holds PBDesc index 1 and +/// UMA index 0. +pub(crate) fn apply_g17p_cl_b2_descriptor_tables( + region_1: &mut [u8], + region_2: &mut [u8], +) -> Result<(), SubmissionError> { + let layout = G17P_CL_B2_PRIMARY_STATE_LAYOUT; + if region_1.len() < layout.object_size || region_2.len() < layout.object_size { + return Err(SubmissionError::KickB2PrimaryStateBufferTooSmall); + } + + put_u64_le(region_1, 0x10, 0x0000_0080_0007_7000); + put_u32_le(region_1, 0x18, 0x17); + put_u32_le(region_1, 0x1c, 0x17); + + put_u64_le(region_2, 0x00, 0x0800_0000_e000_0000); + put_u64_le(region_2, 0x08, 0x0000_3d40_0000_3400); + put_u64_le(region_2, 0x10, 0x0000_0000_0000_1d00); + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PClRetainedPublicationStep { + InitializeSharedSupport, + WriteSchedulerRecord, + RegisterPhysicalQueue, + PublishCommandOwnedState, + CleanCommandOwnedState, + WriteB2PrimaryState, + CleanB2PrimaryState, + SystemBarrierBeforeInnerPayload, + PublishInnerQueuePayload, + SystemBarrierBeforeInnerProducer, + PublishInnerQueueProducer, + SystemBarrierAfterInnerProducer, +} + +/// One-time order that makes the scheduler alias valid before registration. +pub(crate) const G17P_CL_RETAINED_REGISTRATION_ORDER: [G17PClRetainedPublicationStep; 3] = [ + G17PClRetainedPublicationStep::InitializeSharedSupport, + G17PClRetainedPublicationStep::WriteSchedulerRecord, + G17PClRetainedPublicationStep::RegisterPhysicalQueue, +]; + +/// One-time activation order proven by retained command B2. +pub(crate) const G17P_CL_RETAINED_ACTIVATION_ORDER: [G17PClRetainedPublicationStep; 9] = [ + G17PClRetainedPublicationStep::PublishCommandOwnedState, + G17PClRetainedPublicationStep::CleanCommandOwnedState, + G17PClRetainedPublicationStep::WriteB2PrimaryState, + G17PClRetainedPublicationStep::CleanB2PrimaryState, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerPayload, + G17PClRetainedPublicationStep::PublishInnerQueuePayload, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerProducer, + G17PClRetainedPublicationStep::PublishInnerQueueProducer, + G17PClRetainedPublicationStep::SystemBarrierAfterInnerProducer, +]; + +/// Steady-state order proven by command B3 on the same physical queue. +pub(crate) const G17P_CL_RETAINED_SUBMISSION_ORDER: [G17PClRetainedPublicationStep; 6] = [ + G17PClRetainedPublicationStep::PublishCommandOwnedState, + G17PClRetainedPublicationStep::CleanCommandOwnedState, + G17PClRetainedPublicationStep::PublishInnerQueuePayload, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerProducer, + G17PClRetainedPublicationStep::PublishInnerQueueProducer, + G17PClRetainedPublicationStep::SystemBarrierAfterInnerProducer, +]; + +pub(crate) const fn g17p_cl_retained_submission_order( + activation: G17PClB2ActivationState, +) -> &'static [G17PClRetainedPublicationStep] { + match activation { + G17PClB2ActivationState::Pending => &G17P_CL_RETAINED_ACTIVATION_ORDER, + G17PClB2ActivationState::Published => &G17P_CL_RETAINED_SUBMISSION_ORDER, + } +} + +// --------------------------------------------------------------------------- +// 6. Host addKicks MMIO publication (AGX3-specific). +// +// The host does not write the firmware-side `timestamp | (qid << 40)` word +// directly. The pinned G17P/G17G/G17X implementations publish a +// target-specific 64-bit queue tag followed by the zero-extended add count +// as a second 64-bit store to two +// runtime-selected scratch offsets. The host adds nine times the runtime +// stride to both bases for every target. Publication never reads either +// location and preserves the exact word-0 then word-1 order. The separate +// host diagnostic path may read word 1; word 0 must never be read. +// G17P `addKicks` performs `str x20` then `str x19` at +// 0xfffffe0008a57b5c/0xfffffe0008a57b60; the earlier `mov w19,w1` +// zero-extends the add count. +// --------------------------------------------------------------------------- + +/// One ordered transaction for the two SKSM scratch FIFO ports. +/// +/// `word0` must be stored first and `word1` second. There is deliberately no +/// read operation in this publication interface. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17SksmOrderedWritePair { + pub(crate) word0_offset: u32, + pub(crate) word0: u64, + pub(crate) word1_offset: u32, + pub(crate) word1: u64, +} + +/// Minimal backend for an ordered pair of 64-bit FIFO stores. +pub(crate) trait G17SksmPairWriter { + type Error; + + fn write64(&self, offset: u32, value: u64) -> core::result::Result<(), Self::Error>; +} + +/// Publish one pair in the only order accepted by the pinned host sequence. +pub(crate) fn publish_g17p_sksm_write_pair( + port: &P, + pair: G17SksmOrderedWritePair, +) -> core::result::Result<(), P::Error> { + port.write64(pair.word0_offset, pair.word0)?; + port.write64(pair.word1_offset, pair.word1) +} + +/// Host producer family with an independently recovered `addKicks` tag. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum SksmAddKicksTarget { + /// A18 Pro / HAL200 (`AGXKSMKickQueueG16`). + G17P, + /// M5 / HAL300 G target. + G17G, + /// M5 Pro/Max HAL300 X-family target (G17S/G17C host implementation). + G17X, +} + +/// Exact ordered values and offsets for one host `addKicks` publication. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedSksmAddKicksMmio { + /// First store: 64-bit target tag to `tag_offset`. + pub(crate) tag_offset: u32, + pub(crate) tag: u64, + /// Second store: zero-extended number of newly published kicks. + pub(crate) count_offset: u32, + pub(crate) add_count: u32, +} + +impl PreparedSksmAddKicksMmio { + /// Convert the decoded host values to the exact two 64-bit FIFO stores. + pub(crate) const fn write_pair(self) -> G17SksmOrderedWritePair { + G17SksmOrderedWritePair { + word0_offset: self.tag_offset, + word0: self.tag, + word1_offset: self.count_offset, + word1: self.add_count as u64, + } + } +} + +const fn sksm_add_kicks_tag(target: SksmAddKicksTarget, queue_id: u8) -> u64 { + match target { + // mac17p 0x8a57b28-0x8a57b34: base bit 42, qid at bit 43. + SksmAddKicksTarget::G17P => G17P_ADD_KICKS_TAG | ((queue_id as u64) << 43), + // mac17g 0x8b91978-0x8b91984: base bit 43, qid at bit 44. + SksmAddKicksTarget::G17G => G17G_ADD_KICKS_TAG | ((queue_id as u64) << 44), + // mac17j 0x8b53144-0x8b53154: G17G form plus host-family bit 53. + SksmAddKicksTarget::G17X => G17X_ADD_KICKS_TAG | ((queue_id as u64) << 44), + } +} + +const G17P_ADD_KICKS_TAG: u64 = 0x0000_0400_0002_1018; +const G17G_ADD_KICKS_TAG: u64 = 0x0000_0800_0002_1018; +const G17X_ADD_KICKS_TAG: u64 = 0x0020_0800_0002_1018; +/// Prepare the host's two ordered MMIO stores without touching the aperture. +pub(crate) const fn prepare_sksm_add_kicks_mmio( + target: SksmAddKicksTarget, + queue_id: u8, + add_count: u32, + geometry: G17SksmScratchGeometry, +) -> Result { + if queue_id > KICK_QID_MASK as u8 { + return Err(SubmissionError::QueueIdOutOfRange); + } + + let offsets = match geometry.fifo_offsets() { + Ok(offsets) => offsets, + Err(_) => return Err(SubmissionError::KickMmioOffsetOverflow), + }; + + Ok(PreparedSksmAddKicksMmio { + tag_offset: offsets.write0, + tag: sksm_add_kicks_tag(target, queue_id), + count_offset: offsets.write1, + add_count, + }) +} + +// --------------------------------------------------------------------------- +// 7. A18 Pro / G17P SKSM hardware bridge and completion descriptors. +// +// Grounded from the pinned G17P host. `AGXKSMKickQueueG16::configureHardware` +// at 0xfffffe0008a56580-0xfffffe0008a5682c reads the runtime MMIO geometry +// from accelerator-object fields +0x10730/+0x10734/+0x10738, writes a +// 64-bit queue tag and 64-bit queue payload, then publishes the QID bit at +// MMIO +0x21068. The object fields contain base0=0x4000, base1=0x4008 and +// stride=0x10; they are not MMIO locations. The same geometry is used by +// `addKicks` (§6). Linux resource 0 (`sgx`) owns this window. Its two +// scratch locations receive ordered stores; queue enable is a separate +// read/modify/write register after the pair. The production publication +// surface intentionally exposes no scratch read operation. +// +// `AGXAcceleratorG16::populateKSMCompletionQueue` at +// 0xfffffe0008a53034-0xfffffe0008a532b8 emits four 0x20-byte descriptors. +// Each present queue has 0x40-byte entries and a fixed ordinal 0..3. The +// matched firmware consumes those queues. D93AP IRQ 1001 and RTBuddy AKF +// message type 2 converge on source 4 and drain the event rings that carry +// completion stamps. The completion path remains pending until an actual +// SKSM kick proves that complete round trip on D93AP hardware. +// --------------------------------------------------------------------------- + +/// Accelerator-object fields holding the three runtime geometry values. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PSksmRuntimeGeometryObjectLayout { + pub(crate) write0_base_field: u32, + pub(crate) write1_base_field: u32, + pub(crate) stride_field: u32, +} + +pub(crate) const G17P_SKSM_RUNTIME_GEOMETRY_OBJECT_LAYOUT: G17PSksmRuntimeGeometryObjectLayout = + G17PSksmRuntimeGeometryObjectLayout { + write0_base_field: 0x10730, + write1_base_field: 0x10734, + stride_field: 0x10738, + }; + +/// QID-valid mask base written after the two queue-configuration stores. +pub(crate) const G17P_SKSM_QUEUE_ENABLE_MASK_BASE: u32 = 0x21068; +/// QID-disable mask base written before changing a registered queue's routing. +pub(crate) const G17P_SKSM_QUEUE_DISABLE_MASK_BASE: u32 = 0x21078; +/// Queue-entry address bits preserved by the host payload encoder. +pub(crate) const G17P_SKSM_QUEUE_ENTRY_ADDRESS_MASK: u64 = 0x0000_07ff_ffff_ffe0; +/// Fixed portion of the first G17P queue-configuration store. +const G17P_SKSM_QUEUE_CONFIG_TAG: u64 = 0x0000_0400_0002_1008; +const G17P_SKSM_QUEUE_COMPLETION_SELECTOR_MAX: u32 = 4; +/// `_AGFIDataMasterType` selected by the G17P TA channel. +pub(crate) const G17P_TA_DATA_MASTER: u32 = 0; +/// `_AGFIDataMasterType` selected by the G17P 3D channel. +pub(crate) const G17P_3D_DATA_MASTER: u32 = 1; +/// `_AGFIDataMasterType` selected by `AGXCommandQueue::processComputeSetup`. +pub(crate) const G17P_COMPUTE_DATA_MASTER: u32 = 2; + +/// Runtime routing state that the compute-channel setup owns for one queue. +/// +/// The pinned host selects the queue by the CL channel QID, prepares the +/// queue's mapping resource, obtains a completion selector from the channel, +/// and stores `selector | (2 << 32)` at queue `+0x1c`. Keeping the selector +/// and queue address together prevents a caller from supplying a non-compute +/// data-master value to the production compute path. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PComputeQueueRouting { + pub(crate) queue_id: u8, + pub(crate) completion_selector: u32, + pub(crate) entry_gpu_va: u64, + pub(crate) geometry: G17SksmQueueGeometry, +} + +/// Pure description of the ordered host-side publications that make one A18 +/// Pro SKSM queue visible to the hardware bridge. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PSksmQueueConfig { + /// First store: 64-bit QID tag. + pub(crate) tag_offset: u32, + pub(crate) tag: u64, + /// Second store: 64-bit queue address/data-master/completion selector. + pub(crate) payload_offset: u32, + pub(crate) payload: u64, + /// Final stores: high 64 QIDs first, then low 64 QIDs. + pub(crate) enable_pair: G17SksmOrderedWritePair, + /// Stores used before changing an already-enabled queue's routing. + pub(crate) disable_pair: G17SksmOrderedWritePair, +} + +impl PreparedG17PSksmQueueConfig { + /// Convert the queue configuration to its two ordered write-only stores. + pub(crate) const fn write_pair(self) -> G17SksmOrderedWritePair { + G17SksmOrderedWritePair { + word0_offset: self.tag_offset, + word0: self.tag, + word1_offset: self.payload_offset, + word1: self.payload, + } + } +} + +/// Prepare the exact G17P host queue-configuration stores without touching +/// MMIO. Only the paired A18 Pro identity is admitted. +pub(crate) const fn prepare_g17p_sksm_queue_config( + target: G17ChannelAbiTarget, + queue_id: u8, + data_master: u32, + completion_selector: u32, + entry_gpu_va: u64, + geometry: G17SksmScratchGeometry, +) -> Result { + match target { + G17ChannelAbiTarget::A18ProG17P => {} + G17ChannelAbiTarget::M5G17G | G17ChannelAbiTarget::M5ProMaxG17X => { + return Err(SubmissionError::ChannelAbiUnconfirmed) + } + } + if queue_id > KICK_QID_MASK { + return Err(SubmissionError::QueueIdOutOfRange); + } + if data_master >= 3 { + return Err(SubmissionError::KickQueueDataMasterOutOfRange); + } + if completion_selector > G17P_SKSM_QUEUE_COMPLETION_SELECTOR_MAX { + return Err(SubmissionError::KickQueueCompletionSelectorOutOfRange); + } + if entry_gpu_va & !G17P_SKSM_QUEUE_ENTRY_ADDRESS_MASK != 0 { + return Err(SubmissionError::KickQueueEntryAddressUnencodable); + } + + let offsets = match geometry.fifo_offsets() { + Ok(offsets) => offsets, + Err(_) => return Err(SubmissionError::KickMmioOffsetOverflow), + }; + + // Host sequence: 3 * (4 - selector) + 0x3fe, shifted into bit 54. + let completion_field = (3 * (4 - completion_selector) + 0x3fe) as u64; + let payload = + (((data_master as u64) + 1) << 60) | entry_gpu_va | completion_field.wrapping_shl(54); + let low_mask = if queue_id < 64 { 1u64 << queue_id } else { 0 }; + let high_mask = if queue_id >= 64 { + 1u64 << (queue_id - 64) + } else { + 0 + }; + let enable_pair = G17SksmOrderedWritePair { + word0_offset: G17P_SKSM_QUEUE_ENABLE_MASK_BASE + 8, + word0: high_mask, + word1_offset: G17P_SKSM_QUEUE_ENABLE_MASK_BASE, + word1: low_mask, + }; + let disable_pair = G17SksmOrderedWritePair { + word0_offset: G17P_SKSM_QUEUE_DISABLE_MASK_BASE + 8, + word0: high_mask, + word1_offset: G17P_SKSM_QUEUE_DISABLE_MASK_BASE, + word1: low_mask, + }; + + Ok(PreparedG17PSksmQueueConfig { + tag_offset: offsets.write0, + tag: G17P_SKSM_QUEUE_CONFIG_TAG | ((queue_id as u64) << 43), + payload_offset: offsets.write1, + payload, + enable_pair, + disable_pair, + }) +} + +/// Prepare the queue-configuration transaction produced by the G17P compute +/// setup path. +/// +/// `AGXCommandQueue::processComputeSetup` at +/// `0xfffffe0008a077a4..0xfffffe0008a078d4` selects the queue from the CL +/// channel QID, prepares its mapping resource, obtains the channel completion +/// selector, and writes the packed selector/data-master pair. The captured +/// native CL2 queue uses QID 4, selector 2, and entry VA `0x70004d8000`. +pub(crate) const fn prepare_g17p_compute_sksm_queue_config( + routing: G17PComputeQueueRouting, + geometry: G17SksmScratchGeometry, +) -> Result { + match routing.geometry.validate() { + Ok(()) => {} + Err(error) => return Err(error), + } + if routing.entry_gpu_va & (routing.geometry.allocation_alignment() - 1) != 0 { + return Err(SubmissionError::KickQueueEntryAddressUnencodable); + } + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + routing.queue_id, + G17P_COMPUTE_DATA_MASTER, + routing.completion_selector, + routing.entry_gpu_va, + geometry, + ) +} + +/// Prepare the queue-configuration transaction for one G17P *render* queue. +/// +/// Identical to the compute helper except for the data master, which the +/// twelve captured macOS render passes pin to TA = 0 and 3D = 1 in three +/// independent places (outer ring slot `+0x10`, tag-14 `+0x11`, tag-15 +/// `+0x22`). The compute data master is rejected here so a render caller +/// cannot silently configure a CL queue. +pub(crate) const fn prepare_g17p_render_sksm_queue_config( + routing: G17PComputeQueueRouting, + data_master: u32, + geometry: G17SksmScratchGeometry, +) -> Result { + match routing.geometry.validate() { + Ok(()) => {} + Err(error) => return Err(error), + } + if data_master != G17P_TA_DATA_MASTER && data_master != G17P_3D_DATA_MASTER { + return Err(SubmissionError::KickQueueDataMasterOutOfRange); + } + if routing.entry_gpu_va & (routing.geometry.allocation_alignment() - 1) != 0 { + return Err(SubmissionError::KickQueueEntryAddressUnencodable); + } + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + routing.queue_id, + data_master, + routing.completion_selector, + routing.entry_gpu_va, + geometry, + ) +} + +pub(crate) const G17P_KSM_COMPLETION_QUEUE_COUNT: u8 = 4; +pub(crate) const G17P_KSM_COMPLETION_DESCRIPTOR_SIZE: usize = 0x20; +pub(crate) const G17P_KSM_COMPLETION_ENTRY_SIZE: u32 = 0x40; +pub(crate) const G17P_KSM_COMPLETION_CAPACITY: u32 = 0x800; +pub(crate) const G17P_KSM_COMPLETION_BACKING_SIZE: usize = + G17P_KSM_COMPLETION_CAPACITY as usize * G17P_KSM_COMPLETION_ENTRY_SIZE as usize; +pub(crate) const G17P_KSM_COMPLETION_PRESENT_ORDINALS: [u8; 2] = [0, 2]; +/// Normal direct CL completion uses ordinal 0, independently of the compute +/// SKSM queue's numeric completion selector 2. +pub(crate) const G17P_NORMAL_CL_COMPLETION_ORDINAL: u8 = 0; + +/// Exact field offsets in one host-produced G17P completion descriptor. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmCompletionDescriptorLayout { + pub(crate) gpu_va: usize, + pub(crate) cpu_va: usize, + pub(crate) capacity: usize, + pub(crate) entry_size: usize, + pub(crate) ordinal: usize, + pub(crate) processed: usize, +} + +pub(crate) const G17P_KSM_COMPLETION_DESCRIPTOR_LAYOUT: G17PKsmCompletionDescriptorLayout = + G17PKsmCompletionDescriptorLayout { + gpu_va: 0x00, + cpu_va: 0x08, + capacity: 0x10, + entry_size: 0x14, + ordinal: 0x18, + processed: 0x1c, + }; + +/// Encode one present completion queue exactly as the G17P host populates it. +pub(crate) fn encode_g17p_ksm_completion_descriptor( + target: G17ChannelAbiTarget, + ordinal: u8, + gpu_va: u64, + cpu_va: u64, + capacity: u32, +) -> Result<[u8; G17P_KSM_COMPLETION_DESCRIPTOR_SIZE], SubmissionError> { + if target != G17ChannelAbiTarget::A18ProG17P { + return Err(SubmissionError::ChannelAbiUnconfirmed); + } + if ordinal >= G17P_KSM_COMPLETION_QUEUE_COUNT { + return Err(SubmissionError::CompletionQueueOrdinalOutOfRange); + } + if capacity == 0 { + return Err(SubmissionError::CompletionQueueCapacityZero); + } + + let mut raw = [0u8; G17P_KSM_COMPLETION_DESCRIPTOR_SIZE]; + let layout = G17P_KSM_COMPLETION_DESCRIPTOR_LAYOUT; + put_u64_le(&mut raw, layout.gpu_va, gpu_va); + put_u64_le(&mut raw, layout.cpu_va, cpu_va); + put_u32_le(&mut raw, layout.capacity, capacity); + put_u32_le(&mut raw, layout.entry_size, G17P_KSM_COMPLETION_ENTRY_SIZE); + put_u32_le(&mut raw, layout.ordinal, ordinal as u32); + put_u32_le(&mut raw, layout.processed, 0); + Ok(raw) +} + +/// The host-side completion descriptor and SKSM bridge equations are grounded; +/// this does not assert that Linux knows the physical IRQ source or has +/// observed an interrupt round trip. +pub(crate) const G17P_COMPLETION_HOST_ABI_GROUNDED: bool = true; + +// --------------------------------------------------------------------------- +// 8. A18 Pro / G17P top-level channel-command dispatch. +// +// This ABI is deliberately identity-scoped, just like §2c. The pinned G17P +// host grounds the top-level names below through its command-request and +// submit paths. The matching a000 firmware dispatcher is +// `agfaProcessChannelCommand` at 0xfffffc0000010fa8 with the 19-entry table +// at 0xfffffc000001233c. The b000/b100 dispatcher starts at +// 0xfffffc0000010f14 and uses the byte-identical table at +// 0xfffffc00000122a8. +// +// All three firmware variants validate at least four bytes, load a u32 tag +// from command offset zero, and treat tags above 18 as invalid. Their jump tables +// process 0/1/2/3/4/5/7/9/12/14/15/16/18. Tag 6 reaches the dedicated +// `PMRebuild must not be received in SKSM` fatal, while 8/10/11/13/17 +// reach the invalid-command fatal. The host nevertheless names 17 `Nop`; +// that producer identity is preserved, but the admission helper marks it invalid +// because the matched consumer cannot process it. +// +// Tags 6/8/10/11 remain semantically unresolved instead of inheriting an +// AGX2 or nested SKU-stream meaning. SKU-stream opcodes are a distinct +// nested domain and intentionally have no representation in the top-level +// enum. M5-family identities remain unsupported pending their own paired +// producer/consumer pass. +// +// The optional pre-dispatch filter uses literal mask 0x1d03f for tags <= +// 16. It is exposed separately so a future live producer can preserve that +// gate instead of conflating it with complete jump-table dispatch. +// --------------------------------------------------------------------------- + +/// Offset and width of the little-endian top-level command tag. +pub(crate) const G17P_TOP_LEVEL_COMMAND_TAG_OFFSET: u32 = 0; +pub(crate) const G17P_TOP_LEVEL_COMMAND_TAG_WIDTH: u32 = 4; + +/// Host-grounded G17P top-level command types. +/// +/// This type intentionally excludes unresolved tags and every nested +/// SKU-stream opcode. `Nop` records the host ABI but is invalid for +/// [`admit_g17p_top_level_command`] because all matched firmware variants send +/// tag 17 to the invalid-command fatal path. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum G17PTopLevelCommandType { + Ta = 0, + ThreeD = 1, + FastBlit = 2, + Cl = 3, + Barrier = 4, + PmGrow = 5, + RemoteNode = 7, + SharedEventSignal = 9, + AddFinalFrgGtpcKick = 12, + KsmKickQueueAddKicks = 14, + ConfigUpdate = 15, + EntrySignal = 16, + Nop = 17, + LateEvalEventSignal = 18, +} + +/// Highest tag admitted by the G17P jump-table bounds check. +pub(crate) const G17P_CHANNEL_COMMAND_TYPE_MAX: u32 = 18; + +/// Tags named by the pinned host's top-level ABI, including invalid `Nop`. +pub(crate) const G17P_TOP_LEVEL_COMMAND_KNOWN_MASK: u32 = 0x0007_d2bf; +/// In-range top-level tags whose host/firmware semantics remain unresolved. +pub(crate) const G17P_TOP_LEVEL_COMMAND_UNRESOLVED_MASK: u32 = 0x0000_0d40; +/// Tag 13 is in range but is neither a grounded host type nor unresolved. +pub(crate) const G17P_TOP_LEVEL_COMMAND_INVALID_MASK: u32 = 0x0000_2000; + +/// Tags whose firmware jump-table entries reach a normal processing case. +pub(crate) const G17P_CHANNEL_COMMAND_PROCESS_MASK: u32 = 0x0005_d2bf; +/// Tag 6 has a dedicated PM-rebuild-in-SKSM fatal path. +pub(crate) const G17P_CHANNEL_COMMAND_PM_REBUILD_MASK: u32 = 0x0000_0040; +/// In-range tags whose jump-table entries reach the invalid-command fatal. +pub(crate) const G17P_CHANNEL_COMMAND_INVALID_MASK: u32 = 0x0002_2d00; +/// Literal mask used by the optional pre-dispatch filter for tags <= 16. +pub(crate) const G17P_CHANNEL_COMMAND_PREFILTER_MASK: u32 = 0x0001_d03f; + +/// Firmware disposition for one raw command tag. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PChannelCommandDisposition { + Process, + PmRebuildForbiddenFatal, + InvalidCommandFatal, +} + +/// Admit the command ABI only for the exact paired G17P identity. +const fn ensure_g17p_command_abi(target: G17ChannelAbiTarget) -> Result<(), SubmissionError> { + match target { + G17ChannelAbiTarget::A18ProG17P => Ok(()), + G17ChannelAbiTarget::M5G17G | G17ChannelAbiTarget::M5ProMaxG17X => { + Err(SubmissionError::CommandAbiUnconfirmed) + } + } +} + +/// Read the complete little-endian u32 tag at top-level command offset zero. +const fn g17p_raw_top_level_command_tag(command: &[u8]) -> Result { + let offset = G17P_TOP_LEVEL_COMMAND_TAG_OFFSET as usize; + let end = offset + G17P_TOP_LEVEL_COMMAND_TAG_WIDTH as usize; + if command.len() < end { + return Err(SubmissionError::CommandHeaderTooShort); + } + Ok(u32::from_le_bytes([ + command[offset], + command[offset + 1], + command[offset + 2], + command[offset + 3], + ])) +} + +/// Decode a host-grounded top-level G17P command name. +/// +/// This records producer identity only. Call +/// [`admit_g17p_top_level_command`] before treating a decoded tag as +/// consumable by the matched firmware. +pub(crate) const fn decode_g17p_top_level_command( + target: G17ChannelAbiTarget, + command: &[u8], +) -> Result { + match ensure_g17p_command_abi(target) { + Ok(()) => {} + Err(error) => return Err(error), + } + + let raw = match g17p_raw_top_level_command_tag(command) { + Ok(raw) => raw, + Err(error) => return Err(error), + }; + match raw { + 0 => Ok(G17PTopLevelCommandType::Ta), + 1 => Ok(G17PTopLevelCommandType::ThreeD), + 2 => Ok(G17PTopLevelCommandType::FastBlit), + 3 => Ok(G17PTopLevelCommandType::Cl), + 4 => Ok(G17PTopLevelCommandType::Barrier), + 5 => Ok(G17PTopLevelCommandType::PmGrow), + 7 => Ok(G17PTopLevelCommandType::RemoteNode), + 9 => Ok(G17PTopLevelCommandType::SharedEventSignal), + 12 => Ok(G17PTopLevelCommandType::AddFinalFrgGtpcKick), + 14 => Ok(G17PTopLevelCommandType::KsmKickQueueAddKicks), + 15 => Ok(G17PTopLevelCommandType::ConfigUpdate), + 16 => Ok(G17PTopLevelCommandType::EntrySignal), + 17 => Ok(G17PTopLevelCommandType::Nop), + 18 => Ok(G17PTopLevelCommandType::LateEvalEventSignal), + 6 | 8 | 10 | 11 => Err(SubmissionError::CommandTagUnresolved), + _ => Err(SubmissionError::CommandTagInvalid), + } +} + +/// Classify a raw tag exactly like the matched G17P firmware jump table. +pub(crate) const fn classify_g17p_channel_command( + target: G17ChannelAbiTarget, + raw: u32, +) -> Result { + match ensure_g17p_command_abi(target) { + Ok(()) => {} + Err(error) => return Err(error), + } + + if raw > G17P_CHANNEL_COMMAND_TYPE_MAX { + return Ok(G17PChannelCommandDisposition::InvalidCommandFatal); + } + + let bit = 1u32 << raw; + if bit & G17P_CHANNEL_COMMAND_PROCESS_MASK != 0 { + Ok(G17PChannelCommandDisposition::Process) + } else if bit & G17P_CHANNEL_COMMAND_PM_REBUILD_MASK != 0 { + Ok(G17PChannelCommandDisposition::PmRebuildForbiddenFatal) + } else { + Ok(G17PChannelCommandDisposition::InvalidCommandFatal) + } +} + +/// Decode a top-level tag and require a normal matched-firmware dispatch case. +/// +/// In particular, host-grounded `Nop = 17` is invalid here because the +/// firmware sends it to the invalid-command fatal path. +pub(crate) const fn admit_g17p_top_level_command( + target: G17ChannelAbiTarget, + command: &[u8], +) -> Result { + let command_type = match decode_g17p_top_level_command(target, command) { + Ok(command_type) => command_type, + Err(error) => return Err(error), + }; + match classify_g17p_channel_command(target, command_type as u32) { + Ok(G17PChannelCommandDisposition::Process) => Ok(command_type), + Ok( + G17PChannelCommandDisposition::PmRebuildForbiddenFatal + | G17PChannelCommandDisposition::InvalidCommandFatal, + ) => Err(SubmissionError::CommandTagInvalidForFirmware), + Err(error) => Err(error), + } +} + +/// Apply the firmware's optional `tag <= 16 && bit(tag) & 0x1d03f` filter. +pub(crate) const fn g17p_channel_command_prefilter_allows( + target: G17ChannelAbiTarget, + raw: u32, +) -> Result { + match ensure_g17p_command_abi(target) { + Ok(()) => {} + Err(error) => return Err(error), + } + Ok(raw <= 16 && ((1u32 << raw) & G17P_CHANNEL_COMMAND_PREFILTER_MASK) != 0) +} + +// --------------------------------------------------------------------------- +// 9. Provenance ledger + live gaps. +// --------------------------------------------------------------------------- + +/// How each submission-ABI component was grounded, so reuse-vs-fork is explicit. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum AbiProvenance { + /// Byte-identical to the shipping AGX2 producer; reused unchanged. + IdenticalToAgx2, + /// AGX3-specific, grounded first-hand from matched host/firmware artifacts. + ForkedAgx3Grounded, + /// AGX2 layout the driver produces; the G17 firmware's *own* consumer + /// offsets were not read first-hand this pass. Kept unsupported. + Agx2InheritedUnconfirmed, +} + +/// Per-component provenance for the submission transport. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SubmissionAbiLedger { + pub(crate) tx_doorbell: AbiProvenance, + pub(crate) channel_state: AbiProvenance, + pub(crate) g17p_channel_state: AbiProvenance, + pub(crate) ring_state: AbiProvenance, + pub(crate) channel_descriptor: AbiProvenance, + pub(crate) host_channel_state_reset: AbiProvenance, + pub(crate) host_uncached_channel_reset: AbiProvenance, + pub(crate) g17p_command_types: AbiProvenance, + pub(crate) kick_word: AbiProvenance, + pub(crate) kick_queue_producer: AbiProvenance, + pub(crate) host_add_kicks_mmio: AbiProvenance, + pub(crate) g17p_queue_config: AbiProvenance, + pub(crate) g17p_completion_descriptors: AbiProvenance, + pub(crate) kick_record: AbiProvenance, +} + +/// The doorbell is byte-grounded (§1); the SKSM kick word/record and host +/// queue-timestamp producer are AGX3 forks grounded first-hand (§2-§5). The +/// generic channel/ring producer records remain inherited and unsupported. +/// A separate G17P-only channel-state entry records the matched A18 Pro host +/// producer and firmware consumer (§2c); it does not admit either M5 family. +/// The queue bridge/completion descriptors and command dispatch are grounded, +/// G17P-only AGX3 forks (§7-§8): the M5 +/// identities remain unsupported; unresolved values do not inherit AGX2 or +/// nested SKU meanings, and the host's `Nop = 17` is not firmware-admitted. +pub(crate) const SUBMISSION_ABI_LEDGER: SubmissionAbiLedger = SubmissionAbiLedger { + tx_doorbell: AbiProvenance::IdenticalToAgx2, + channel_state: AbiProvenance::Agx2InheritedUnconfirmed, + g17p_channel_state: AbiProvenance::ForkedAgx3Grounded, + ring_state: AbiProvenance::Agx2InheritedUnconfirmed, + channel_descriptor: AbiProvenance::ForkedAgx3Grounded, + host_channel_state_reset: AbiProvenance::ForkedAgx3Grounded, + host_uncached_channel_reset: AbiProvenance::ForkedAgx3Grounded, + g17p_command_types: AbiProvenance::ForkedAgx3Grounded, + kick_word: AbiProvenance::ForkedAgx3Grounded, + kick_queue_producer: AbiProvenance::ForkedAgx3Grounded, + host_add_kicks_mmio: AbiProvenance::ForkedAgx3Grounded, + g17p_queue_config: AbiProvenance::ForkedAgx3Grounded, + g17p_completion_descriptors: AbiProvenance::ForkedAgx3Grounded, + kick_record: AbiProvenance::ForkedAgx3Grounded, +}; + +/// The pinned G17P/G17G/G17X hosts all use the target-specific §6 MMIO +/// publication for `KSMKickQueue::addKicks`. EP 0x21 remains the separate +/// channel/work-ring doorbell; the two producers are complementary, not an +/// unresolved selector. +pub(crate) const HOST_SKSM_ADD_KICKS_PATH_SELECTED: bool = true; + +/// Whether the AGX3 per-pipe channel *descriptor* stride is proven. The G17S +/// firmware directly walks twelve 0x20-byte entries at +/// `runtime_pointers+0x020..0x19f` (§2a). +pub(crate) const CHANNEL_DESCRIPTOR_STRIDE_CONFIRMED: bool = true; + +/// The offline-unprovable pieces that keep the transport non-executable. Each +/// needs a live target device. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SubmissionLiveGaps { + /// Linux has an ordered pair writer on resource 0, including fixed-slot + /// geometry and serialization across queue owners. + pub(crate) sksm_resource0_ordered_write_implemented: bool, + /// EP 0x21 plus the retained CL2 channel queue has executed real work. + pub(crate) channel_doorbell_round_trip_proven: bool, + /// Linux allocates and publishes every object in the measured first-partial + /// graphics contract through its runtime path. + pub(crate) partial_graphics_runtime_implemented: bool, + /// The measured first-partial channel path executed paired graphics work. + pub(crate) partial_graphics_round_trip_proven: bool, + /// A native SKSM queue entry followed by the host `addKicks` MMIO pair has + /// executed real work. The shipping kext uses this path, but the classic + /// first-partial graphics path is independently executable. + pub(crate) sksm_add_kicks_round_trip_proven: bool, + /// Apple's host builds platform interrupt event sources from runtime + /// `interrupts`/`interrupts-valid` properties. D93AP physical IRQ 1001 and + /// RTBuddy AKF type 2 both converge exactly on source 4 and drain the main + /// firmware event rings, but an on-device SKSM completion round trip has + /// not proven that this ingress is sufficient for Linux completion. + pub(crate) completion_irq_route_proven: bool, + /// Ordinal-0 notification, entry, QID/timestamp, and EP0x20 stamp + /// correlation is implemented as a pure transaction before outstanding + /// subtraction. Physical IRQ/event-ring/ACK ownership remains separate. + pub(crate) normal_cl_completion_consumer_implemented: bool, +} + +impl SubmissionLiveGaps { + pub(crate) const fn classic_graphics_proven(&self) -> bool { + self.channel_doorbell_round_trip_proven + && self.partial_graphics_runtime_implemented + && self.partial_graphics_round_trip_proven + && self.completion_irq_route_proven + } + + pub(crate) const fn native_sksm_proven(&self) -> bool { + self.sksm_resource0_ordered_write_implemented + && self.sksm_add_kicks_round_trip_proven + && self.normal_cl_completion_consumer_implemented + && self.completion_irq_route_proven + } + + /// True when either independently modeled transport can complete work. + pub(crate) const fn all_proven(&self) -> bool { + self.classic_graphics_proven() || self.native_sksm_proven() + } +} + +/// The host-side writer is implemented. The channel-doorbell execution result +/// comes from `20260827-g17p-compact-contract-queue-live12`: exact 64-float +/// add3 output, then two more submissions on the same CL2 queue at queue and +/// channel counters 6/6/6 and 9/9/9. That does not prove the separate native +/// SKSM `addKicks` FIFO used by the shipping graphics channels. +pub(crate) const SUBMISSION_LIVE_GAPS: SubmissionLiveGaps = SubmissionLiveGaps { + sksm_resource0_ordered_write_implemented: true, + channel_doorbell_round_trip_proven: true, + partial_graphics_runtime_implemented: false, + partial_graphics_round_trip_proven: true, + sksm_add_kicks_round_trip_proven: false, + completion_irq_route_proven: false, + normal_cl_completion_consumer_implemented: true, +}; + +#[cfg(test)] +mod tests { + use super::*; + + /// Witnessed against the macOS capture: a real Metal render leaves the + /// parameter-buffer ring state at {7, 7, 0, 0}, and `PBState[+0x28]` + /// holds the full tagged address `0x1000190000`. The encode and decode + /// must be exact inverses -- that is what makes the field map from the two + /// kernelcache functions trustworthy. + #[test] + fn g17p_pbdesc_round_trips_the_witnessed_ring_state() { + assert_eq!(g17p_pbdesc_segment(0x10_0019_0000), 0); + assert_eq!( + g17p_pbdesc_reconstructed_address(0x10_0019_0000), + 0x10_0019_0000 + ); + assert_eq!( + g17p_pbdesc_reconstructed_address(0x70_0247_8000), + 0x70_0247_8000 + ); + assert_eq!( + g17p_pbdesc_reconstructed_address(0x0247_8000), + 0x80_0247_8000 + ); + let state = G17PPbDescRingState { + counter_a: 7, + counter_b: 7, + flag_high: 0, + flag_low: 0, + }; + let descriptor = encode_g17p_pbdesc([0; 4], 0x10_0019_0000, 0, state); + // dword0 carries the address >> 4; dword1 the segment >> 8 up top. + assert_eq!(descriptor[0], 0x0001_9000); + assert_eq!(descriptor[1] >> 28, 0x0); + assert_eq!(decode_g17p_pbdesc(descriptor), state); + + // Bits the firmware owns survive a re-encode. + let previous = [0x0000_0005u32, 0x0abc_defu32, 0xf000_0000u32, 0x4000_0000u32]; + let again = encode_g17p_pbdesc(previous, 0x10_0019_0000, 0x0012_3456, state); + assert_eq!(again[0] & 0x6, previous[0] & 0x6); + assert_eq!(again[1] & 0x003f_ffff, 0x0012_3456); + assert_eq!(again[2] & !0x003f_ffff, previous[2] & !0x003f_ffff); + assert_eq!(decode_g17p_pbdesc(again), state); + + assert_eq!(g17p_pbdesc_offset(2), 2 * G17P_PBDESC_ENTRY_SIZE); + } + + fn get_u64_le(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) + } + + fn get_u32_le(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) + } + + fn get_u16_le(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) + } + + fn source_fixture_fingerprint(raw: &[u8]) -> u64 { + raw.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) + } + + fn rce_binding_from_packed(word: u64) -> G17PClRceBinding { + G17PClRceBinding { + address: (word & G17P_CL_PACKED_ADDRESS_MASK) << 5, + tag: ((word >> 43) & KICK_QID_MASK as u64) as u8, + } + } + + fn native_add3_compute_operands( + descriptor: u64, + queue: u64, + converted_command_timestamp: u64, + event_mask: [u64; 4], + rce_words: [u64; 2], + ) -> G17PClKickEntryOperands<'static> { + let empty = G17PClRceBinding { address: 0, tag: 0 }; + G17PClKickEntryOperands { + descriptor_flag_4c: false, + descriptor_flag_5c8: false, + converted_command_timestamp, + barriers: &[], + mcache: None, + payload: [descriptor, queue], + event_mask, + rce_kind: 0, + rce_bindings: [ + rce_binding_from_packed(rce_words[0]), + rce_binding_from_packed(rce_words[1]), + empty, + empty, + ], + auxiliary: 0x003f_ffff_ffff_ffff, + } + } + + #[test] + fn compute_command_state_tracks_each_descriptor_slot() { + let first = prepare_g17p_compute_cl_command_state(0x70_0034_0000, 0, 0x26).unwrap(); + assert_eq!(first.event_mask, [0, 0, 4, 0]); + assert_eq!( + first.rce_bindings, + [ + G17PClRceBinding { + address: 0x70_0034_0040, + tag: 0x26, + }, + G17PClRceBinding { + address: 0x70_0034_0760, + tag: 0x04, + }, + G17PClRceBinding { address: 0, tag: 0 }, + G17PClRceBinding { address: 0, tag: 0 }, + ] + ); + + let second = prepare_g17p_compute_cl_command_state(0x70_0034_4000, 1, 0x26).unwrap(); + assert_eq!(second.event_mask, [0, 2, 8, 0]); + assert_eq!(second.rce_bindings[0].address, 0x70_0034_4040); + assert_eq!(second.rce_bindings[1].address, 0x70_0034_4760); + } + + #[test] + fn render_command_state_selects_native_descriptor_register_arrays() { + let ta = prepare_g17p_render_cl_command_state(0x70_0001_0740, true).unwrap(); + assert_eq!(ta.event_mask, [0; 4]); + assert_eq!( + ta.rce_bindings, + [ + G17PClRceBinding { + address: 0x70_0001_07a0, + tag: 0x47, + }, + G17PClRceBinding { address: 0, tag: 0 }, + G17PClRceBinding { address: 0, tag: 0 }, + G17PClRceBinding { address: 0, tag: 0 }, + ] + ); + + let fragment = + prepare_g17p_render_cl_command_state(0x70_0016_5800, false).unwrap(); + assert_eq!( + fragment.rce_bindings, + [ + G17PClRceBinding { + address: 0x70_0016_58a0, + tag: 0x56, + }, + G17PClRceBinding { + address: 0x70_0016_5fc0, + tag: 0x10, + }, + G17PClRceBinding { + address: 0x70_0016_66e0, + tag: 0x17, + }, + G17PClRceBinding { + address: 0x70_0016_6e00, + tag: 0x0a, + }, + ] + ); + } + + #[test] + fn tx_doorbell_matches_the_shipping_agx2_wire_form() { + // gpu.rs: MSG_TX_DOORBELL = 0x83 << 48; run_job kicks + // MSG_TX_DOORBELL | pipe_type | (index << 2). + const MSG_TX_DOORBELL: u64 = 0x83 << 48; + assert_eq!(EP_DOORBELL, 0x21); + for (pipe, raw) in [ + (PipeType::Vertex, 0u64), + (PipeType::Fragment, 1), + (PipeType::Compute, 2), + ] { + for index in 0u8..=7 { + let msg = encode_tx_doorbell(pipe, index).unwrap(); + assert_eq!(msg, MSG_TX_DOORBELL | raw | ((index as u64) << 2)); + // Firmware type extraction: ubfx x8,x1,#48,#6 == 3. + assert_eq!(doorbell_type(msg), DOORBELL_TYPE_TX); + assert_eq!(decode_tx_doorbell(msg), Ok((pipe, index))); + } + } + assert_eq!( + encode_tx_doorbell(PipeType::Vertex, 8), + Err(SubmissionError::DoorbellIndexOutOfRange) + ); + } + + #[test] + fn native_initial_render_uses_the_paired_priority_zero_doorbell() { + // Pipe identity at priority zero: 3D is the low Fragment pipe 1 and + // TA is the low Vertex pipe 0. Compute is the remaining pipe 2. + assert_eq!( + encode_tx_doorbell(PipeType::Fragment, 0), + Ok(0x0083_0000_0000_0001) + ); + assert_eq!( + encode_tx_doorbell(PipeType::Vertex, 0), + Ok(0x0083_0000_0000_0000) + ); + assert_eq!( + encode_tx_doorbell(PipeType::Compute, 0), + Ok(0x0083_0000_0000_0002) + ); + + assert_eq!(G17P_NATIVE_RENDER_DOORBELL_PRIORITY, 0); + assert_eq!(G17P_NATIVE_3D_DOORBELL, 0x0083_0000_0000_0001); + assert_eq!(G17P_NATIVE_TA_DOORBELL, 0x0083_0000_0000_0000); + assert_eq!( + encode_tx_doorbell(PipeType::Fragment, 0), + Ok(G17P_NATIVE_3D_DOORBELL) + ); + assert_eq!( + encode_tx_doorbell(PipeType::Vertex, 0), + Ok(G17P_NATIVE_TA_DOORBELL) + ); + } + + #[test] + fn doorbell_types_come_from_the_firmware_dispatch() { + // 0x83/0x84/0x85 & 0x3f -> 3/4/5, the firmware's cmp targets. + assert_eq!(DOORBELL_TYPE_TX, (0x83u8 & 0x3f)); + assert_eq!(DOORBELL_TYPE_FWCTL, (0x84u8 & 0x3f)); + assert_eq!(DOORBELL_TYPE_HALT, (0x85u8 & 0x3f)); + // A FwCtl (0x84<<48) doorbell is not a TX doorbell. + assert_eq!( + decode_tx_doorbell(0x84u64 << 48), + Err(SubmissionError::NotATxDoorbell) + ); + } + + #[test] + fn firmware_marks_pipe_three_invalid() { + // The firmware `cmp w8,#0x3 ; b.eq done` path: pipe 3 is not a pipe. + let msg = (0x83u64 << 48) | 0x3; + assert_eq!(doorbell_type(msg), DOORBELL_TYPE_TX); + assert_eq!(decode_tx_doorbell(msg), Err(SubmissionError::ReservedPipe)); + } + + #[test] + fn wakeup_doorbell_payloads_decode_like_the_firmware() { + // DOORBELL_KICKFW=0x10 -> pipe 0, index 4; DOORBELL_DEVCTRL=0x11 -> + // pipe 1, index 4 (msg & 3, (msg>>2)&7). + let kickfw = (0x83u64 << 48) | DOORBELL_KICKFW as u64; + assert_eq!(decode_tx_doorbell(kickfw), Ok((PipeType::Vertex, 4))); + let devctrl = (0x83u64 << 48) | DOORBELL_DEVCTRL as u64; + assert_eq!(decode_tx_doorbell(devctrl), Ok((PipeType::Fragment, 4))); + } + + #[test] + fn g17p_running_client_power_gate_precedes_every_submit_mutation() { + let contract = G17P_RUNNING_CLIENT_POWER_CONTRACT; + assert_eq!(contract.role_index, 1); + assert_eq!(contract.service_id, 0x20); + assert_eq!(contract.target, 1); + assert!(!contract.management_request_used); + assert!(contract.managed); + assert!(!contract.standalone); + assert_eq!( + G17P_COMPUTE_POWER_LEASE_ORDER, + [ + G17PComputePowerLeaseStep::AcquireBeforeReadiness, + G17PComputePowerLeaseStep::HoldThroughClaim, + G17PComputePowerLeaseStep::HoldThroughCompletion, + G17PComputePowerLeaseStep::ReleaseAfterCompletion, + ] + ); + + let mut vm_bind_mutations = 0; + let mut readiness_mutations = 0; + let mut sksm_mutations = 0; + let mut mailbox_mutations = 0; + let gated: Result<(), SubmissionError> = (|| { + require_g17p_running_client_power_transition(contract, true, false)?; + vm_bind_mutations += 1; + readiness_mutations += 1; + sksm_mutations += 1; + mailbox_mutations += 1; + Ok(()) + })(); + assert_eq!( + gated, + Err(SubmissionError::RunningClientPowerTransitionUnavailable) + ); + assert_eq!(vm_bind_mutations, 0); + assert_eq!(readiness_mutations, 0); + assert_eq!(sksm_mutations, 0); + assert_eq!(mailbox_mutations, 0); + assert_eq!( + require_g17p_running_client_power_transition(contract, true, true), + Ok(()) + ); + } + + #[test] + fn g17p_native_boot_is_ep21_quiet_until_first_real_user_cl() { + assert!(G17P_NATIVE_PRE_USER_EP21_MESSAGES.is_empty()); + assert_eq!( + G17P_NATIVE_FIRST_USER_EP21_MESSAGES, + [ + encode_g17p_cl_activation(2).unwrap(), + encode_g17p_direct_cl_kick(), + ] + ); + assert_eq!( + G17P_NATIVE_FIRST_USER_EP21_MESSAGES.map(doorbell_type), + [3, 3] + ); + assert!(!G17P_NATIVE_FIRST_USER_EP21_MESSAGES + .iter() + .any(|message| doorbell_type(*message) == 4)); + } + + #[test] + fn g17p_direct_cl_notifications_match_first_and_repeat_fixtures() { + const ACTIVATION: &[u8; 8] = + include_bytes!("fixtures/g17p_first_submit/ep21-first-activation.bin"); + const DIRECT_KICK: &[u8; 8] = + include_bytes!("fixtures/g17p_first_submit/ep21-direct-kick.bin"); + const FIRST_SEQUENCE: &[u8; 16] = + include_bytes!("fixtures/g17p_first_submit/ep21-first-direct-sequence.bin"); + + let first_state = G17PDirectClNotificationState { + cached_channel_identity: 0, + submitted_channel_identity: 0xffff_fc20_c07a_1dc0, + firmware_ready: true, + direct_feature: true, + submission_suppressed: false, + command_submission_enabled: true, + }; + let first = prepare_g17p_direct_cl_notifications(first_state).unwrap(); + assert_eq!(first.activation.unwrap().to_le_bytes(), *ACTIVATION); + assert_eq!(first.direct_kick.unwrap().to_le_bytes(), *DIRECT_KICK); + let mut sequence = [0u8; 16]; + sequence[..8].copy_from_slice(&first.activation.unwrap().to_le_bytes()); + sequence[8..].copy_from_slice(&first.direct_kick.unwrap().to_le_bytes()); + assert_eq!(sequence, *FIRST_SEQUENCE); + + let repeat = prepare_g17p_direct_cl_notifications(G17PDirectClNotificationState { + cached_channel_identity: first_state.submitted_channel_identity, + ..first_state + }) + .unwrap(); + assert_eq!(repeat.activation, None); + assert_eq!(repeat.direct_kick.unwrap().to_le_bytes(), *DIRECT_KICK); + + let not_ready = prepare_g17p_direct_cl_notifications(G17PDirectClNotificationState { + firmware_ready: false, + ..first_state + }) + .unwrap(); + assert_eq!(not_ready.activation, None); + assert_eq!(not_ready.direct_kick.unwrap().to_le_bytes(), *DIRECT_KICK); + + for disabled in [ + G17PDirectClNotificationState { + direct_feature: false, + ..first_state + }, + G17PDirectClNotificationState { + submission_suppressed: true, + ..first_state + }, + G17PDirectClNotificationState { + command_submission_enabled: false, + ..first_state + }, + ] { + let prepared = prepare_g17p_direct_cl_notifications(disabled).unwrap(); + assert_eq!(prepared.activation.unwrap().to_le_bytes(), *ACTIVATION); + assert_eq!(prepared.direct_kick, None); + } + + assert_eq!( + T8140_G17P_CL_BACKEND, + G17PClBackendConfig { + backend: G17ClSubmissionBackend::G17PHal200Hybrid, + direct_feature: true, + submission_suppressed: false, + command_submission_enabled: true, + } + ); + assert!(g17p_first_bind_tag15_required(true)); + assert!(g17p_first_bind_tag16_required( + T8140_G17P_CL_BACKEND, + true + )); + assert!(!g17p_first_bind_tag16_required( + G17PClBackendConfig { + backend: G17ClSubmissionBackend::Classic, + ..T8140_G17P_CL_BACKEND + }, + true + )); + assert!(!g17p_first_bind_tag16_required( + G17PClBackendConfig { + direct_feature: false, + ..T8140_G17P_CL_BACKEND + }, + true + )); + assert!(G17P_FIRST_BIND_COMMAND_PLAN.hardware_release_ready()); + assert!(G17P_FIRST_BIND_DIRECT_HARDWARE_RELEASE_READY); + + let first_lifecycle = prepare_g17p_direct_cl_lifecycle( + G17PDirectClLifecycleState { + queue_config_dirty: true, + cached_channel_identity: 0, + next_correlation_token: 1, + }, + first_state.submitted_channel_identity, + true, + T8140_G17P_CL_BACKEND, + ) + .unwrap(); + assert!(first_lifecycle.publish_config_update); + assert!(first_lifecycle.publish_entry_signal); + assert_eq!(first_lifecycle.correlation_token, 1); + assert_eq!(first_lifecycle.notifications, first); + assert_eq!( + first_lifecycle.next, + G17PDirectClLifecycleState { + queue_config_dirty: false, + cached_channel_identity: first_state.submitted_channel_identity, + next_correlation_token: 2, + } + ); + + let repeat_lifecycle = prepare_g17p_direct_cl_lifecycle( + first_lifecycle.next, + first_state.submitted_channel_identity, + true, + T8140_G17P_CL_BACKEND, + ) + .unwrap(); + assert!(!repeat_lifecycle.publish_config_update); + assert!(!repeat_lifecycle.publish_entry_signal); + assert_eq!(repeat_lifecycle.correlation_token, 2); + assert_eq!(repeat_lifecycle.notifications.activation, None); + assert_eq!(repeat_lifecycle.notifications.direct_kick, first.direct_kick); + assert_eq!(repeat_lifecycle.next.next_correlation_token, 3); + assert_eq!( + prepare_g17p_direct_cl_lifecycle( + G17PDirectClLifecycleState { + next_correlation_token: 0, + ..first_lifecycle.next + }, + first_state.submitted_channel_identity, + true, + T8140_G17P_CL_BACKEND, + ), + Err(SubmissionError::ComputeEventCounterOverflow) + ); + + let mut command_ring_mutations = 0; + let mut sksm_storage_mutations = 0; + let mut sksm_mmio_mutations = 0; + let mut outer_ring_mutations = 0; + let mut mailbox_mutations = 0; + let blocked_plan = G17PFirstBindCommandPlan { + optional_correlation_ring: G17PCorrelationRingImplementationState::Pending, + ..G17P_FIRST_BIND_COMMAND_PLAN + }; + let gated: Result<(), SubmissionError> = (|| { + require_g17p_first_bind_direct_hardware_release(blocked_plan)?; + command_ring_mutations += 2; + sksm_storage_mutations += 1; + sksm_mmio_mutations += 1; + outer_ring_mutations += 1; + mailbox_mutations += 2; + Ok(()) + })(); + assert_eq!( + gated, + Err(SubmissionError::FirstBindCommandPublicationIncomplete) + ); + assert_eq!(command_ring_mutations, 0); + assert_eq!(sksm_storage_mutations, 0); + assert_eq!(sksm_mmio_mutations, 0); + assert_eq!(outer_ring_mutations, 0); + assert_eq!(mailbox_mutations, 0); + + let resolved = G17P_FIRST_BIND_COMMAND_PLAN; + assert_eq!(resolved.fallback_logical_flush_id, 0); + assert!(!resolved.flush_id_serialized_in_tag15); + assert!(!resolved.flush_id_completion_release_required); + assert!(resolved.hardware_release_ready()); + assert_eq!( + require_g17p_first_bind_direct_hardware_release(resolved), + Ok(()) + ); + for unresolved_implementation in [ + G17PFlushIdImplementationState::Pending, + G17PFlushIdImplementationState::ProvenAbsentFallbackUnverified, + G17PFlushIdImplementationState::ProvenPresentUnimplemented, + ] { + assert!(!G17PFirstBindCommandPlan { + dedicated_flush_id: unresolved_implementation, + ..resolved + } + .hardware_release_ready()); + } + assert!(G17PFirstBindCommandPlan { + dedicated_flush_id: G17PFlushIdImplementationState::ProvenPresentImplemented, + ..resolved + } + .hardware_release_ready()); + assert!(!G17PFirstBindCommandPlan { + optional_correlation_ring: G17PCorrelationRingImplementationState::Pending, + ..resolved + } + .hardware_release_ready()); + assert!(!G17PFirstBindCommandPlan { + native_command_pointer_ring: G17PContractPredicateState::Pending, + ..resolved + } + .hardware_release_ready()); + assert!(G17PFirstBindCommandPlan { + optional_correlation_ring: G17PCorrelationRingImplementationState::ProvenInactive, + ..resolved + } + .hardware_release_ready()); + assert!(G17PFirstBindCommandPlan { + optional_correlation_ring: + G17PCorrelationRingImplementationState::ProvenRequiredImplemented, + ..resolved + } + .hardware_release_ready()); + assert!(!g17p_first_bind_tag16_required( + G17PClBackendConfig { + backend: G17ClSubmissionBackend::Hal300, + ..T8140_G17P_CL_BACKEND + }, + true + )); + } + + #[test] + fn pending_pipe_bit_matches_1_shl_pipe() { + assert_eq!(pending_pipe_bit(PipeType::Vertex), 1); + assert_eq!(pending_pipe_bit(PipeType::Fragment), 2); + assert_eq!(pending_pipe_bit(PipeType::Compute), 4); + } + + #[test] + fn inherited_channel_and_ring_state_offsets_are_labeled_agx2() { + // channel.rs raw::ChannelState: read_ptr@0x00, write_ptr@0x20. + assert_eq!(AGX2_CHANNEL_STATE_READ_PTR_OFFSET, 0x00); + assert_eq!(AGX2_CHANNEL_STATE_WRITE_PTR_OFFSET, 0x20); + assert_eq!(AGX2_CHANNEL_STATE_SIZE, 0x40); + // fw/workqueue.rs raw::RingState: doneptr@0x00, rptr@0x30, wptr@0x40, + // size@0x50 (0x10-strided). + assert_eq!(AGX2_RING_STATE_GPU_DONEPTR_OFFSET, 0x00); + assert_eq!(AGX2_RING_STATE_GPU_RPTR_OFFSET, 0x30); + assert_eq!(AGX2_RING_STATE_CPU_WPTR_OFFSET, 0x40); + assert_eq!(AGX2_RING_STATE_RB_SIZE_OFFSET, 0x50); + assert_eq!(AGX2_RING_ENTRY_BYTES, 8); + assert_eq!(HOST_MAX_OUTSTANDING_JOB_SLOTS, 127); + } + + #[test] + fn g17p_native_channel_layout_is_identity_scoped() { + assert_eq!( + g17_channel_abi(G17ChannelAbiTarget::A18ProG17P), + Ok(G17P_CHANNEL_ABI) + ); + assert_eq!( + g17_channel_abi(G17ChannelAbiTarget::M5G17G), + Err(SubmissionError::ChannelAbiUnconfirmed) + ); + assert_eq!( + g17_channel_abi(G17ChannelAbiTarget::M5ProMaxG17X), + Err(SubmissionError::ChannelAbiUnconfirmed) + ); + + assert_eq!( + G17P_CHANNEL_ABI.cached, + G17PCachedChannelStateLayout { + uncached_state_gpu_va: 0x00, + command_pointer_ring_gpu_va: 0x08, + coherency_cursor: 0x1c, + record_size: 0x80, + } + ); + assert_eq!( + G17P_CHANNEL_ABI.uncached, + G17PUncachedChannelStateLayout { + consumer: 0x00, + coherency_shadow: 0x30, + producer: 0x40, + recovery_target: 0x50, + ring_entry_count: 0x60, + cursor_width: 4, + write_extent: 0x64, + } + ); + assert_eq!(G17P_CHANNEL_ABI.command_pointer_entry_bytes, 8); + assert_eq!(G17P_CHANNEL_ABI.accelerator_target_width, 2); + assert!(G17P_CHANNEL_PRODUCER_REQUIRES_DMB_ISH); + } + + #[test] + fn g17p_classic_work_slots_and_sksm_pointer_entries_are_distinct() { + assert_eq!(G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE, 0x18); + assert_eq!(G17P_CHANNEL_ABI.command_pointer_entry_bytes, 0x08); + + let sksm = prepare_g17p_channel_write( + G17ChannelAbiTarget::A18ProG17P, + 1, + 3, + 4, + ) + .unwrap(); + assert_eq!(sksm.entry_byte_offset, 0x08); + + let classic = prepare_g17p_partial_opening_outer_slot( + G17PPartialOpeningOuterChannelState { + consumers: [0, 0], + producer: 1, + queue_gpu_va: 0xffff_fc20_c000_00c0, + queue_write_index: 3, + }, + G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX, + 1, + 1, + ) + .unwrap(); + assert_eq!(classic.slot_byte_offset, 0x18); + assert_eq!(get_u64_le(&classic.slot, 0x08), 0xffff_fc20_c000_00c0); + } + + #[test] + fn g17p_channel_write_plan_matches_host_ordering_and_wrap() { + assert_eq!( + prepare_g17p_channel_write(G17ChannelAbiTarget::A18ProG17P, 2, 0, 4), + Ok(G17PChannelWritePlan { + entry_byte_offset: 0x10, + next_producer: 3, + }) + ); + assert_eq!( + prepare_g17p_channel_write(G17ChannelAbiTarget::A18ProG17P, 3, 2, 4), + Ok(G17PChannelWritePlan { + entry_byte_offset: 0x18, + next_producer: 0, + }) + ); + assert_eq!( + prepare_g17p_channel_write(G17ChannelAbiTarget::A18ProG17P, 1, 2, 4), + Err(SubmissionError::RingFull) + ); + assert_eq!( + prepare_g17p_channel_write(G17ChannelAbiTarget::A18ProG17P, 0, 0, 0), + Err(SubmissionError::RingSizeZero) + ); + assert_eq!( + prepare_g17p_channel_write(G17ChannelAbiTarget::M5ProMaxG17X, 0, 2, 4), + Err(SubmissionError::ChannelAbiUnconfirmed) + ); + } + + #[test] + fn g17p_partial_opening_pages_and_context_match_the_successful_graph() { + assert_eq!(G17P_PARTIAL_OPENING_PAGE_SIZE, 0x4000); + assert_eq!(G17P_PARTIAL_OPENING_SCHEDULER_GPU_VA, 0xffff_fc20_015e_0000); + assert_eq!( + G17P_PARTIAL_OPENING_COMPUTED_GPU_VA, + 0xffff_fc20_015d_8000 + ); + assert_eq!( + G17P_PARTIAL_OPENING_PRIMARY_INDEX_FIRMWARE_GPU_VA, + 0xffff_fc20_c084_8000 + ); + assert_eq!( + G17P_PARTIAL_OPENING_PRIMARY_INDEX_GPU_VA, + 0x0000_0010_0019_0000 + ); + assert_eq!(G17P_PARTIAL_OPENING_PRIMARY_INDEX_SIZE, 0x10000); + + let scheduler = build_g17p_partial_opening_scheduler_page(); + assert_eq!(get_u64_le(&scheduler, 0), 0x0000_0020_0001_9000); + assert!(scheduler[8..].iter().all(|value| *value == 0)); + let mut scheduler_slice = [0xa5; G17P_PARTIAL_OPENING_PAGE_SIZE]; + assert_eq!( + apply_g17p_partial_opening_scheduler_page(&mut scheduler_slice), + Ok(()) + ); + assert_eq!(scheduler_slice, scheduler); + + let primary = build_g17p_partial_opening_primary_index_page(); + let expected = [ + 0x11, 0x12, 0x13, 0x14, 0x16, 0x17, 0x18, 0x19, 0x1b, 0x1c, 0x1d, 0x1e, 0x20, 0x21, + 0x22, 0x23, 0x25, 0x26, 0x27, 0x28, 0x2a, 0x2b, 0x2c, 0x2d, 0x3c, 0x3d, 0x3e, 0x3f, + 0x41, 0x42, 0x43, 0x44, + ]; + for (index, value) in expected.into_iter().enumerate() { + assert_eq!(get_u32_le(&primary, index * 4), value); + } + assert!(primary[expected.len() * 4..] + .iter() + .all(|value| *value == 0)); + let mut primary_slice = [0xa5; G17P_PARTIAL_OPENING_PAGE_SIZE]; + assert_eq!( + apply_g17p_partial_opening_primary_index_page(&mut primary_slice), + Ok(()) + ); + assert_eq!(primary_slice, primary); + assert_eq!( + apply_g17p_partial_opening_scheduler_page(&mut []), + Err(SubmissionError::PartialOpeningPageSize) + ); + assert_eq!( + apply_g17p_partial_opening_primary_index_page(&mut [0; 1]), + Err(SubmissionError::PartialOpeningPageSize) + ); + + assert_eq!( + G17P_PARTIAL_OPENING_CONTEXT, + G17PPartialOpeningContextContract { + context0_root_slot: 0, + context0_context_id: 0, + render_root_slot: 1, + render_context_id: 1, + context0_empty_high_root_slot: 0, + render_empty_high_root_slot: 1, + transport_pair: 0, + descriptor_pair: 0, + shared_queue_pair_namespace: 0, + optional_identity_context_id: 1, + descriptor_context_id: 1, + scheduler_class: 2, + optional_grids: [0, 1], + queue_identity: 0x15, + optional_submission_ordinal: 0, + event_counter: 0x102, + work_doorbell_channel: 0, + context0_low_root_present_at_first_work: true, + render_low_root_present_at_first_work: true, + source_firmware_high_root_table_resident: false, + } + ); + } + + #[test] + fn g17p_cold_opening_support_graph_matches_the_source_bytes() { + let addresses = G17PColdOpeningGraphAddresses { + primary_index: 0x70_0100_0000, + secondary_index: 0x70_0100_4000, + pool_a_slots: 0x70_0100_8000, + pool_b_slots: 0x70_0100_c000, + shared_slots: 0x70_0101_0000, + flag: 0x70_0101_4000, + }; + let mut page = [0xa5; G17P_PARTIAL_OPENING_PAGE_SIZE]; + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::SecondaryIndex, &mut page).unwrap(); + for (index, expected) in [0x11, 0x16, 0x1b, 0x20, 0x25, 0x2a, 0x3c, 0x41] + .into_iter() + .enumerate() + { + assert_eq!(get_u64_le(&page, index * 8), expected); + } + assert_eq!(source_fixture_fingerprint(&page), 0x5538_4e84_8d5a_840b); + assert!(page[0x40..].iter().all(|value| *value == 0)); + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::PoolASlots, &mut page).unwrap(); + assert_eq!(get_u32_le(&page, 4), 2); + assert_eq!(source_fixture_fingerprint(&page), 0xcfc1_218a_7364_2c07); + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::PoolBSlots, &mut page).unwrap(); + assert_eq!(source_fixture_fingerprint(&page), 0x9c1b_da7f_8c87_2325); + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::SharedSlots, &mut page).unwrap(); + assert_eq!(get_u32_le(&page, 0), 8); + assert_eq!(get_u32_le(&page, 4), 8); + assert_eq!(get_u32_le(&page, 0x60), 1); + assert_eq!(source_fixture_fingerprint(&page), 0x53c2_7f11_bcfb_4124); + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::Flag, &mut page).unwrap(); + assert_eq!(get_u32_le(&page, 0), 1); + assert_eq!(source_fixture_fingerprint(&page), 0xba85_253c_a046_2324); + + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::PrimaryIndex, &mut page).unwrap(); + assert_eq!(source_fixture_fingerprint(&page), 0x85e8_d166_b202_3e2d); + + let mut pool_a = [0u8; G17P_COLD_OPENING_RECORD_POOL_A_SIZE]; + apply_g17p_cold_opening_record_pool_a(addresses, &mut pool_a).unwrap(); + assert_eq!(get_u64_le(&pool_a, 0), addresses.pool_a_slots); + assert_eq!(get_u32_le(&pool_a, 0x10), 0x50); + assert_eq!( + get_u64_le(&pool_a, 34 * 0x100), + addresses.pool_a_slots + 34 * 4 + ); + assert_eq!(source_fixture_fingerprint(&pool_a), 0x42cb_4f83_a145_54be); + + let mut pool_b = [0u8; G17P_COLD_OPENING_RECORD_POOL_B_SIZE]; + apply_g17p_cold_opening_record_pool_b(addresses, &mut pool_b).unwrap(); + assert_eq!(get_u32_le(&pool_b, 0), 0x80004); + assert_eq!(get_u32_le(&pool_b, 0x28), 0x178020); + assert_eq!(get_u32_le(&pool_b, 35 * 0x80 + 0x28), 0x178000); + assert_eq!(get_u32_le(&pool_b, 36 * 0x80 + 0x28), 0x178020); + assert_eq!(get_u64_le(&pool_b, 0x40), addresses.shared_slots + 0x40); + assert_eq!(get_u32_le(&pool_b, 0x4c), 1); + assert_eq!(source_fixture_fingerprint(&pool_b), 0xb98b_d0a4_1c25_9a6e); + + let mut shared = [0u8; G17P_COLD_OPENING_SHARED_OBJECT_SIZE]; + apply_g17p_cold_opening_shared_object(addresses, &mut shared).unwrap(); + assert_eq!(get_u32_le(&shared, 0x0c), 0); + assert_eq!( + get_u32_le(&shared, 0x0c), + u32::from(G17P_PARTIAL_OPENING_CONTEXT.shared_queue_pair_namespace) + ); + assert_eq!(get_u64_le(&shared, 0x20), addresses.primary_index); + assert_eq!(get_u32_le(&shared, 0x34), 0x20); + assert_eq!(get_u32_le(&shared, 0x3c), 8); + assert_eq!(get_u64_le(&shared, 0x44), addresses.secondary_index); + assert_eq!(get_u64_le(&shared, 0x4c), addresses.shared_slots); + assert_eq!(get_u32_le(&shared, 0x54), 0x1f); + assert_eq!(get_u64_le(&shared, 0x64), addresses.flag); + assert_eq!(source_fixture_fingerprint(&shared), 0x5d97_133c_cbf5_a77c); + + let graph_range = 0x70_0100_0000..0x70_0101_8000; + for address in [ + get_u64_le(&pool_a, 0), + get_u64_le(&pool_a, 34 * 0x100), + get_u64_le(&pool_b, 0x08), + get_u64_le(&pool_b, 78 * 0x80 + 0x08), + get_u64_le(&pool_b, 0x40), + get_u64_le(&shared, 0x20), + get_u64_le(&shared, 0x44), + get_u64_le(&shared, 0x4c), + get_u64_le(&shared, 0x64), + ] { + assert!(graph_range.contains(&address)); + } + + let mut control = [0u8; G17P_COLD_OPENING_SHARED_CONTROL_SIZE]; + apply_g17p_cold_opening_shared_control(0x70_0020_8000, 0x70_0102_0000, &mut control) + .unwrap(); + assert_eq!(get_u32_le(&control, 0), 1); + assert_eq!(get_u32_le(&control, 0x10), 2); + assert_eq!(get_u64_le(&control, 0x20), 0x0000_1900_0000_0000); + assert_eq!(get_u64_le(&control, 0x30), 0x70_0020_8000); + assert_eq!(get_u32_le(&control, 0x48), 0xe0); + assert_eq!(get_u64_le(&control, 0x4c), 0x70_0102_0000); + assert_eq!(get_u32_le(&control, 0x60), 3); + assert_eq!(source_fixture_fingerprint(&control), 0x9457_5ea8_9f02_1db4); + assert!(G17P_COLD_OPENING_OPERAND_TABLE_GPU_VA >= 0x70_0000_0000); + assert!(G17P_COLD_OPENING_OPERAND_TABLE_GPU_VA < 0x71_0000_0000); + assert!((0x70_0102_0000..0x70_0102_4000).contains(&get_u64_le(&control, 0x4c))); + let mut inner = [0u8; G17P_COLD_OPENING_SHARED_CONTROL_INNER_SIZE]; + apply_g17p_cold_opening_shared_control_inner(&mut inner).unwrap(); + assert_eq!(get_u32_le(&inner, 0), 2); + assert_eq!(source_fixture_fingerprint(&inner), 0x8d1a_ce90_4a39_8d17); + + let mut channel = [0u8; G17P_COLD_OPENING_CHANNEL_CONTROL_SIZE]; + apply_g17p_cold_opening_channel_control(&mut channel).unwrap(); + assert_eq!(get_u64_le(&channel, 0), 0x0000_0100_0000_ffff); + assert_eq!(get_u64_le(&channel, 0x20), 0x0002_0000_0000_0000); + assert_eq!(get_u64_le(&channel, 0x30), 0x0000_0000_ff00_0000); + assert_eq!(source_fixture_fingerprint(&channel), 0x448e_89f5_b330_26cb); + } + + #[test] + fn g17p_native_cold_pool_a_starts_at_initialized_record_one_only() { + let addresses = G17PColdOpeningGraphAddresses { + primary_index: 0x70_0100_0000, + secondary_index: 0x70_0100_4000, + pool_a_slots: 0x70_0100_8000, + pool_b_slots: 0x70_0100_c000, + shared_slots: 0x70_0101_0000, + flag: 0x70_0101_4000, + }; + assert_eq!(g17p_render_pool_a_record_index(0, false), 0); + assert_eq!(g17p_render_pool_a_record_index(0, true), 1); + assert_eq!(g17p_render_pool_a_record_index(1, false), 2); + assert_eq!(g17p_render_pool_a_record_index(1, true), 2); + + let mut slots = [0xa5; G17P_PARTIAL_OPENING_PAGE_SIZE]; + apply_g17p_cold_opening_pool_a_start_slot(1, &mut slots).unwrap(); + assert_eq!(get_u32_le(&slots, 4), 1); + assert_eq!(get_u32_le(&slots, 8), 0); + assert!(slots[..4].iter().all(|byte| *byte == 0)); + assert!(slots[12..].iter().all(|byte| *byte == 0)); + + let mut records = [0xa5; G17P_COLD_OPENING_RECORD_POOL_A_SIZE]; + apply_g17p_cold_opening_record_pool_a_at(addresses, 1, &mut records).unwrap(); + assert_eq!(get_u64_le(&records, 0), addresses.pool_a_slots); + assert_eq!(get_u32_le(&records, 0x10), 0); + assert_eq!(get_u64_le(&records, 0x100), addresses.pool_a_slots + 4); + assert_eq!(get_u32_le(&records, 0x110), 0x50); + assert_eq!(get_u32_le(&records, 0x108), 0); + assert_eq!( + get_u64_le( + &records, + (G17P_COLD_OPENING_RECORD_POOL_A_COUNT - 1) + * G17P_COLD_OPENING_RECORD_POOL_A_STRIDE, + ), + addresses.pool_a_slots + + ((G17P_COLD_OPENING_RECORD_POOL_A_COUNT - 1) * 4) as u64 + ); + } + + #[test] + fn g17p_cold_opening_work_records_keep_identity_context_and_relocatable_pointers() { + let optional_addresses = G17PColdOpeningOptionalAddresses { + context_scratch: 0x70_0200_0000, + firmware_scratch: 0x70_0200_4000, + usc_priv_mem_flist: 0x70_0200_8000, + usc_freelist_hardware_buffer_id: 0, + channel_control: 0x70_0200_c000, + tiling_shared_object: 0x70_0201_0000, + parameter_buffer_token: 0, + owner_pid: 0x15, + ta_hardware_buffer_id: 0, + }; + let supplied = 0x70_0200_0000..0x70_0202_0000; + for stage in [G17PColdOpeningStage::Tiling, G17PColdOpeningStage::Fragment] { + let mut optional = [0u8; G17P_COLD_OPENING_OPTIONAL_SIZE]; + apply_g17p_cold_opening_optional( + stage, + optional_addresses, + 0, + 0, + 0, + 1, + 1, + &mut optional, + ) + .unwrap(); + assert_eq!(get_u32_le(&optional, 0), 0x0f); + for offset in [0x08, 0x10, 0x36, 0x4a] { + assert!(supplied.contains(&get_u64_le(&optional, offset))); + } + assert_eq!(get_u16_le(&optional, 0x1e), 2); + assert_eq!(get_u16_le(&optional, 0x32), 1); + assert_eq!(get_u16_le(&optional, 0x46), 0); + assert_eq!(get_u16_le(&optional, 0x56), 0); + assert_eq!(get_u32_le(&optional, 0x5a), 0x15); + assert_eq!(get_u16_le(&optional, 0x5e), 2); + assert_eq!(get_u16_le(&optional, 0x3e), 0); + assert_eq!( + source_fixture_fingerprint(&optional), + match stage { + G17PColdOpeningStage::Tiling => 0x3511_6ec9_a9f3_7149, + G17PColdOpeningStage::Fragment => 0x2a62_5e4c_5170_22ef, + } + ); + + let mut event = [0u8; G17P_COLD_OPENING_EVENT_SIZE]; + apply_g17p_cold_opening_event(stage, 0, &mut event).unwrap(); + assert_eq!(get_u32_le(&event, 0x08), 0x102); + assert_eq!( + source_fixture_fingerprint(&event), + match stage { + G17PColdOpeningStage::Tiling => 0x9eb4_fa84_39f9_667b, + G17PColdOpeningStage::Fragment => 0x86a6_d8e2_4f16_eea1, + } + ); + + let descriptor = if stage == G17PColdOpeningStage::Tiling { + G17P_COLD_OPENING_TA_DESCRIPTOR_BASE + } else { + G17P_COLD_OPENING_3D_DESCRIPTOR_BASE + }; + let queue = if stage == G17PColdOpeningStage::Tiling { + 0x70_0201_c000 + } else { + 0x70_0201_e000 + }; + let mut context = [0u8; G17P_COLD_OPENING_QUEUE_CONTEXT_SIZE]; + apply_g17p_cold_opening_queue_context(stage, descriptor, queue, &mut context).unwrap(); + assert_eq!(get_u64_le(&context, 0x210), descriptor); + assert_eq!(get_u64_le(&context, 0x218), queue); + assert_eq!( + get_u64_le(&context, 0x200), + match stage { + G17PColdOpeningStage::Tiling => 0x1000_0000_0000_0004, + G17PColdOpeningStage::Fragment => 0x1000_0400_0000_0004, + } + ); + if stage == G17PColdOpeningStage::Fragment { + assert_eq!(get_u64_le(&context, 0x208), 0); + assert_eq!(get_u64_le(&context, 0x358), 0x0000_8003_8000_4c3e); + assert_eq!(get_u64_le(&context, 0x360), 0x0000_b803_8000_4c77); + assert_eq!(get_u64_le(&context, 0x368), 0x0000_5003_8000_4cb0); + } else { + assert_eq!(get_u64_le(&context, 0x350), 0x0002_3803_8000_0003); + } + assert!(supplied.contains(&queue)); + assert_eq!( + source_fixture_fingerprint(&context), + match stage { + G17PColdOpeningStage::Tiling => 0xd30d_fd3c_7336_4025, + G17PColdOpeningStage::Fragment => 0xdfda_d80d_e0cc_292b, + } + ); + } + + let queue_addresses = G17PColdOpeningQueueAddresses { + pointers: 0x70_0200_0000, + item_ring: 0x70_0200_4000, + job_list: 0x70_0200_8000, + channel_control: 0x70_0200_c000, + owner_pid: 0x15, + }; + let mut queue = [0u8; G17P_COLD_OPENING_QUEUE_RECORD_SIZE]; + apply_g17p_cold_opening_queue_record(queue_addresses, &mut queue).unwrap(); + for offset in [0x00, 0x08, 0x10, 0x70] { + assert!(supplied.contains(&get_u64_le(&queue, offset))); + } + assert_eq!(get_u32_le(&queue, 0x48), 0); + assert_eq!(get_u32_le(&queue, 0x28), 2); + assert_eq!(get_u32_le(&queue, 0x3c), 2); + assert_eq!(get_u32_le(&queue, 0x40), u32::MAX); + assert_eq!(get_u32_le(&queue, 0x44), 0x15); + assert_eq!(source_fixture_fingerprint(&queue), 0x0704_bf11_9ee1_8084); + + let mut pointers = [0u8; G17P_COLD_OPENING_POINTER_BLOCK_SIZE]; + apply_g17p_cold_opening_pointer_block(&mut pointers).unwrap(); + assert_eq!(get_u32_le(&pointers, 0x50), u32::MAX); + assert_eq!(get_u64_le(&pointers, 0x60), 0x500); + assert_eq!(source_fixture_fingerprint(&pointers), 0x41b7_257d_6bc8_1cb2); + + let mut descriptor = [0u8; G17P_PARTIAL_OPENING_PAGE_SIZE]; + apply_g17p_cold_opening_descriptor_overrides(G17PColdOpeningStage::Tiling, &mut descriptor) + .unwrap(); + assert_eq!(get_u32_le(&descriptor, 0x0c), 1); + assert_eq!(descriptor[0x0789], 0x08); + assert_eq!(&descriptor[0x093e..0x0940], &[0xd0, 0x91]); + assert_eq!( + source_fixture_fingerprint(&descriptor), + 0x0a67_4d00_ea32_e630 + ); + + descriptor.fill(0); + apply_g17p_cold_opening_descriptor_overrides( + G17PColdOpeningStage::Fragment, + &mut descriptor, + ) + .unwrap(); + assert_eq!( + source_fixture_fingerprint(&descriptor), + 0xaac5_6b92_e70a_d459 + ); + } + + #[test] + fn g17p_cold_opening_context_locators_follow_descriptor_relocation() { + for (stage, base, locator_offsets) in [ + ( + G17PColdOpeningStage::Tiling, + G17P_COLD_OPENING_TA_DESCRIPTOR_BASE, + &[0x350usize][..], + ), + ( + G17PColdOpeningStage::Fragment, + G17P_COLD_OPENING_3D_DESCRIPTOR_BASE, + &[0x350usize, 0x358, 0x360, 0x368][..], + ), + ] { + let mut native = [0u8; G17P_COLD_OPENING_QUEUE_CONTEXT_SIZE]; + let mut relocated = [0u8; G17P_COLD_OPENING_QUEUE_CONTEXT_SIZE]; + apply_g17p_cold_opening_queue_context(stage, base, 0x70_0201_c000, &mut native) + .unwrap(); + apply_g17p_cold_opening_queue_context( + stage, + base + 0x8000, + 0x70_0201_c000, + &mut relocated, + ) + .unwrap(); + + for offset in locator_offsets { + assert_eq!( + get_u64_le(&relocated, *offset), + get_u64_le(&native, *offset) + 0x400 + ); + } + assert_eq!(get_u64_le(&relocated, 0x210), base + 0x8000); + + assert_eq!( + apply_g17p_cold_opening_queue_context( + stage, + base - 0x20, + 0x70_0201_c000, + &mut relocated, + ), + Err(SubmissionError::PartialOpeningDescriptorLocatorUnencodable) + ); + assert_eq!( + apply_g17p_cold_opening_queue_context( + stage, + base + 1, + 0x70_0201_c000, + &mut relocated, + ), + Err(SubmissionError::PartialOpeningDescriptorLocatorUnencodable) + ); + } + } + + #[test] + fn g17p_cold_opening_operand_graph_is_exact() { + let mut table = [0u8; G17P_PARTIAL_OPENING_PAGE_SIZE]; + apply_g17p_cold_opening_operand_table(&mut table).unwrap(); + assert_eq!(source_fixture_fingerprint(&table), 0x9c1b_da7f_8c87_2325); + assert!(table.iter().all(|value| *value == 0)); + + apply_g17p_populated_operand_table(&mut table).unwrap(); + assert_eq!( + get_u64_le(&table, 0), + (1u64 << 60) | G17P_COLD_OPENING_OPERAND_BUFFER_BASE + ); + assert_eq!( + get_u64_le(&table, 27 * 0x40), + (1u64 << 60) + | (G17P_COLD_OPENING_OPERAND_BUFFER_BASE + + 27 * G17P_COLD_OPENING_OPERAND_BUFFER_STRIDE) + ); + assert!(table[28 * 0x40..].iter().all(|value| *value == 0)); + assert_eq!(source_fixture_fingerprint(&table), 0x61a7_b063_3bea_e785); + + let mut directory = [0u8; G17P_COLD_OPENING_OPERAND_DIRECTORY_SIZE]; + apply_g17p_cold_opening_operand_directory(&mut directory).unwrap(); + assert_eq!( + source_fixture_fingerprint(&directory), + 0xeb05_052e_a5b6_2325 + ); + assert!(directory.iter().all(|value| *value == 0)); + + apply_g17p_populated_operand_directory(&mut directory).unwrap(); + assert_eq!(get_u64_le(&directory, 0), 0x70_0022_0000); + assert_eq!(get_u64_le(&directory, 255 * 8), 0x70_0031_f000); + assert_eq!(get_u64_le(&directory, 256 * 8), 0x70_0032_8000); + assert_eq!(get_u64_le(&directory, (7168 - 1) * 8), 0x70_01ef_7000); + assert!(directory[7168 * 8..].iter().all(|value| *value == 0)); + assert_eq!( + source_fixture_fingerprint(&directory), + 0xd5c6_623f_4366_03a5 + ); + } + + #[test] + fn g17p_partial_opening_status_is_exact_and_checked() { + const KERNEL_VA_BASE: u64 = 0xffff_fc20_0000_0000; + assert_eq!( + prepare_g17p_partial_opening_pre_0x84_status(KERNEL_VA_BASE), + Ok([ + G17PPartialOpeningStatusWrite { + offset: 0x4018, + value: 0x0005_0601_0000_0000, + }, + G17PPartialOpeningStatusWrite { + offset: 0x4020, + value: 61, + }, + G17PPartialOpeningStatusWrite { + offset: 0x40b0, + value: 5, + }, + G17PPartialOpeningStatusWrite { + offset: 0x48e0, + value: KERNEL_VA_BASE + 0x001c_0000, + }, + G17PPartialOpeningStatusWrite { + offset: 0x48e8, + value: KERNEL_VA_BASE + 0x001c_0040, + }, + ]) + ); + assert_eq!( + prepare_g17p_partial_opening_pre_0x84_status(u64::MAX), + Err(SubmissionError::PartialOpeningAddressOverflow) + ); + } + + #[test] + fn g17p_partial_opening_publishes_3d_then_ta_with_one_pair_kick() { + assert_eq!( + G17P_PARTIAL_OPENING_PUBLICATION_ORDER, + [ + G17PPartialOpeningPublicationStep::PrimarySchedulerPage, + G17PPartialOpeningPublicationStep::PrimaryIndexPage, + G17PPartialOpeningPublicationStep::PreControlStatus, + G17PPartialOpeningPublicationStep::ControlDone0x84, + G17PPartialOpeningPublicationStep::FragmentState, + G17PPartialOpeningPublicationStep::EmptyHighRoots, + G17PPartialOpeningPublicationStep::FragmentProducer, + G17PPartialOpeningPublicationStep::LateTilingState, + G17PPartialOpeningPublicationStep::SystemBarrierBeforeTilingProducer, + G17PPartialOpeningPublicationStep::TilingProducer, + G17PPartialOpeningPublicationStep::SystemBarrierBeforePairedDoorbell, + G17PPartialOpeningPublicationStep::PairedWorkDoorbell0x83, + ] + ); + let prepared = prepare_g17p_partial_opening_work_pair( + G17PPartialOpeningOuterChannelState { + consumers: [0, 0], + producer: 0, + queue_gpu_va: 0xffff_fc20_c000_00c0, + queue_write_index: 3, + }, + G17PPartialOpeningOuterChannelState { + consumers: [0, 0], + producer: 0, + queue_gpu_va: 0xffff_fc20_c000_0000, + queue_write_index: 3, + }, + 1, + 0, + true, + ) + .unwrap(); + assert_eq!( + prepared.fragment, + G17PPartialOpeningOuterSlotPlan { + channel_table_index: 7, + slot_index: 0, + slot_byte_offset: 0, + slot: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0xc0, 0x00, 0x00, 0xc0, 0x20, 0xfc, 0xff, 0xff, 1, 0, + 0, 0, 3, 0, 1, 1, + ], + next_producer: 1, + }, + 1, + 0 + ); + assert_eq!( + prepared.tiling, + G17PPartialOpeningOuterSlotPlan { + channel_table_index: 6, + slot_index: 0, + slot_byte_offset: 0, + slot: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0x00, 0x00, 0x00, 0xc0, 0x20, 0xfc, 0xff, 0xff, 0, 0, + 0, 0, 3, 0, 0, 1, + ], + next_producer: 1, + } + ); + assert_eq!(G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE, 0x18); + assert_eq!(G17P_PARTIAL_OPENING_OUTER_SLOT_COUNT, 0x100); + assert_eq!(G17P_PARTIAL_OPENING_TILING_CHANNEL_TABLE_INDEX, 6); + assert_eq!(G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX, 7); + assert_eq!(G17P_PARTIAL_OPENING_WORK_DOORBELL_CHANNEL, 0); + assert_eq!(prepared.paired_doorbell, 0x0083_0000_0000_0000); + assert_eq!( + prepared.paired_doorbell, + encode_tx_doorbell(PipeType::Vertex, 0).unwrap() + ); + + let compute = prepare_g17p_compute_channel_work( + G17PPartialOpeningOuterChannelState { + consumers: [0, 0], + producer: 0, + queue_gpu_va: 0xffff_fc20_c100_0000, + queue_write_index: 3, + }, + 4, + false, + ) + .unwrap(); + assert_eq!(compute.outer.channel_table_index, 8); + assert_eq!(compute.outer.slot_index, 0); + assert_eq!(compute.outer.next_producer, 1); + assert_eq!( + u64::from_le_bytes(compute.outer.slot[8..16].try_into().unwrap()), + 0xffff_fc20_c100_0000 + ); + assert_eq!( + u32::from_le_bytes(compute.outer.slot[0x10..0x14].try_into().unwrap()), + 2 + ); + assert_eq!( + u32::from_le_bytes(compute.outer.slot[0x14..0x18].try_into().unwrap()), + 0x0104_0003 + ); + assert_eq!(compute.doorbell, 0x0083_0000_0000_000a); + + let compute_repeat = prepare_g17p_compute_channel_work( + G17PPartialOpeningOuterChannelState { + consumers: [1, 1], + producer: 1, + queue_gpu_va: 0xffff_fc20_c100_0000, + queue_write_index: 7, + }, + 4, + true, + ) + .unwrap(); + assert_eq!( + u32::from_le_bytes( + compute_repeat.outer.slot[0x14..0x18] + .try_into() + .unwrap() + ), + 0x0004_0007 + ); + + let full = G17PPartialOpeningOuterChannelState { + consumers: [0, 7], + producer: 6, + queue_gpu_va: 0, + queue_write_index: 3, + }; + assert_eq!( + prepare_g17p_partial_opening_work_pair(full, full, 1, 0, true), + Err(SubmissionError::RingFull) + ); + assert_eq!( + prepare_g17p_partial_opening_work_pair( + G17PPartialOpeningOuterChannelState { + producer: 0x100, + ..full + }, + full, + 1, + 0, + true, + ), + Err(SubmissionError::PartialOpeningChannelCounterOutOfRange) + ); + assert_eq!( + prepare_g17p_partial_opening_work_pair( + G17PPartialOpeningOuterChannelState { + consumers: [0, 0], + producer: 0, + queue_write_index: 0x1_0000, + ..full + }, + full, + 1, + 0, + true, + ), + Err(SubmissionError::PartialOpeningQueueHeadOutOfRange) + ); + } + + #[test] + fn g17_channel_descriptor_geometry_matches_the_firmware_walk() { + assert_eq!(G17_CHANNEL_DESCRIPTOR_TABLE_OFFSET, 0x20); + assert_eq!(G17_CHANNEL_DESCRIPTORS_PER_PRIORITY, 3); + assert_eq!(G17_CHANNEL_PRIORITY_COUNT, 4); + assert_eq!(G17_CHANNEL_DESCRIPTOR_COUNT, 12); + assert_eq!(G17_CHANNEL_DESCRIPTOR_STRIDE, 0x20); + assert_eq!(G17_CHANNEL_ENTRY_STRIDE, 0x18); + assert_eq!( + G17_CHANNEL_DESCRIPTOR_LAYOUT, + G17ChannelDescriptorLayout { + dispatch_cursor: 0x00, + coherency_cursor: 0x08, + producer_cursor: 0x10, + entries: 0x18, + } + ); + + for priority in 0..G17_CHANNEL_PRIORITY_COUNT { + for (pipe, pipe_index) in [ + (PipeType::Vertex, 0u32), + (PipeType::Fragment, 1), + (PipeType::Compute, 2), + ] { + let linear = priority as u32 * 3 + pipe_index; + assert_eq!( + g17_channel_descriptor_offset(priority, pipe), + Ok(0x20 + linear * 0x20) + ); + } + } + + // Descriptor eleven occupies 0x180..0x19f. Priority four must not + // alias the independent cursor pair beginning at +0x1a0. + assert_eq!( + g17_channel_descriptor_offset(3, PipeType::Compute), + Ok(0x180) + ); + assert_eq!( + g17_channel_descriptor_offset(4, PipeType::Vertex), + Err(SubmissionError::ChannelPriorityOutOfRange) + ); + } + + #[test] + fn g17_host_channel_state_reset_matches_all_three_pinned_hosts() { + let values = G17HostChannelStateResetValues { + gpu_va_00: 0x0011_2233_4455_6677, + gpu_va_08: 0x8899_aabb_ccdd_eeff, + gpu_va_10: 0x1020_3040_5060_7080, + word_44: 0x7654_3210, + gpu_va_70: 0xfedc_ba98_7654_3210, + }; + let raw = encode_g17_host_channel_state_reset(values); + let layout = G17_HOST_CHANNEL_STATE_RESET_LAYOUT; + + assert_eq!(raw.len(), 0x80); + assert_eq!( + &raw[layout.gpu_va_00..layout.gpu_va_00 + 8], + &values.gpu_va_00.to_le_bytes() + ); + assert_eq!( + &raw[layout.gpu_va_08..layout.gpu_va_08 + 8], + &values.gpu_va_08.to_le_bytes() + ); + assert_eq!( + &raw[layout.gpu_va_10..layout.gpu_va_10 + 8], + &values.gpu_va_10.to_le_bytes() + ); + assert_eq!(&raw[layout.zero_18..layout.zero_18 + 4], &[0; 4]); + assert_eq!(&raw[layout.zero_1c..layout.zero_1c + 4], &[0; 4]); + assert_eq!(&raw[layout.zero_20..layout.zero_20 + 4], &[0; 4]); + assert_eq!( + &raw[layout.sentinel_24..layout.sentinel_24 + 4], + &u32::MAX.to_le_bytes() + ); + assert_eq!( + &raw[layout.fixed_4_28..layout.fixed_4_28 + 4], + &4u32.to_le_bytes() + ); + assert_eq!( + &raw[layout.sentinel_40..layout.sentinel_40 + 4], + &u32::MAX.to_le_bytes() + ); + assert_eq!( + &raw[layout.word_44..layout.word_44 + 4], + &values.word_44.to_le_bytes() + ); + assert_eq!(&raw[layout.zero_58..layout.zero_58 + 4], &[0; 4]); + assert_eq!( + &raw[layout.gpu_va_70..layout.gpu_va_70 + 8], + &values.gpu_va_70.to_le_bytes() + ); + + // resetChannelState clears the entire record before its scalar stores. + // Every byte outside the nonzero runtime/fixed ranges stays zero. + for (index, byte) in raw.iter().enumerate() { + let nonzero_field = (0x00..0x18).contains(&index) + || (0x24..0x2c).contains(&index) + || (0x40..0x48).contains(&index) + || (0x70..0x78).contains(&index); + if !nonzero_field { + assert_eq!(*byte, 0, "unexpected reset byte at {index:#x}"); + } + } + assert!(G17_HOST_CHANNEL_RESET_TARGETS_CONFIRMED); + } + + #[test] + fn g17_uncached_channel_reset_preserves_unwritten_bytes() { + let mut raw = [0xa5; G17_UNCACHED_CHANNEL_RESET_WRITE_EXTENT]; + let ring_entry_count = 0x800; + apply_g17_uncached_channel_reset(&mut raw, ring_entry_count); + let layout = G17_UNCACHED_CHANNEL_RESET_LAYOUT; + + for offset in [ + layout.zero_00, + layout.zero_10, + layout.zero_20, + layout.zero_30, + layout.zero_40, + ] { + assert_eq!(&raw[offset..offset + 4], &[0; 4]); + } + assert_eq!( + &raw[layout.sentinel_50..layout.sentinel_50 + 4], + &u32::MAX.to_le_bytes() + ); + assert_eq!( + &raw[layout.ring_entry_count_60..layout.ring_entry_count_60 + 4], + &ring_entry_count.to_le_bytes() + ); + + for (index, byte) in raw.iter().enumerate() { + let written = [0x00usize, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60] + .iter() + .any(|offset| (*offset..*offset + 4).contains(&index)); + if !written { + assert_eq!(*byte, 0xa5, "unproven byte changed at {index:#x}"); + } + } + } + + /// Native tag-14 AddKicks, PROVEN on all 25 captured publishes: + /// `+0x00 = 0x0e`, `+0x04 u16 = qid`, `+0x06 u16 = 1` kick, `+0x08 u8 = 2` + /// queue priority, `+0x09 u64 = entry stamp`, `+0x11 u8 = data master`, + /// everything else zero. + #[test] + fn g17p_retained_event_carries_native_addkicks_bytes() { + for (stage, qid, data_master) in [ + (G17PColdOpeningStage::Tiling, 0u16, 0u8), + (G17PColdOpeningStage::Fragment, 1u16, 1u8), + ] { + let mut event = [0xa5u8; G17P_COLD_OPENING_EVENT_SIZE]; + apply_g17p_retained_event_with_stamp(stage, 0x3d9, qid, &mut event).unwrap(); + assert_eq!(get_u32_le(&event, 0x00), 0x0e); + assert_eq!(get_u16_le(&event, 0x04), qid); + assert_eq!(get_u16_le(&event, 0x06), 1); + assert_eq!(event[0x08], G17P_NATIVE_RENDER_DOORBELL_PRIORITY); + assert_eq!( + u32::from_le_bytes([event[0x09], event[0x0a], event[0x0b], 0]), + 0x3d9 + ); + assert_eq!(event[0x11], data_master); + for (index, byte) in event.iter().enumerate() { + if !(0x00..0x0c).contains(&index) && index != 0x11 { + assert_eq!(*byte, 0, "unproven byte set at {index:#x}"); + } + } + } + // The native 40-bit producer wraps to zero. Values above its width + // must be refused before any record is modified. + let mut event = [0u8; G17P_COLD_OPENING_EVENT_SIZE]; + assert!( + apply_g17p_retained_event_with_stamp(G17PColdOpeningStage::Tiling, 1u64 << 40, 0, &mut event) + .is_err() + ); + } + + /// The render pair configures TA on data master 0 and 3D on data master 1; + /// the compute data master must be refused on this path. + #[test] + fn g17p_render_queue_config_admits_only_render_data_masters() { + let geometry = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY; + let scratch = g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch; + for (queue_id, data_master, entry) in [ + ( + G17P_PARTIAL_OPENING_TILING_CHANNEL_TABLE_INDEX, + G17P_TA_DATA_MASTER, + 0x70_0043_8000u64, + ), + ( + G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX, + G17P_3D_DATA_MASTER, + 0x70_0046_0000u64, + ), + ] { + let routing = G17PComputeQueueRouting { + queue_id, + completion_selector: 2, + entry_gpu_va: entry, + geometry, + }; + let config = + prepare_g17p_render_sksm_queue_config(routing, data_master, scratch).unwrap(); + assert_eq!(config.payload & G17P_SKSM_QUEUE_ENTRY_ADDRESS_MASK, entry); + assert_eq!(config.payload >> 60, data_master as u64 + 1); + assert_eq!(config.enable_pair.word1, 1u64 << queue_id); + assert_eq!(config.enable_pair.word0, 0); + assert!(prepare_g17p_render_sksm_queue_config( + routing, + G17P_COMPUTE_DATA_MASTER, + scratch + ) + .is_err()); + } + } + + #[test] + fn g17p_secondary_render_pair_matches_native_configure_words() { + let geometry = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY; + let scratch = g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch; + for (queue_id, data_master, entry, tag, payload, mask) in [ + ( + 2, + G17P_TA_DATA_MASTER, + 0x70_01f4_8000, + 0x0000_1400_0002_1008, + 0x1280_0070_01f4_8000, + 0x4, + ), + ( + 3, + G17P_3D_DATA_MASTER, + 0x70_01f7_0000, + 0x0000_1c00_0002_1008, + 0x2280_0070_01f7_0000, + 0x8, + ), + ] { + let config = prepare_g17p_render_sksm_queue_config( + G17PComputeQueueRouting { + queue_id, + completion_selector: 0, + entry_gpu_va: entry, + geometry, + }, + data_master, + scratch, + ) + .unwrap(); + assert_eq!(config.tag, tag); + assert_eq!(config.payload, payload); + assert_eq!(config.enable_pair.word0, 0); + assert_eq!(config.enable_pair.word1, mask); + } + } + + #[test] + fn g17p_render_config_update_carries_context_hardware_buffer_id_and_pid() { + let addresses = G17PColdOpeningOptionalAddresses { + context_scratch: 0x70_0200_0000, + firmware_scratch: 0x70_0200_4000, + usc_priv_mem_flist: 0x70_0200_8000, + usc_freelist_hardware_buffer_id: 5, + channel_control: 0x70_0200_c000, + tiling_shared_object: 0x70_0201_0000, + parameter_buffer_token: 0, + owner_pid: 0x625, + ta_hardware_buffer_id: 7, + }; + for context_id in [5, 62] { + for stage in [ + G17PColdOpeningStage::Tiling, + G17PColdOpeningStage::Fragment, + ] { + let mut optional = [0u8; G17P_COLD_OPENING_OPTIONAL_SIZE]; + apply_g17p_retained_optional( + stage, + addresses, + 0, + false, + 0, + 1, + 5, + 1, + context_id, + &mut optional, + ) + .unwrap(); + assert_eq!(get_u16_le(&optional, 0x32), context_id); + assert_eq!(get_u32_le(&optional, 0x46), 5); + assert_eq!(get_u16_le(&optional, 0x56), 5); + assert_eq!(get_u32_le(&optional, 0x5a), 0x625); + assert_eq!( + get_u32_le(&optional, 0x7e), + if stage == G17PColdOpeningStage::Tiling { + 7 + } else { + u32::MAX + } + ); + } + } + } + + #[test] + fn g17p_retained_canonical_records_advance_logically_without_moving() { + let addresses = G17PColdOpeningOptionalAddresses { + context_scratch: 0x70_0200_0000, + firmware_scratch: 0x70_0200_4000, + usc_priv_mem_flist: 0x70_0200_8000, + usc_freelist_hardware_buffer_id: 0, + channel_control: 0x70_0200_c000, + tiling_shared_object: 0x70_0201_0000, + parameter_buffer_token: 0, + owner_pid: 0x15, + ta_hardware_buffer_id: 0, + }; + for stage in [G17PColdOpeningStage::Tiling, G17PColdOpeningStage::Fragment] { + let mut optional = [0u8; G17P_COLD_OPENING_OPTIONAL_SIZE]; + apply_g17p_retained_optional( + stage, + addresses, + 1, + false, + 0, + 0, + 0, + 1, + 1, + &mut optional, + ) + .unwrap(); + assert_eq!(get_u16_le(&optional, 0x1a), 0); + assert_eq!(get_u16_le(&optional, 0x2a), 1); + assert_eq!(get_u16_le(&optional, 0x2e), 0x100); + assert_eq!(get_u16_le(&optional, 0x3e), 1); + assert_eq!(get_u16_le(&optional, 0x52), 0); + assert_eq!(get_u16_le(&optional, 0x62), 0); + assert_eq!( + get_u16_le(&optional, 0x76), + if stage == G17PColdOpeningStage::Tiling { + 0 + } else { + u16::MAX + } + ); + + let mut event = [0u8; G17P_COLD_OPENING_EVENT_SIZE]; + apply_g17p_retained_event(stage, 1, 0, &mut event).unwrap(); + assert_eq!(get_u32_le(&event, 0x08), 0x202); + } + + let pair = prepare_g17p_partial_opening_work_pair( + G17PPartialOpeningOuterChannelState { + consumers: [1, 1], + producer: 1, + queue_gpu_va: 0x70_0300_00c0, + queue_write_index: 3, + }, + G17PPartialOpeningOuterChannelState { + consumers: [1, 1], + producer: 1, + queue_gpu_va: 0x70_0300_0000, + queue_write_index: 3, + }, + 1, + 0, + false, + ) + .unwrap(); + assert_eq!(pair.fragment.slot_index, 1); + assert_eq!(pair.tiling.slot_index, 1); + assert_eq!(pair.fragment.next_producer, 2); + assert_eq!(pair.tiling.next_producer, 2); + assert_eq!(get_u32_le(&pair.fragment.slot, 0x14), 0x0001_0003); + assert_eq!(get_u32_le(&pair.tiling.slot, 0x14), 0x0000_0003); + } + + #[test] + fn g17p_context_update_is_fresh_graph_state_not_global_ordinal_state() { + let addresses = G17PColdOpeningOptionalAddresses { + context_scratch: 0x70_0200_0000, + firmware_scratch: 0x70_0200_4000, + usc_priv_mem_flist: 0x70_0200_8000, + usc_freelist_hardware_buffer_id: 0, + channel_control: 0x70_0200_c000, + tiling_shared_object: 0x70_0201_0000, + parameter_buffer_token: 0, + owner_pid: 0x15, + ta_hardware_buffer_id: 0, + }; + for stage in [G17PColdOpeningStage::Tiling, G17PColdOpeningStage::Fragment] { + for (ordinal, context_update) in [(0, true), (195, true), (196, false)] { + let mut optional = [0u8; G17P_COLD_OPENING_OPTIONAL_SIZE]; + apply_g17p_retained_optional_with_flist_generation_and_context_update( + stage, addresses, ordinal, 2022, false, 2, 3, 2, 0, 6, + context_update, &mut optional, + ) + .unwrap(); + assert_eq!(get_u16_le(&optional, 0x1a), 0); + assert_eq!(get_u16_le(&optional, 0x52), if ordinal == 0 { 1 } else { 0 }); + assert_eq!(get_u16_le(&optional, 0x62), u16::from(context_update)); + assert_eq!(get_u16_le(&optional, 0x32), 6); + } + } + } + + #[test] + fn ring_index_wraps_at_size() { + assert_eq!(ring_next(0, 4), Ok(1)); + assert_eq!(ring_next(3, 4), Ok(0)); + assert_eq!(ring_next(0x4ff, 0x500), Ok(0)); + assert_eq!(ring_next(0, 0), Err(SubmissionError::RingSizeZero)); + assert_eq!(ring_next(4, 4), Err(SubmissionError::RingIndexOutOfRange)); + } + + #[test] + fn kick_word_fields_match_the_firmware_extraction() { + // and x0,x5,#0xffffffffff ; ubfx x1,x5,#40,#7 ; ubfx x6,x5,#46,#1. + assert_eq!(KICK_TS_BITS, 40); + assert_eq!(KICK_TS_MASK, 0xff_ffff_ffff); + assert_eq!(KICK_QID_SHIFT, 40); + assert_eq!(KICK_QID_MASK, 0x7f); + assert_eq!(KICK_SELECTOR_BIT, 46); + + // Compose the exact word consumed from the firmware scratch FIFO; + // QID bit 6 is the mask selector. + let ts = 0x89_abcd_ef42u64; + let word = encode_kick_word(ts, 0x7f).unwrap(); + assert_eq!( + decode_kick_word(word), + Ok(SksmKickWord { + timestamp: ts, + queue_id: 0x7f, + mask_selector: 1, + }), + 1, + 0 + ); + // A word with bit 47 set is outside the modeled form. + assert_eq!( + decode_kick_word(1 << 47), + Err(SubmissionError::KickWordUnmodeledBits) + ); + assert_eq!( + encode_kick_word(KICK_TS_MASK + 1, 0), + Err(SubmissionError::KickTimestampOutOfRange) + ); + assert_eq!( + encode_kick_word(0, 0x80), + Err(SubmissionError::QueueIdOutOfRange) + ); + } + + #[test] + fn g17p_3d_completion_scratch_matches_native_full_command() { + assert_eq!( + encode_g17p_3d_completion_scratch( + 0xf76, + 1, + ), + Ok(0x0000_0100_0000_0f76), + ); + assert_eq!( + encode_g17p_3d_completion_scratch( + KICK_TS_MASK + 1, + 1, + ), + Err(SubmissionError::KickTimestampOutOfRange), + ); + } + + #[test] + fn g17p_serialized_render_scratch_payload_advances_and_wraps_independently() { + for (ordinal, timestamp, expected) in [ + (0, 1, 0x0000_0100_0000_0001), + (1, 2, 0x0000_0200_0000_0002), + (2, 3, 0x0000_0300_0000_0003), + (254, 255, 0x0000_ff00_0000_00ff), + (255, 256, 0x0000_0000_0000_0100), + (256, 257, 0x0000_0100_0000_0101), + // A channel payload is not inferred from the physical queue's + // slot or timestamp: distinct valid values must remain distinct. + (1, 0x1234_5678_9a, 0x0000_0212_3456_789a), + ] { + assert_eq!( + encode_g17p_3d_completion_scratch( + timestamp, g17p_serialized_render_scratch_payload(ordinal), + ), + Ok(expected), + ); + } + assert_eq!(g17p_serialized_render_scratch_payload(u64::from(u32::MAX)), 0); + } + + #[test] + fn kick_queue_preparation_couples_slot_queue_and_next_timestamp() { + let timestamp = encode_kick_timestamp(KickTimestamp { + stamp: 0x1234_5678, + stamp_index: 7, + }); + let prepared = prepare_kick_submission(timestamp, 0x2a, 16, 0x180).unwrap(); + + assert_eq!(prepared.entry_index, 7); + assert_eq!(prepared.entry_offset, 7 * 0x180); + assert_eq!(prepared.timestamp, 0x12_3456_7807); + assert_eq!(prepared.queue_id, 0x2a); + assert_eq!( + decode_kick_timestamp(prepared.next_timestamp), + Ok(KickTimestamp { + stamp: 0x1234_5678, + stamp_index: 8, + }) + ); + } + + #[test] + fn kick_queue_timestamp_wrap_matches_the_host_divide_remainder_sequence() { + // Slot 15 wraps modulo 16, so the host adds 0x100 to the timestamp: + // the low byte becomes zero and the 32-bit stamp increments. + let timestamp = encode_kick_timestamp(KickTimestamp { + stamp: 0x1234_5678, + stamp_index: 15, + }); + let prepared = prepare_kick_submission(timestamp, 0x7f, 16, 0x180).unwrap(); + assert_eq!(prepared.entry_index, 15); + assert_eq!(prepared.entry_offset, 15 * 0x180); + assert_eq!( + decode_kick_timestamp(prepared.next_timestamp), + Ok(KickTimestamp { + stamp: 0x1234_5679, + stamp_index: 0, + }) + ); + + // A 256-slot queue is representable despite num_stamps being larger + // than u8; slot 255 wraps and the 32-bit stamp wraps exactly like the + // host's masked add. + let timestamp = encode_kick_timestamp(KickTimestamp { + stamp: u32::MAX, + stamp_index: 0xff, + }); + let prepared = prepare_kick_submission(timestamp, 0x40, 256, 0x180).unwrap(); + assert_eq!(prepared.entry_index, 0xff); + assert_eq!( + decode_kick_timestamp(prepared.next_timestamp), + Ok(KickTimestamp { + stamp: 0, + stamp_index: 0, + }) + ); + } + + #[test] + fn g17p_hal200_queue_geometry_is_exact_and_other_backends_stay_unconfirmed() { + let geometry = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY; + assert_eq!(geometry.validate(), Ok(())); + assert_eq!(geometry.allocation_alignment(), 0x100); + assert_eq!(geometry.entry_stride(), 0x200); + assert_eq!(geometry.stamp_count(), 0x100); + assert_eq!(geometry.timestamp_seed(), 1); + assert_eq!(geometry.backing_size(), 0x20000); + assert_eq!( + geometry.entry_stride() as usize * geometry.stamp_count() as usize, + geometry.backing_size() + ); + assert_eq!( + g17_sksm_queue_geometry( + G17ChannelAbiTarget::A18ProG17P, + G17ClSubmissionBackend::G17PHal200Hybrid, + ), + Some(geometry) + ); + for backend in [ + G17ClSubmissionBackend::Classic, + G17ClSubmissionBackend::Hal300, + ] { + assert_eq!( + g17_sksm_queue_geometry(G17ChannelAbiTarget::A18ProG17P, backend), + None + ); + } + for target in [ + G17ChannelAbiTarget::M5G17G, + G17ChannelAbiTarget::M5ProMaxG17X, + ] { + assert_eq!( + g17_sksm_queue_geometry(target, G17ClSubmissionBackend::G17PHal200Hybrid), + None + ); + } + + let producer = geometry + .initial_producer_state( + 4, + 2, + G17PClQosConfig { + queue_byte_32: 1, + class_40: 8, + word_48: 0xffff, + }, + ) + .unwrap(); + assert_eq!(producer.geometry, geometry); + assert_eq!(producer.submitted, 0); + assert_eq!(producer.current_timestamp, geometry.timestamp_seed()); + assert_eq!(producer.last_add_kicks_timestamp, geometry.timestamp_seed()); + assert_eq!(producer.base_timestamp, geometry.timestamp_seed()); + assert!(!producer.previous_valid); + } + + #[test] + fn g17p_hal200_queue_geometry_drives_last_slot_and_wrap() { + let geometry = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY; + let timestamp = encode_kick_timestamp(KickTimestamp { + stamp: 0x1234_5678, + stamp_index: (geometry.stamp_count() - 1) as u8, + }); + let prepared = prepare_kick_submission( + timestamp, + 4, + geometry.stamp_count(), + geometry.entry_stride(), + ) + .unwrap(); + + assert_eq!(prepared.entry_index, 0xff); + assert_eq!( + prepared.entry_offset as usize + geometry.entry_stride() as usize, + geometry.backing_size() + ); + assert_eq!( + decode_kick_timestamp(prepared.next_timestamp), + Ok(KickTimestamp { + stamp: 0x1234_5679, + stamp_index: 0, + }) + ); + } + + #[test] + fn kick_queue_preparation_matches_all_runtime_stamp_geometries() { + for num_stamps in 1u16..=KICK_NUM_STAMPS_MAX { + for index in 0u16..num_stamps { + let current = KickTimestamp { + stamp: 0x89ab_cdef, + stamp_index: index as u8, + }; + let timestamp = encode_kick_timestamp(current); + let prepared = prepare_kick_submission(timestamp, 3, num_stamps, 0x180).unwrap(); + let expected_index = (index + 1) % num_stamps; + let expected_stamp = if expected_index == 0 { + current.stamp.wrapping_add(1) + } else { + current.stamp + }; + + assert_eq!(prepared.entry_index, index as u8); + assert_eq!(prepared.entry_offset, index as u32 * 0x180); + assert_eq!(prepared.timestamp, timestamp); + assert_eq!( + decode_kick_timestamp(prepared.next_timestamp), + Ok(KickTimestamp { + stamp: expected_stamp, + stamp_index: expected_index as u8, + }) + ); + } + } + } + + #[test] + fn kick_queue_preparation_reports_unrepresentable_state_before_publication() { + assert_eq!( + prepare_kick_submission(1 << 40, 0, 1, 0x180), + Err(SubmissionError::KickTimestampOutOfRange) + ); + assert_eq!( + prepare_kick_submission(0, 0, 0, 0x180), + Err(SubmissionError::KickStampCountOutOfRange) + ); + assert_eq!( + prepare_kick_submission(0, 0, 257, 0x180), + Err(SubmissionError::KickStampCountOutOfRange) + ); + assert_eq!( + prepare_kick_submission(4, 0, 4, 0x180), + Err(SubmissionError::KickStampIndexOutOfRange) + ); + assert_eq!( + prepare_kick_submission(0, 0, 1, 0), + Err(SubmissionError::KickEntryStrideZero) + ); + assert_eq!( + prepare_kick_submission(2, 0, 3, u32::MAX), + Err(SubmissionError::KickEntryOffsetOverflow) + ); + assert_eq!( + prepare_kick_submission(0, 0x80, 1, 0x180), + Err(SubmissionError::QueueIdOutOfRange) + ); + } + + #[test] + fn g17p_cl_kick_layout_and_publication_order_are_exact() { + assert_eq!(G17P_CL_KICK_ENTRY_SIZE, 0x180); + assert_eq!(G17P_CL_DEPENDENCIES_MAX, 32); + assert_eq!(G17P_CL_EXPLICIT_DEPENDENCIES_MAX, 31); + assert_eq!( + G17P_CL_KICK_QUEUE_STATE_LAYOUT, + G17PClKickQueueStateLayout { + queue_id: 0x0c, + num_stamps: 0x10, + entry_stride: 0x14, + completion_selector: 0x1c, + submitted: 0x44, + current_timestamp: 0x48, + last_add_kicks_timestamp: 0x50, + entry_cpu_base: 0x78, + entry_gpu_base: 0x80, + base_timestamp: 0x90, + previous_valid: 0x98, + previous_timestamp: 0xa0, + } + ); + assert_eq!( + G17P_CL_KICK_ENTRY_LAYOUT, + G17PClKickEntryLayout { + header: 0x00, + mcache: 0x08, + payload: [0x10, 0x18], + dependency_control: 0x20, + dependencies: 0x28, + event_mask: [0x128, 0x130, 0x138, 0x140], + rce_kind: 0x148, + rce_bindings: [0x150, 0x158, 0x160, 0x168], + zero_170: 0x170, + auxiliary: 0x178, + } + ); + assert_eq!( + G17P_CL_KICK_PUBLICATION_ORDER, + [ + G17PClKickPublicationStep::ZeroEntryStride, + G17PClKickPublicationStep::EntryBody, + G17PClKickPublicationStep::InnerShareableBarrier, + G17PClKickPublicationStep::NextTimestamp, + G17PClKickPublicationStep::SubmittedAtomicAdd, + G17PClKickPublicationStep::PreviousValid, + G17PClKickPublicationStep::PreviousTimestamp, + ] + ); + assert_eq!( + G17P_FIRST_SUBMIT_PUBLICATION_ORDER, + [ + G17PFirstSubmitPublicationStep::AcquireGfx1RunningPower, + G17PFirstSubmitPublicationStep::PublishB2, + G17PFirstSubmitPublicationStep::PublishConfigUpdatePointer, + G17PFirstSubmitPublicationStep::ConfigureQid4Hardware, + G17PFirstSubmitPublicationStep::ZeroSksmStride, + G17PFirstSubmitPublicationStep::WriteSksmEntry, + G17PFirstSubmitPublicationStep::EntryBarrier, + G17PFirstSubmitPublicationStep::PublishSksmProducer, + G17PFirstSubmitPublicationStep::AdvanceDescriptorGeneration, + G17PFirstSubmitPublicationStep::RecordHostCorrelationToken, + G17PFirstSubmitPublicationStep::PublishEntrySignalPointer, + G17PFirstSubmitPublicationStep::AddKicks, + G17PFirstSubmitPublicationStep::PublishCl2, + G17PFirstSubmitPublicationStep::SendActivation, + G17PFirstSubmitPublicationStep::DirectKickBarrier, + G17PFirstSubmitPublicationStep::SendDirectKick, + ] + ); + } + + #[test] + fn g17p_conditional_transport_events_match_the_independent_fixtures() { + const PRIMARY: &[u8; G17P_COMPUTE_PRIMARY_EVENT_SIZE] = + include_bytes!("fixtures/g17p_first_submit/channel-event-primary.bin"); + const OPTIONAL: &[u8; G17P_COMPUTE_OPTIONAL_EVENT_SIZE] = + include_bytes!("fixtures/g17p_first_submit/channel-event-optional.bin"); + + let primary = prepare_g17p_compute_primary_event(0, 0, 4, 2, 1).unwrap(); + assert_eq!(primary.descriptor_generation, 1); + assert_eq!(primary.bytes, *PRIMARY); + assert_eq!(advance_g17p_compute_descriptor_generation(0, 0), Ok(1)); + assert_eq!(advance_g17p_compute_descriptor_generation(2, 3), Ok(6)); + + let mut optional = [0u8; G17P_COMPUTE_OPTIONAL_EVENT_SIZE]; + encode_g17p_compute_optional_event( + G17PComputeOptionalEvent { + queue_id: 4, + old_timestamp: 1, + }, + &mut optional, + ) + .unwrap(); + assert_eq!(optional, *OPTIONAL); + + assert_eq!( + prepare_g17p_compute_primary_event(u32::MAX, 0, 4, 2, 1), + Err(SubmissionError::ComputeEventCounterOverflow) + ); + assert_eq!( + prepare_g17p_compute_primary_event(u16::MAX as u32, 0, 4, 2, 1), + Err(SubmissionError::ComputeEventCounterOverflow) + ); + + // The tag-14 transport builder reproduces the same captured record. + // mac17p 0xfffffe00089fa39c writes type 14, QID 4, one kick, the queue + // QoS priority 2, the pre-increment entry stamp 1, and data master 2. + let add_kicks = prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: 4, + add_count: 1, + queue_priority: 2, + entry_stamp: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.timestamp_seed(), + }) + .unwrap(); + assert_eq!(add_kicks.bytes, *PRIMARY); + assert_eq!(add_kicks.bytes.len(), G17P_KSM_ADD_KICKS_COMMAND_SIZE); + assert!(G17P_KSM_ADD_KICKS_COMMAND_SIZE <= G17P_KSM_ADD_KICKS_COMMAND_FETCH_SIZE); + assert_eq!(add_kicks.bytes[0x08], 2); + assert_eq!( + u32::from_le_bytes(add_kicks.bytes[0x11..0x15].try_into().unwrap()), + G17P_COMPUTE_DATA_MASTER + ); + + assert_eq!( + prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: 4, + add_count: 0, + queue_priority: 2, + entry_stamp: 1, + }), + Err(SubmissionError::KsmAddKicksCountZero) + ); + assert_eq!( + prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: KICK_QID_MASK + 1, + add_count: 1, + queue_priority: 2, + entry_stamp: 1, + }), + Err(SubmissionError::QueueIdOutOfRange) + ); + assert_eq!( + prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: 4, + add_count: 1, + queue_priority: 4, + entry_stamp: 1, + }), + Err(SubmissionError::KickQueueCompletionSelectorOutOfRange) + ); + assert_eq!( + prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: 4, + add_count: 1, + queue_priority: 2, + entry_stamp: 1 << KICK_TS_BITS, + }), + Err(SubmissionError::KickTimestampOutOfRange) + ); + assert_eq!( + prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: 4, + add_count: u16::MAX as u32 + 1, + queue_priority: 2, + entry_stamp: 1, + }), + Err(SubmissionError::KsmAddKicksCountOutOfRange) + ); + } + + #[test] + fn g17p_cl_hardware_positive_retained_items_are_exact() { + const QUEUE: u64 = 0xffff_fc20_c000_0300; + const DESCRIPTOR: u64 = 0xffff_fc20_c035_8000; + const DESCRIPTOR_STRIDE: u64 = 0x1040; + let item0_queue = G17PClKickProducerState { + queue_id: 4, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 2, + qos: G17PClQosConfig { + queue_byte_32: 1, + class_40: 8, + word_48: 0xffff, + }, + submitted: 0, + current_timestamp: 1, + last_add_kicks_timestamp: 1, + base_timestamp: 1, + previous_valid: false, + previous_timestamp: 0, + }; + let item0 = prepare_g17p_cl_kick_entry( + item0_queue, + native_add3_compute_operands( + DESCRIPTOR, + QUEUE, + 1, + [0, 0, 4, 0], + [0x0001_1003_8001_a002, 0x0000_2003_8001_a03b], + ), + ) + .unwrap(); + let mut expected0 = [0u8; G17P_CL_KICK_ENTRY_SIZE]; + for (offset, value) in [ + (0x00, 0x1000_1000_0000_0004), + (0x10, DESCRIPTOR), + (0x18, QUEUE), + (0x20, 0xffff_0801_0000_0001), + (0x28, 0x0000_0400_0000_0000), + (0x138, 4), + (0x150, 0x0001_1003_8001_a002), + (0x158, 0x0000_2003_8001_a03b), + (0x178, 0x003f_ffff_ffff_ffff), + ] { + put_u64_le(&mut expected0, offset, value); + } + assert_eq!(item0.entry, expected0); + assert_eq!(item0.zero_length, 0x200); + assert_eq!(item0.entry_index, 1); + assert_eq!(item0.entry_offset, 0x200); + assert_eq!(item0.current_timestamp, 1); + assert_eq!(item0.next_timestamp, 2); + assert_eq!(item0.submitted_before, 0); + assert_eq!(item0.submitted_after, 1); + assert_eq!(item0.previous_timestamp_after, 1); + assert_eq!(item0.producer_after.current_timestamp, 2); + assert_eq!(item0.producer_after.submitted, 1); + assert!(item0.producer_after.previous_valid); + assert_eq!(item0.producer_after.previous_timestamp, 1); + assert_eq!(item0.producer_after.qos, item0_queue.qos); + + let item1 = prepare_g17p_cl_kick_entry( + item0.producer_after, + native_add3_compute_operands( + DESCRIPTOR + DESCRIPTOR_STRIDE, + QUEUE, + 0, + [0, 2, 8, 0], + [0x0001_1003_8001_a084, 0x0000_2003_8001_a0bd], + ), + ) + .unwrap(); + let mut expected1 = [0u8; G17P_CL_KICK_ENTRY_SIZE]; + for (offset, value) in [ + (0x00, 0x1000_1000_0000_0008), + (0x10, DESCRIPTOR + DESCRIPTOR_STRIDE), + (0x18, QUEUE), + (0x20, 0xffff_0801_0000_0001), + (0x28, 0x0000_0400_0000_0001), + (0x130, 2), + (0x138, 8), + (0x150, 0x0001_1003_8001_a084), + (0x158, 0x0000_2003_8001_a0bd), + (0x178, 0x003f_ffff_ffff_ffff), + ] { + put_u64_le(&mut expected1, offset, value); + } + assert_eq!(item1.entry, expected1); + assert_eq!(item1.zero_length, 0x200); + assert_eq!(item1.entry_index, 2); + assert_eq!(item1.entry_offset, 0x400); + assert_eq!(item1.current_timestamp, 2); + assert_eq!(item1.next_timestamp, 3); + assert_eq!(item1.submitted_before, 1); + assert_eq!(item1.submitted_after, 2); + assert_eq!(item1.previous_timestamp_after, 2); + assert_eq!(item1.producer_after.current_timestamp, 3); + assert_eq!(item1.producer_after.submitted, 2); + assert_eq!(item1.producer_after.previous_timestamp, 2); + assert_eq!(item1.producer_after.qos, item0_queue.qos); + + // Command two keeps physical queue 0's QoS word while advancing its + // command-owned event mask from 4 to 8. + assert_eq!(get_u64_le(&item1.entry, 0x00), 0x1000_1000_0000_0008); + assert_eq!(get_u64_le(&item1.entry, 0x28), 0x0000_0400_0000_0001); + assert_eq!( + get_u64_le(&item1.entry, 0x20), + get_u64_le(&item0.entry, 0x20) + ); + assert_eq!(get_u64_le(&item0.entry, 0x138), 4); + assert_eq!(get_u64_le(&item1.entry, 0x138), 8); + assert_eq!( + get_u64_le(&item1.entry, 0x178), + get_u64_le(&item0.entry, 0x178) + ); + } + + #[test] + fn g17p_first_submit_matches_the_independent_full_stride_fixture() { + const BEFORE: &[u8; 0x200] = + include_bytes!("fixtures/g17p_first_submit/sksm-entry-before.bin"); + const AFTER: &[u8; 0x200] = + include_bytes!("fixtures/g17p_first_submit/sksm-entry-after.bin"); + const RELOCATION_MASK: &[u8; 0x200] = + include_bytes!("fixtures/g17p_first_submit/sksm-entry-relocation-mask.bin"); + const DESCRIPTOR_HIGH: u64 = 0xffff_fc20_c035_8000; + const DESCRIPTOR_LOW: u64 = 0x70_0034_0000; + const QUEUE: u64 = 0xffff_fc20_c000_0300; + + assert!(BEFORE.iter().all(|byte| *byte == 0)); + assert_eq!(G17P_FIRST_BIND_COMMAND_PLAN.fallback_logical_flush_id, 0); + assert!(!G17P_FIRST_BIND_COMMAND_PLAN.flush_id_serialized_in_tag15); + assert!(!G17P_FIRST_BIND_COMMAND_PLAN.flush_id_completion_release_required); + assert_eq!( + RELOCATION_MASK + .iter() + .map(|byte| byte.count_ones()) + .sum::(), + 204 + ); + + let command = prepare_g17p_compute_cl_command_state(DESCRIPTOR_LOW, 0, 0x26).unwrap(); + let empty = G17PClRceBinding { address: 0, tag: 0 }; + assert_eq!( + command.rce_bindings, + [ + G17PClRceBinding { + address: DESCRIPTOR_LOW + 0x40, + tag: 0x26, + }, + G17PClRceBinding { + address: DESCRIPTOR_LOW + 0x760, + tag: 0x04, + }, + empty, + empty, + ] + ); + let before = G17PClKickProducerState { + queue_id: 4, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 2, + qos: G17PClQosConfig { + queue_byte_32: 1, + class_40: 8, + word_48: 0xffff, + }, + submitted: 0, + current_timestamp: 1, + last_add_kicks_timestamp: 1, + base_timestamp: 1, + previous_valid: false, + previous_timestamp: 0, + }; + let prepared = prepare_g17p_cl_kick_entry( + before, + G17PClKickEntryOperands { + descriptor_flag_4c: false, + descriptor_flag_5c8: false, + converted_command_timestamp: 1, + barriers: &[], + mcache: None, + payload: [DESCRIPTOR_HIGH, QUEUE], + event_mask: command.event_mask, + rce_kind: 0, + rce_bindings: command.rce_bindings, + auxiliary: 0x003f_ffff_ffff_ffff, + }, + ) + .unwrap(); + let mut actual = [0u8; 0x200]; + actual[..G17P_CL_KICK_ENTRY_SIZE].copy_from_slice(&prepared.entry); + + for index in 0..actual.len() { + assert_eq!( + actual[index] & !RELOCATION_MASK[index], + AFTER[index] & !RELOCATION_MASK[index], + "normalized fixture mismatch at offset {:#x}", + index + ); + } + // The fixture uses the same captured addresses as this test, so the + // relocation-bearing fields must also agree without normalization. + assert_eq!(actual, *AFTER); + assert_eq!(prepared.zero_length, 0x200); + assert_eq!(prepared.entry_index, 1); + assert_eq!(prepared.entry_offset, 0x200); + assert_eq!(prepared.current_timestamp, 1); + assert_eq!(prepared.next_timestamp, 2); + assert_eq!(prepared.submitted_before, 0); + assert_eq!(prepared.submitted_after, 1); + assert_eq!( + prepared.producer_after, + G17PClKickProducerState { + submitted: 1, + current_timestamp: 2, + previous_valid: true, + previous_timestamp: 1, + ..before + } + ); + + const SECOND_AFTER: &[u8; 0x200] = + include_bytes!("fixtures/g17p_first_submit/sksm-entry-second-after.bin"); + let second_command = + prepare_g17p_compute_cl_command_state(DESCRIPTOR_LOW + 0x1040, 1, 0x26).unwrap(); + let second = prepare_g17p_cl_kick_entry( + prepared.producer_after, + G17PClKickEntryOperands { + descriptor_flag_4c: false, + descriptor_flag_5c8: false, + converted_command_timestamp: 0, + barriers: &[], + mcache: None, + payload: [DESCRIPTOR_HIGH + 0x1040, QUEUE], + event_mask: second_command.event_mask, + rce_kind: 0, + rce_bindings: second_command.rce_bindings, + auxiliary: 0x003f_ffff_ffff_ffff, + }, + ) + .unwrap(); + let mut second_actual = [0u8; 0x200]; + second_actual[..G17P_CL_KICK_ENTRY_SIZE].copy_from_slice(&second.entry); + assert_eq!(second_actual, *SECOND_AFTER); + assert_eq!(second.entry_index, 2); + assert_eq!(second.entry_offset, 0x400); + assert_eq!(second.current_timestamp, 2); + assert_eq!(second.next_timestamp, 3); + assert_eq!(second.submitted_before, 1); + assert_eq!(second.submitted_after, 2); + assert_eq!(second.producer_after.current_timestamp, 3); + assert_eq!(second.producer_after.previous_timestamp, 2); + assert_eq!(second.producer_after.last_add_kicks_timestamp, 1); + } + + #[test] + fn g17p_cl_descriptor_flags_map_directly_to_header_bits() { + let empty = G17PClRceBinding { address: 0, tag: 0 }; + let producer = G17PClKickProducerState { + queue_id: 4, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 2, + qos: G17PClQosConfig { + queue_byte_32: 1, + class_40: 8, + word_48: 0xffff, + }, + submitted: 0, + current_timestamp: 1, + last_add_kicks_timestamp: 1, + base_timestamp: 1, + previous_valid: false, + previous_timestamp: 0, + }; + let flag_mask = (1u64 << 1) | (1u64 << 61); + + for (flag_4c, flag_5c8, expected) in [ + (false, false, 0), + (true, false, 1u64 << 1), + (false, true, 1u64 << 61), + (true, true, flag_mask), + ] { + let prepared = prepare_g17p_cl_kick_entry( + producer, + G17PClKickEntryOperands { + descriptor_flag_4c: flag_4c, + descriptor_flag_5c8: flag_5c8, + converted_command_timestamp: 1, + barriers: &[], + mcache: None, + payload: [0; 2], + event_mask: [0; 4], + rce_kind: 0, + rce_bindings: [empty; 4], + auxiliary: 0, + }, + ) + .unwrap(); + + assert_eq!(get_u64_le(&prepared.entry, 0) & flag_mask, expected); + } + } + + #[test] + fn g17p_cl_completion_updates_only_the_submitted_count() { + let producer = G17PClKickProducerState { + queue_id: 4, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 2, + qos: G17PClQosConfig { + queue_byte_32: 1, + class_40: 8, + word_48: 0xffff, + }, + submitted: 2, + current_timestamp: 3, + last_add_kicks_timestamp: 2, + base_timestamp: 1, + previous_valid: true, + previous_timestamp: 2, + }; + + assert_eq!(complete_g17p_cl_kicks(producer, 0), Ok(producer)); + let after_one = complete_g17p_cl_kicks(producer, 1).unwrap(); + assert_eq!( + after_one, + G17PClKickProducerState { + submitted: 1, + ..producer + } + ); + let empty = complete_g17p_cl_kicks(producer, 2).unwrap(); + assert_eq!(empty.submitted, 0); + assert_eq!(complete_g17p_cl_kicks(empty, 0), Ok(empty)); + assert_eq!( + complete_g17p_cl_kicks(empty, 1), + Err(SubmissionError::KickCompletionUnderflow) + ); + assert_eq!( + complete_g17p_cl_kicks(producer, 3), + Err(SubmissionError::KickCompletionUnderflow) + ); + } + + #[test] + fn g17p_cl_retained_storage_overlap_and_order_are_exact() { + let layout = G17P_CL_RETAINED_STORAGE_LAYOUT; + assert_eq!( + layout, + G17PClRetainedStorageLayout { + shared_support: G17PClStorageRange { + offset: 0, + size: 0x4000, + }, + scheduler: G17PClStorageRange { + offset: 0x200, + size: 0x100, + }, + } + ); + assert_eq!( + layout.scheduler.offset - layout.shared_support.offset, + 0x200 + ); + assert!(g17p_cl_range_contains( + layout.shared_support, + layout.scheduler + )); + assert!(g17p_cl_ranges_overlap( + layout.shared_support, + layout.scheduler + )); + assert!(g17p_cl_full_support_initialization_allowed(false)); + assert!(!g17p_cl_full_support_initialization_allowed(true)); + + assert_eq!( + G17P_CL_RETAINED_REGISTRATION_ORDER, + [ + G17PClRetainedPublicationStep::InitializeSharedSupport, + G17PClRetainedPublicationStep::WriteSchedulerRecord, + G17PClRetainedPublicationStep::RegisterPhysicalQueue, + ] + ); + assert_eq!( + G17P_CL_RETAINED_ACTIVATION_ORDER, + [ + G17PClRetainedPublicationStep::PublishCommandOwnedState, + G17PClRetainedPublicationStep::CleanCommandOwnedState, + G17PClRetainedPublicationStep::WriteB2PrimaryState, + G17PClRetainedPublicationStep::CleanB2PrimaryState, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerPayload, + G17PClRetainedPublicationStep::PublishInnerQueuePayload, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerProducer, + G17PClRetainedPublicationStep::PublishInnerQueueProducer, + G17PClRetainedPublicationStep::SystemBarrierAfterInnerProducer, + ] + ); + assert_eq!( + G17P_CL_RETAINED_SUBMISSION_ORDER, + [ + G17PClRetainedPublicationStep::PublishCommandOwnedState, + G17PClRetainedPublicationStep::CleanCommandOwnedState, + G17PClRetainedPublicationStep::PublishInnerQueuePayload, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerProducer, + G17PClRetainedPublicationStep::PublishInnerQueueProducer, + G17PClRetainedPublicationStep::SystemBarrierAfterInnerProducer, + ] + ); + assert_eq!( + g17p_cl_retained_submission_order(G17PClB2ActivationState::Pending), + G17P_CL_RETAINED_ACTIVATION_ORDER + ); + assert_eq!( + g17p_cl_retained_submission_order(G17PClB2ActivationState::Published), + G17P_CL_RETAINED_SUBMISSION_ORDER + ); + } + + #[test] + fn g17p_cl_b2_primary_state_builder_is_byte_exact() { + const FILL: u8 = 0xa5; + const CHANNEL_CONTROL: u64 = 0xffff_fc20_1234_8000; + let layout = G17P_CL_B2_PRIMARY_STATE_LAYOUT; + assert_eq!( + layout, + G17PClB2PrimaryStateLayout { + object_size: 0x4000, + computed_write_extent: 0xe30, + region_1_write_extent: 0x20, + region_2_write_extent: 0x18, + } + ); + + let mut computed = [FILL; 0x4000]; + let mut region_1 = [FILL; 0x4000]; + let mut region_2 = [FILL; 0x4000]; + let activation = apply_g17p_cl_b2_primary_state( + G17PClB2ActivationState::Pending, + &mut computed, + &mut region_1, + &mut region_2, + CHANNEL_CONTROL, + ) + .unwrap(); + assert_eq!(activation, G17PClB2ActivationState::Published); + + let mut expected_computed = [FILL; 0x4000]; + expected_computed[0x000] = 0xff; + expected_computed[0x008] = 0xff; + expected_computed[0x010..0x012].copy_from_slice(&0x0f01u16.to_le_bytes()); + expected_computed[0x018..0x01a].copy_from_slice(&0x0f01u16.to_le_bytes()); + for (offset, value) in [(0x408, 2), (0x40c, 2), (0x604, 4), (0xc04, 0x1000)] { + put_u32_le(&mut expected_computed, offset, value); + } + for (offset, value) in [ + (0x808, CHANNEL_CONTROL), + (0xe00, 7), + (0xe08, 0x0000_0001_0e22_618a), + (0xe20, 4), + (0xe28, 4), + ] { + put_u64_le(&mut expected_computed, offset, value); + } + + let mut expected_region_1 = [FILL; 0x4000]; + put_u64_le(&mut expected_region_1, 0x10, 0x0000_0080_0007_7000); + put_u32_le(&mut expected_region_1, 0x18, 0x17); + put_u32_le(&mut expected_region_1, 0x1c, 0x17); + + let mut expected_region_2 = [FILL; 0x4000]; + put_u64_le(&mut expected_region_2, 0x00, 0x0800_0000_e000_0000); + put_u64_le(&mut expected_region_2, 0x08, 0x0000_3d40_0000_3400); + put_u64_le(&mut expected_region_2, 0x10, 0x0000_0000_0000_1d00); + + assert_eq!(computed, expected_computed); + assert_eq!(region_1, expected_region_1); + assert_eq!(region_2, expected_region_2); + assert_eq!( + apply_g17p_cl_b2_primary_state( + activation, + &mut computed, + &mut region_1, + &mut region_2, + CHANNEL_CONTROL, + ), + Err(SubmissionError::KickB2PrimaryStateAlreadyPublished) + ); + assert_eq!(computed, expected_computed); + assert_eq!(region_1, expected_region_1); + assert_eq!(region_2, expected_region_2); + + let mut short = [0u8; 1]; + assert_eq!( + apply_g17p_cl_b2_primary_state( + G17PClB2ActivationState::Pending, + &mut short, + &mut region_1, + &mut region_2, + CHANNEL_CONTROL, + ), + Err(SubmissionError::KickB2PrimaryStateBufferTooSmall) + ); + } + + #[test] + fn g17p_cl_entry_encodes_every_optional_section() { + let barriers = [G17PClBarrierDependency { + native_timestamp: None, + event_word: 0x100, + shared_stamp: 0, + queue_id: 3, + }]; + let rce_bindings = [ + G17PClRceBinding { + address: 0x2000, + tag: 1, + }, + G17PClRceBinding { + address: 0x4000, + tag: 2, + }, + G17PClRceBinding { + address: 0x6000, + tag: 3, + }, + G17PClRceBinding { + address: 0x8000, + tag: 4, + }, + ]; + let queue = G17PClKickProducerState { + queue_id: 4, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 2, + qos: G17PClQosConfig { + queue_byte_32: 0x12, + class_40: 0x1b, + word_48: 0x3456, + }, + submitted: 2, + current_timestamp: 2, + last_add_kicks_timestamp: 1, + base_timestamp: 1, + previous_valid: true, + previous_timestamp: 7, + }; + let prepared = prepare_g17p_cl_kick_entry( + queue, + G17PClKickEntryOperands { + descriptor_flag_4c: true, + descriptor_flag_5c8: true, + converted_command_timestamp: 0, + barriers: &barriers, + mcache: Some(G17PClMcacheAperture { + address: 0x12340, + index: 5, + count: 3, + }), + payload: [0x1111_2222_3333_4444, 0x5555_6666_7777_8888], + event_mask: [0x10, 0x20, 0x30, 0x40], + rce_kind: 2, + rce_bindings, + auxiliary: 0xa5a5_5a5a_1234_5678, + }, + ) + .unwrap(); + let header = + (2u64 << 2) | (4u64 << 42) | (1u64 << 58) | (2u64 << 59) | (1u64 << 1) | (1u64 << 61); + assert_eq!(get_u64_le(&prepared.entry, 0x00), header); + assert_eq!( + get_u64_le(&prepared.entry, 0x08), + 5 | ((0x12340u64 >> 5) << 6) | (2u64 << 54) + ); + assert_eq!(get_u64_le(&prepared.entry, 0x10), 0x1111_2222_3333_4444); + assert_eq!(get_u64_le(&prepared.entry, 0x18), 0x5555_6666_7777_8888); + assert_eq!(get_u64_le(&prepared.entry, 0x20), 0x3456_1b12_0000_0003); + assert_eq!(get_u64_le(&prepared.entry, 0x28), (3u64 << 40) | 1); + assert_eq!(get_u64_le(&prepared.entry, 0x30), (4u64 << 40) | 7); + assert_eq!( + [ + get_u64_le(&prepared.entry, 0x128), + get_u64_le(&prepared.entry, 0x130), + get_u64_le(&prepared.entry, 0x138), + get_u64_le(&prepared.entry, 0x140), + ], + [0x10, 0x20, 0x30, 0x40] + ); + assert_eq!(prepared.entry[0x148], 2); + for (index, binding) in rce_bindings.into_iter().enumerate() { + assert_eq!( + get_u64_le(&prepared.entry, 0x150 + index * 8), + (binding.address >> 5) | ((binding.tag as u64) << 43) + ); + } + assert_eq!(get_u64_le(&prepared.entry, 0x170), 0); + assert_eq!(get_u64_le(&prepared.entry, 0x178), 0xa5a5_5a5a_1234_5678); + assert_eq!(prepared.submitted_after, 3); + assert_eq!(prepared.previous_timestamp_after, 2); + } + + /// G17P's native graphics wrappers select stage-specific launch classes, + /// while direct entry-ring captures show the same 0xffff word for both + /// ordinary TA and 3D kicks. Keep those exact +0x20 qwords pinned + /// independently of the CL profile. + #[test] + fn g17p_native_render_dependency_control_words_are_exact() { + fn operands(barriers: &[G17PClBarrierDependency]) -> G17PClKickEntryOperands<'_> { + let empty = G17PClRceBinding { address: 0, tag: 0 }; + G17PClKickEntryOperands { + descriptor_flag_4c: false, + descriptor_flag_5c8: false, + converted_command_timestamp: 1, + barriers, + mcache: None, + payload: [0; 2], + event_mask: [0; 4], + rce_kind: 0, + rce_bindings: [empty; 4], + auxiliary: 0, + } + } + + let ta = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY + .initial_producer_state( + 0, + 2, + G17PClQosConfig { + queue_byte_32: 2, + class_40: 0x0c, + word_48: 0xffff, + }, + ) + .unwrap(); + let ta = prepare_g17p_cl_kick_entry(ta, operands(&[])).unwrap(); + assert_eq!(get_u64_le(&ta.entry, 0x20), 0xffff_0c02_0000_0001); + + let fragment_barrier = [G17PClBarrierDependency { + native_timestamp: None, + event_word: 0x100, + shared_stamp: 0, + queue_id: 0, + }]; + let fragment = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY + .initial_producer_state( + 1, + 2, + G17PClQosConfig { + queue_byte_32: 2, + class_40: 0x18, + word_48: 0xffff, + }, + ) + .unwrap(); + let fragment = + prepare_g17p_cl_kick_entry(fragment, operands(&fragment_barrier)).unwrap(); + assert_eq!( + get_u64_le(&fragment.entry, 0x20), + 0xffff_1802_0000_0003 + ); + } + + #[test] + fn g17p_cl_entry_reports_invalid_geometry_and_fields() { + let empty = G17PClRceBinding { address: 0, tag: 0 }; + let queue = G17PClKickProducerState { + queue_id: 0, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 0, + qos: G17PClQosConfig { + queue_byte_32: 0, + class_40: 0, + word_48: 0, + }, + submitted: 0, + current_timestamp: 1, + last_add_kicks_timestamp: 0, + base_timestamp: 1, + previous_valid: false, + previous_timestamp: 0, + }; + let operands = G17PClKickEntryOperands { + descriptor_flag_4c: true, + descriptor_flag_5c8: true, + converted_command_timestamp: 1, + barriers: &[], + mcache: None, + payload: [0; 2], + event_mask: [0; 4], + rce_kind: 0, + rce_bindings: [empty; 4], + auxiliary: 0, + }; + + assert_eq!( + prepare_g17p_cl_kick_entry( + G17PClKickProducerState { + geometry: G17SksmQueueGeometry { + entry_stride: 0, + ..T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY + }, + ..queue + }, + operands, + ), + Err(SubmissionError::KickQueueGeometryInvalid) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + G17PClKickProducerState { + geometry: G17SksmQueueGeometry { + entry_stride: 0x17f, + ..T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY + }, + ..queue + }, + operands, + ), + Err(SubmissionError::KickQueueGeometryInvalid) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + G17PClKickProducerState { + geometry: G17SksmQueueGeometry { + stamp_count: 0, + ..T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY + }, + ..queue + }, + operands, + ), + Err(SubmissionError::KickQueueGeometryInvalid) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + G17PClKickProducerState { + submitted: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.stamp_count() as u32, + ..queue + }, + operands, + ), + Err(SubmissionError::KickQueueSubmittedFull) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + G17PClKickProducerState { + qos: G17PClQosConfig { + class_40: 0x20, + ..queue.qos + }, + ..queue + }, + operands, + ), + Err(SubmissionError::KickQosClassOutOfRange) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + queue, + G17PClKickEntryOperands { + mcache: Some(G17PClMcacheAperture { + address: 0x20, + index: 0x40, + count: 1, + }), + ..operands + }, + ), + Err(SubmissionError::KickMcacheFieldOutOfRange) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + queue, + G17PClKickEntryOperands { + rce_bindings: [G17PClRceBinding { address: 1, tag: 0 }, empty, empty, empty,], + ..operands + }, + ), + Err(SubmissionError::KickRceAddressUnencodable) + ); + + let too_many = [G17PClBarrierDependency { + native_timestamp: None, + event_word: 0, + shared_stamp: 0, + queue_id: 0, + }; G17P_CL_DEPENDENCIES_MAX]; + assert_eq!( + prepare_g17p_cl_kick_entry( + queue, + G17PClKickEntryOperands { + barriers: &too_many, + ..operands + }, + ), + Err(SubmissionError::KickDependencyCountOutOfRange) + ); + } + + #[test] + fn host_add_kicks_tags_match_all_three_pinned_producers() { + let geometry = G17SksmScratchGeometry { + write0_base: 0x100, + write1_base: 0x104, + stride: 3, + }; + + for queue_id in 0u8..=0x7f { + let p = prepare_sksm_add_kicks_mmio(SksmAddKicksTarget::G17P, queue_id, 5, geometry) + .unwrap(); + assert_eq!(p.tag, 0x0000_0400_0002_1018 | ((queue_id as u64) << 43)); + assert_eq!(p.tag_offset, 0x11b); + assert_eq!(p.count_offset, 0x11f); + assert_eq!(p.add_count, 5); + assert_eq!( + p.write_pair(), + G17SksmOrderedWritePair { + word0_offset: 0x11b, + word0: 0x0000_0400_0002_1018 | ((queue_id as u64) << 43), + word1_offset: 0x11f, + word1: 5, + } + ); + + let g = prepare_sksm_add_kicks_mmio( + SksmAddKicksTarget::G17G, + queue_id, + 0x89ab_cdef, + geometry, + ) + .unwrap(); + assert_eq!(g.tag, 0x0000_0800_0002_1018 | ((queue_id as u64) << 44)); + + let x = + prepare_sksm_add_kicks_mmio(SksmAddKicksTarget::G17X, queue_id, u32::MAX, geometry) + .unwrap(); + assert_eq!(x.tag, 0x0020_0800_0002_1018 | ((queue_id as u64) << 44)); + assert_eq!(x.add_count, u32::MAX); + } + } + + #[test] + fn pair_writer_publishes_two_64_bit_words_in_order() { + use std::cell::RefCell; + + struct Recorder(RefCell>); + + impl G17SksmPairWriter for Recorder { + type Error = (); + + fn write64(&self, offset: u32, value: u64) -> core::result::Result<(), Self::Error> { + self.0.borrow_mut().push((offset, value)); + Ok(()) + } + } + + let pair = prepare_sksm_add_kicks_mmio( + SksmAddKicksTarget::G17P, + 0x2a, + 0x89ab_cdef, + g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch, + ) + .unwrap() + .write_pair(); + let recorder = Recorder(RefCell::new(Vec::new())); + + assert_eq!(publish_g17p_sksm_write_pair(&recorder, pair), Ok(())); + assert_eq!( + recorder.0.into_inner(), + vec![ + (0x4090, 0x0001_5400_0002_1018), + (0x4098, 0x0000_0000_89ab_cdef), + ] + ); + } + + #[test] + fn pair_writer_does_not_publish_word1_after_word0_error() { + use std::cell::Cell; + + struct FirstWriteError(Cell); + + impl G17SksmPairWriter for FirstWriteError { + type Error = u32; + + fn write64(&self, _offset: u32, _value: u64) -> core::result::Result<(), Self::Error> { + let calls = self.0.get() + 1; + self.0.set(calls); + Err(calls) + } + } + + let writer = FirstWriteError(Cell::new(0)); + let pair = G17SksmOrderedWritePair { + word0_offset: 0x4090, + word0: 1, + word1_offset: 0x4098, + word1: 2, + }; + assert_eq!(publish_g17p_sksm_write_pair(&writer, pair), Err(1)); + assert_eq!(writer.0.get(), 1); + } + + #[test] + fn host_add_kicks_mmio_offsets_report_overflow() { + let valid = G17SksmScratchGeometry { + write0_base: 0, + write1_base: 0, + stride: u32::MAX / 9, + }; + let publication = + prepare_sksm_add_kicks_mmio(SksmAddKicksTarget::G17G, 0, 1, valid).unwrap(); + assert_eq!(publication.tag_offset, (u32::MAX / 9) * 9); + + assert_eq!( + prepare_sksm_add_kicks_mmio( + SksmAddKicksTarget::G17P, + 0x80, + 1, + G17SksmScratchGeometry { + write0_base: 0, + write1_base: 4, + stride: 0, + }, + ), + Err(SubmissionError::QueueIdOutOfRange) + ); + for geometry in [ + G17SksmScratchGeometry { + write0_base: 0, + write1_base: 0, + stride: u32::MAX / 9 + 1, + }, + G17SksmScratchGeometry { + write0_base: u32::MAX, + write1_base: 0, + stride: 1, + }, + G17SksmScratchGeometry { + write0_base: 0, + write1_base: u32::MAX, + stride: 1, + }, + ] { + assert_eq!( + prepare_sksm_add_kicks_mmio(SksmAddKicksTarget::G17X, 0, 1, geometry), + Err(SubmissionError::KickMmioOffsetOverflow) + ); + } + } + + #[test] + fn g17p_sksm_queue_configuration_matches_the_host_store_sequence() { + assert_eq!( + G17P_SKSM_RUNTIME_GEOMETRY_OBJECT_LAYOUT, + G17PSksmRuntimeGeometryObjectLayout { + write0_base_field: 0x10730, + write1_base_field: 0x10734, + stride_field: 0x10738, + } + ); + assert_eq!(G17P_SKSM_QUEUE_ENABLE_MASK_BASE, 0x21068); + assert_eq!(G17P_SKSM_QUEUE_ENTRY_ADDRESS_MASK, 0x0000_07ff_ffff_ffe0); + + let config = prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0x40, + 1, + 0, + 0x12340, + g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch, + ) + .unwrap(); + + // configureHardware: base + 9*stride, tag, payload, then QID mask. + assert_eq!(config.tag_offset, 0x4090); + assert_eq!(config.tag, 0x0002_0400_0002_1008); + assert_eq!(config.payload_offset, 0x4098); + // This hard-coded value catches both the 32-bit selector arithmetic + // and the host's ADD of `(data_master + 1) << 60` before the OR. + assert_eq!(config.payload, 0x2280_0000_0001_2340); + assert_eq!( + config.write_pair(), + G17SksmOrderedWritePair { + word0_offset: 0x4090, + word0: 0x0002_0400_0002_1008, + word1_offset: 0x4098, + word1: 0x2280_0000_0001_2340, + } + ); + assert_eq!( + config.enable_pair, + G17SksmOrderedWritePair { + word0_offset: 0x21070, + word0: 1, + word1_offset: 0x21068, + word1: 0, + } + ); + assert_eq!( + config.disable_pair, + G17SksmOrderedWritePair { + word0_offset: 0x21080, + word0: 1, + word1_offset: 0x21078, + word1: 0, + } + ); + } + + #[test] + fn g17p_compute_queue_matches_the_captured_native_cl2_tuple() { + assert_eq!(G17P_TA_DATA_MASTER, 0); + assert_eq!(G17P_3D_DATA_MASTER, 1); + assert_eq!(G17P_COMPUTE_DATA_MASTER, 2); + + let config = prepare_g17p_compute_sksm_queue_config( + G17PComputeQueueRouting { + queue_id: 4, + completion_selector: 2, + entry_gpu_va: 0x0000_0070_004d_8000, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + }, + g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch, + ) + .unwrap(); + + assert_eq!(config.tag_offset, 0x4090); + assert_eq!(config.tag, 0x0000_2400_0002_1008); + assert_eq!(config.payload_offset, 0x4098); + assert_eq!(config.payload, 0x3100_0070_004d_8000); + assert_eq!( + config.enable_pair, + G17SksmOrderedWritePair { + word0_offset: 0x21070, + word0: 0, + word1_offset: 0x21068, + word1: 1 << 4, + } + ); + assert_eq!( + config.disable_pair, + G17SksmOrderedWritePair { + word0_offset: 0x21080, + word0: 0, + word1_offset: 0x21078, + word1: 1 << 4, + } + ); + } + + #[test] + fn g17p_sksm_queue_configuration_is_identity_and_range_scoped() { + let geometry = g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch; + + for target in [ + G17ChannelAbiTarget::M5G17G, + G17ChannelAbiTarget::M5ProMaxG17X, + ] { + assert_eq!( + prepare_g17p_sksm_queue_config(target, 0, 0, 0, 0x20, geometry), + Err(SubmissionError::ChannelAbiUnconfirmed) + ); + } + assert_eq!( + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0x80, + 0, + 0, + 0x20, + geometry, + ), + Err(SubmissionError::QueueIdOutOfRange) + ); + assert_eq!( + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0, + 3, + 0, + 0x20, + geometry, + ), + Err(SubmissionError::KickQueueDataMasterOutOfRange) + ); + assert_eq!( + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0, + 0, + 5, + 0x20, + geometry, + ), + Err(SubmissionError::KickQueueCompletionSelectorOutOfRange) + ); + for address in [0x21, 0x0000_0800_0000_0000] { + assert_eq!( + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0, + 0, + 0, + address, + geometry, + ), + Err(SubmissionError::KickQueueEntryAddressUnencodable) + ); + } + assert_eq!( + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0, + 0, + 0, + 0x20, + G17SksmScratchGeometry { + write0_base: u32::MAX, + write1_base: 0, + stride: 1, + }, + ), + Err(SubmissionError::KickMmioOffsetOverflow) + ); + } + + #[test] + fn g17p_completion_descriptors_match_all_four_host_slots() { + assert_eq!(G17P_KSM_COMPLETION_QUEUE_COUNT, 4); + assert_eq!(G17P_KSM_COMPLETION_DESCRIPTOR_SIZE, 0x20); + assert_eq!(G17P_KSM_COMPLETION_ENTRY_SIZE, 0x40); + assert_eq!(G17P_KSM_COMPLETION_CAPACITY, 0x800); + assert_eq!(G17P_KSM_COMPLETION_BACKING_SIZE, 0x20000); + assert_eq!(G17P_KSM_COMPLETION_PRESENT_ORDINALS, [0, 2]); + assert_eq!(G17P_NORMAL_CL_COMPLETION_ORDINAL, 0); + assert_eq!( + G17P_KSM_COMPLETION_DESCRIPTOR_LAYOUT, + G17PKsmCompletionDescriptorLayout { + gpu_va: 0x00, + cpu_va: 0x08, + capacity: 0x10, + entry_size: 0x14, + ordinal: 0x18, + processed: 0x1c, + } + ); + + for ordinal in 0..G17P_KSM_COMPLETION_QUEUE_COUNT { + let gpu_va = 0x1234_0000 + ordinal as u64 * 0x1000; + let cpu_va = 0xabcd_0000 + ordinal as u64 * 0x2000; + let capacity = 0x80 + ordinal as u32; + let raw = encode_g17p_ksm_completion_descriptor( + G17ChannelAbiTarget::A18ProG17P, + ordinal, + gpu_va, + cpu_va, + capacity, + ) + .unwrap(); + + assert_eq!(&raw[0x00..0x08], &gpu_va.to_le_bytes()); + assert_eq!(&raw[0x08..0x10], &cpu_va.to_le_bytes()); + assert_eq!(&raw[0x10..0x14], &capacity.to_le_bytes()); + assert_eq!(&raw[0x14..0x18], &0x40u32.to_le_bytes()); + assert_eq!(&raw[0x18..0x1c], &(ordinal as u32).to_le_bytes()); + assert_eq!(&raw[0x1c..0x20], &[0; 4]); + } + assert!(G17P_COMPLETION_HOST_ABI_GROUNDED); + } + + #[test] + fn g17p_completion_descriptors_report_invalid_inputs() { + assert_eq!( + encode_g17p_ksm_completion_descriptor( + G17ChannelAbiTarget::M5G17G, + 0, + 0x1000, + 0x2000, + 1, + ), + Err(SubmissionError::ChannelAbiUnconfirmed) + ); + assert_eq!( + encode_g17p_ksm_completion_descriptor( + G17ChannelAbiTarget::A18ProG17P, + 4, + 0x1000, + 0x2000, + 1, + ), + Err(SubmissionError::CompletionQueueOrdinalOutOfRange) + ); + assert_eq!( + encode_g17p_ksm_completion_descriptor( + G17ChannelAbiTarget::A18ProG17P, + 0, + 0x1000, + 0x2000, + 0, + ), + Err(SubmissionError::CompletionQueueCapacityZero) + ); + } + + #[test] + fn kick_record_table_uses_the_firmware_geometry() { + // add x2,x22,#0x10 ; add x2,x2,QID,lsl#5 -> base 0x10, stride 0x20. + assert_eq!(KICK_RECORD_TABLE_BASE, 0x10); + assert_eq!(KICK_RECORD_STRIDE, 0x20); + assert_eq!(KICK_RECORD_COUNT, 128); + assert_eq!(kick_record_offset(0), Ok(0x10)); + assert_eq!(kick_record_offset(1), Ok(0x30)); + assert_eq!(kick_record_offset(0x7f), Ok(0x0ff0)); + assert_eq!( + kick_record_offset(0x80), + Err(SubmissionError::QueueIdOutOfRange) + ); + // The whole 128-entry table fits in 0x10 + 128*0x20 = 0x1010 bytes. + assert_eq!( + KICK_RECORD_TABLE_BASE + KICK_RECORD_COUNT * KICK_RECORD_STRIDE, + 0x1010 + ); + } + + #[test] + fn kick_record_internal_layout_is_exact() { + // Written across 0x10cb0-0x10d2c. + assert_eq!( + KICK_RECORD_LAYOUT, + KickRecordLayout { + ts_first: 0x00, + ts_last: 0x08, + slot_mask: 0x10, + valid_flag: 0x18, + slot_index: 0x1c, + } + ); + // Every field lies inside one 0x20-byte record. + for off in [ + KICK_RECORD_LAYOUT.ts_first, + KICK_RECORD_LAYOUT.ts_last, + KICK_RECORD_LAYOUT.slot_mask, + KICK_RECORD_LAYOUT.valid_flag, + KICK_RECORD_LAYOUT.slot_index, + ] { + assert!(off < KICK_RECORD_STRIDE); + } + } + + #[test] + fn g17p_top_level_command_names_match_the_host_abi() { + let known = [ + (G17PTopLevelCommandType::Ta, 0), + (G17PTopLevelCommandType::ThreeD, 1), + (G17PTopLevelCommandType::FastBlit, 2), + (G17PTopLevelCommandType::Cl, 3), + (G17PTopLevelCommandType::Barrier, 4), + (G17PTopLevelCommandType::PmGrow, 5), + (G17PTopLevelCommandType::RemoteNode, 7), + (G17PTopLevelCommandType::SharedEventSignal, 9), + (G17PTopLevelCommandType::AddFinalFrgGtpcKick, 12), + (G17PTopLevelCommandType::KsmKickQueueAddKicks, 14), + (G17PTopLevelCommandType::ConfigUpdate, 15), + (G17PTopLevelCommandType::EntrySignal, 16), + (G17PTopLevelCommandType::Nop, 17), + (G17PTopLevelCommandType::LateEvalEventSignal, 18), + ]; + for (command_type, raw) in known { + assert_eq!(command_type as u32, raw); + } + + assert_eq!(G17P_TOP_LEVEL_COMMAND_TAG_OFFSET, 0); + assert_eq!(G17P_TOP_LEVEL_COMMAND_TAG_WIDTH, 4); + assert_eq!(G17P_CHANNEL_COMMAND_TYPE_MAX, 18); + assert_eq!(G17P_TOP_LEVEL_COMMAND_KNOWN_MASK, 0x0007_d2bf); + assert_eq!(G17P_TOP_LEVEL_COMMAND_UNRESOLVED_MASK, 0x0000_0d40); + assert_eq!(G17P_TOP_LEVEL_COMMAND_INVALID_MASK, 0x0000_2000); + assert_eq!( + G17P_TOP_LEVEL_COMMAND_KNOWN_MASK + | G17P_TOP_LEVEL_COMMAND_UNRESOLVED_MASK + | G17P_TOP_LEVEL_COMMAND_INVALID_MASK, + (1 << (G17P_CHANNEL_COMMAND_TYPE_MAX + 1)) - 1 + ); + } + + #[test] + fn g17p_top_level_command_decoder_reads_exact_little_endian_bytes() { + let known = [ + G17PTopLevelCommandType::Ta, + G17PTopLevelCommandType::ThreeD, + G17PTopLevelCommandType::FastBlit, + G17PTopLevelCommandType::Cl, + G17PTopLevelCommandType::Barrier, + G17PTopLevelCommandType::PmGrow, + G17PTopLevelCommandType::RemoteNode, + G17PTopLevelCommandType::SharedEventSignal, + G17PTopLevelCommandType::AddFinalFrgGtpcKick, + G17PTopLevelCommandType::KsmKickQueueAddKicks, + G17PTopLevelCommandType::ConfigUpdate, + G17PTopLevelCommandType::EntrySignal, + G17PTopLevelCommandType::Nop, + G17PTopLevelCommandType::LateEvalEventSignal, + ]; + for command_type in known { + let mut command = [0xa5; 9]; + command[..4].copy_from_slice(&(command_type as u32).to_le_bytes()); + assert_eq!( + decode_g17p_top_level_command(G17ChannelAbiTarget::A18ProG17P, &command), + Ok(command_type) + ); + } + + for length in 0..4 { + assert_eq!( + decode_g17p_top_level_command( + G17ChannelAbiTarget::A18ProG17P, + &[0, 0, 0, 0][..length] + ), + Err(SubmissionError::CommandHeaderTooShort) + ); + } + + // The dispatcher reads all four little-endian bytes at offset zero; + // it neither truncates to byte zero nor scans for a nested opcode. + assert_eq!( + decode_g17p_top_level_command( + G17ChannelAbiTarget::A18ProG17P, + &[0, 0, 0, 1, 18, 0, 0, 0] + ), + Err(SubmissionError::CommandTagInvalid) + ); + } + + #[test] + fn g17p_top_level_command_decode_reports_bad_tag_and_identity() { + for raw in [6u32, 8, 10, 11] { + assert_eq!( + decode_g17p_top_level_command(G17ChannelAbiTarget::A18ProG17P, &raw.to_le_bytes()), + Err(SubmissionError::CommandTagUnresolved) + ); + } + for raw in [13u32, 19, u32::MAX] { + assert_eq!( + decode_g17p_top_level_command(G17ChannelAbiTarget::A18ProG17P, &raw.to_le_bytes()), + Err(SubmissionError::CommandTagInvalid) + ); + } + for target in [ + G17ChannelAbiTarget::M5G17G, + G17ChannelAbiTarget::M5ProMaxG17X, + ] { + assert_eq!( + decode_g17p_top_level_command(target, &0u32.to_le_bytes()), + Err(SubmissionError::CommandAbiUnconfirmed) + ); + assert_eq!( + classify_g17p_channel_command(target, 0), + Err(SubmissionError::CommandAbiUnconfirmed) + ); + } + } + + #[test] + fn g17p_command_dispatch_matches_the_firmware_jump_table() { + assert_eq!(G17P_CHANNEL_COMMAND_PROCESS_MASK, 0x0005_d2bf); + assert_eq!(G17P_CHANNEL_COMMAND_PM_REBUILD_MASK, 0x0000_0040); + assert_eq!(G17P_CHANNEL_COMMAND_INVALID_MASK, 0x0002_2d00); + assert_eq!( + G17P_CHANNEL_COMMAND_PROCESS_MASK + | G17P_CHANNEL_COMMAND_PM_REBUILD_MASK + | G17P_CHANNEL_COMMAND_INVALID_MASK, + (1 << (G17P_CHANNEL_COMMAND_TYPE_MAX + 1)) - 1 + ); + + for raw in 0..=G17P_CHANNEL_COMMAND_TYPE_MAX { + let expected = match raw { + 0 | 1 | 2 | 3 | 4 | 5 | 7 | 9 | 12 | 14 | 15 | 16 | 18 => { + G17PChannelCommandDisposition::Process + } + 6 => G17PChannelCommandDisposition::PmRebuildForbiddenFatal, + 8 | 10 | 11 | 13 | 17 => G17PChannelCommandDisposition::InvalidCommandFatal, + _ => unreachable!(), + }; + assert_eq!( + classify_g17p_channel_command(G17ChannelAbiTarget::A18ProG17P, raw), + Ok(expected) + ); + } + for raw in [19, u32::MAX] { + assert_eq!( + classify_g17p_channel_command(G17ChannelAbiTarget::A18ProG17P, raw), + Ok(G17PChannelCommandDisposition::InvalidCommandFatal) + ); + } + + // Host ABI identity and matched-firmware admission intentionally split. + let nop = (G17PTopLevelCommandType::Nop as u32).to_le_bytes(); + assert_eq!( + decode_g17p_top_level_command(G17ChannelAbiTarget::A18ProG17P, &nop), + Ok(G17PTopLevelCommandType::Nop) + ); + assert_eq!( + admit_g17p_top_level_command(G17ChannelAbiTarget::A18ProG17P, &nop), + Err(SubmissionError::CommandTagInvalidForFirmware) + ); + + for command_type in [ + G17PTopLevelCommandType::Ta, + G17PTopLevelCommandType::ThreeD, + G17PTopLevelCommandType::FastBlit, + G17PTopLevelCommandType::Cl, + G17PTopLevelCommandType::Barrier, + G17PTopLevelCommandType::PmGrow, + G17PTopLevelCommandType::RemoteNode, + G17PTopLevelCommandType::SharedEventSignal, + G17PTopLevelCommandType::AddFinalFrgGtpcKick, + G17PTopLevelCommandType::KsmKickQueueAddKicks, + G17PTopLevelCommandType::ConfigUpdate, + G17PTopLevelCommandType::EntrySignal, + G17PTopLevelCommandType::LateEvalEventSignal, + ] { + assert_eq!( + admit_g17p_top_level_command( + G17ChannelAbiTarget::A18ProG17P, + &(command_type as u32).to_le_bytes() + ), + Ok(command_type) + ); + } + } + + #[test] + fn g17p_command_prefilter_matches_literal_mask() { + assert_eq!(G17P_CHANNEL_COMMAND_PREFILTER_MASK, 0x0001_d03f); + for raw in 0..=20 { + let expected = matches!(raw, 0 | 1 | 2 | 3 | 4 | 5 | 12 | 14 | 15 | 16); + assert_eq!( + g17p_channel_command_prefilter_allows(G17ChannelAbiTarget::A18ProG17P, raw), + Ok(expected) + ); + } + assert_eq!( + g17p_channel_command_prefilter_allows(G17ChannelAbiTarget::M5G17G, 0), + Err(SubmissionError::CommandAbiUnconfirmed) + ); + } + + #[test] + fn provenance_ledger_is_explicit_about_reuse_and_fork() { + assert_eq!( + SUBMISSION_ABI_LEDGER.tx_doorbell, + AbiProvenance::IdenticalToAgx2 + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.kick_word, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.kick_queue_producer, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.host_add_kicks_mmio, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.g17p_queue_config, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.g17p_completion_descriptors, + AbiProvenance::ForkedAgx3Grounded + ); + assert!(HOST_SKSM_ADD_KICKS_PATH_SELECTED); + assert_eq!( + SUBMISSION_ABI_LEDGER.kick_record, + AbiProvenance::ForkedAgx3Grounded + ); + // The channel/ring state records stay inherited and unsupported. The + // widened native descriptor geometry is grounded first-hand. + assert_eq!( + SUBMISSION_ABI_LEDGER.channel_state, + AbiProvenance::Agx2InheritedUnconfirmed + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.g17p_channel_state, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.ring_state, + AbiProvenance::Agx2InheritedUnconfirmed + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.channel_descriptor, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.host_channel_state_reset, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.host_uncached_channel_reset, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.g17p_command_types, + AbiProvenance::ForkedAgx3Grounded + ); + assert!(CHANNEL_DESCRIPTOR_STRIDE_CONFIRMED); + } + + #[test] + fn transport_tracks_classic_graphics_and_native_sksm_independently() { + assert!(GROUNDED_SUBMISSION_PRODUCER_AVAILABLE); + assert!(!EXECUTABLE_SUBMISSION_TRANSPORT_AVAILABLE); + assert!(!SUBMISSION_LIVE_GAPS.all_proven()); + assert!(!SUBMISSION_LIVE_GAPS.classic_graphics_proven()); + assert!(!SUBMISSION_LIVE_GAPS.native_sksm_proven()); + assert!(SUBMISSION_LIVE_GAPS.sksm_resource0_ordered_write_implemented); + assert!(SUBMISSION_LIVE_GAPS.channel_doorbell_round_trip_proven); + assert!(!SUBMISSION_LIVE_GAPS.partial_graphics_runtime_implemented); + assert!(SUBMISSION_LIVE_GAPS.partial_graphics_round_trip_proven); + assert!(!SUBMISSION_LIVE_GAPS.sksm_add_kicks_round_trip_proven); + assert!(!SUBMISSION_LIVE_GAPS.completion_irq_route_proven); + assert!(SUBMISSION_LIVE_GAPS.normal_cl_completion_consumer_implemented); + + let classic = SubmissionLiveGaps { + sksm_resource0_ordered_write_implemented: true, + channel_doorbell_round_trip_proven: true, + partial_graphics_runtime_implemented: true, + partial_graphics_round_trip_proven: true, + sksm_add_kicks_round_trip_proven: false, + completion_irq_route_proven: true, + normal_cl_completion_consumer_implemented: true, + }; + assert!(classic.classic_graphics_proven()); + assert!(!classic.native_sksm_proven()); + assert!(classic.all_proven()); + assert!(!classic.sksm_add_kicks_round_trip_proven); + + let native = SubmissionLiveGaps { + sksm_resource0_ordered_write_implemented: true, + channel_doorbell_round_trip_proven: false, + partial_graphics_runtime_implemented: false, + partial_graphics_round_trip_proven: false, + sksm_add_kicks_round_trip_proven: true, + completion_irq_route_proven: true, + normal_cl_completion_consumer_implemented: true, + }; + assert!(!native.classic_graphics_proven()); + assert!(native.native_sksm_proven()); + assert!(native.all_proven()); + assert!(!SubmissionLiveGaps { + normal_cl_completion_consumer_implemented: false, + ..native + } + .native_sksm_proven()); + } +} + +// --------------------------------------------------------------------------- +// Parameter-buffer descriptor handshake +// +// Metal performs this around every render submit and this driver has never +// performed it at all. Decoded from the 25G83 host kernelcache, which carries +// both halves symbolised: +// +// AGXAcceleratorG16::halLoadPBDesc @ 0xfffffe0008a52890 before submit +// AGXAcceleratorG16::halStorePBDesc @ 0xfffffe0008a527f0 on completion +// +// They are exact inverses, which is what makes the field map trustworthy: the +// store reads back precisely the four quantities the load wrote. Confirmed on +// hardware -- a real Metal render leaves the ring state {7, 7, 0, 0}. +// +// The ring lives at `[[accelerator + 0x11358] + 0x60]` with 16-byte entries +// indexed by the `uint` third argument, and `halStorePBDesc` runs on +// `kernel_task`, i.e. from the completion path rather than the submitting +// thread. +// --------------------------------------------------------------------------- + +/// One parameter-buffer descriptor ring entry. +pub(crate) const G17P_PBDESC_ENTRY_SIZE: usize = 16; + +/// The four quantities the handshake exchanges: two 22-bit counters and two +/// flags. This is `AGXSPBDescRingState`. +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct G17PPbDescRingState { + /// Descriptor dword2, low 22 bits. + pub(crate) counter_a: u32, + /// Descriptor dword3, low 22 bits. + pub(crate) counter_b: u32, + /// Descriptor dword3 bit 31. + pub(crate) flag_high: u32, + /// Descriptor dword0 bit 0. + pub(crate) flag_low: u32, +} + +/// Encode the high segment of a parameter-buffer address. +/// +/// `PBState[+0x28]` contains the full tagged address: native passes +/// `0x1000190000`. Hardware adds an implicit `0x1000000000` bias when it +/// rebuilds the address from this segment and dword0's low 36 bits. Bit 42 +/// selects between the two kext encoding forms. +pub(crate) const fn g17p_pbdesc_segment(pb_address: u64) -> u64 { + let base: u64 = if pb_address & 0x400_0000_0000 == 0 { + 0x70_0000_0000 + } else { + 0 + }; + let high = (pb_address >> 3) & 0x80_0000_0000; + ((base.wrapping_add(pb_address)) & 0x70_0000_0000) | high +} + +/// Address reconstructed by the first G17P PB page-list read. This models the +/// hardware bias witnessed by the exact `0x8002478000` fault when the host +/// incorrectly encoded stripped address `0x2478000`. +pub(crate) const fn g17p_pbdesc_reconstructed_address(pb_address: u64) -> u64 { + (g17p_pbdesc_segment(pb_address).wrapping_add(0x10_0000_0000)) + | (pb_address & 0x0f_ffff_ffff) +} + +/// `halLoadPBDesc`: fold the ring state and the parameter-buffer address into +/// the descriptor, preserving every bit the firmware owns. +pub(crate) const fn encode_g17p_pbdesc( + previous: [u32; 4], + pb_address: u64, + pb_state_word_34: u32, + state: G17PPbDescRingState, +) -> [u32; 4] { + let segment = g17p_pbdesc_segment(pb_address); + + // dword0: address >> 4, low 3 bits kept, then bit 0 = flag_low. + let mut dword0 = ((pb_address >> 4) as u32 & !0x7) | (previous[0] & 0x7); + dword0 = (dword0 & !1) | if state.flag_low != 0 { 1 } else { 0 }; + + // dword1: segment >> 8 in the top nibble, low 28 kept, then low 22 from + // PBState[+0x34]. + let mut dword1 = ((segment >> 8) as u32 & !0x0fff_ffff) | (previous[1] & 0x0fff_ffff); + dword1 = (dword1 & !0x003f_ffff) | (pb_state_word_34 & 0x003f_ffff); + + // dword2/dword3: the two 22-bit counters, plus bit 31. + let dword2 = (previous[2] & !0x003f_ffff) | (state.counter_a & 0x003f_ffff); + let mut dword3 = (previous[3] & !0x003f_ffff) | (state.counter_b & 0x003f_ffff); + dword3 = (dword3 & !0x8000_0000) | ((state.flag_high & 1) << 31); + + [dword0, dword1, dword2, dword3] +} + +/// `halStorePBDesc`: read the ring state back out. Exact inverse of the encode. +pub(crate) const fn decode_g17p_pbdesc(descriptor: [u32; 4]) -> G17PPbDescRingState { + G17PPbDescRingState { + counter_a: descriptor[2] & 0x003f_ffff, + counter_b: descriptor[3] & 0x003f_ffff, + flag_high: descriptor[3] >> 31, + flag_low: descriptor[0] & 1, + } +} + +/// Byte offset of one descriptor within the ring. +pub(crate) const fn g17p_pbdesc_offset(index: u32) -> usize { + index as usize * G17P_PBDESC_ENTRY_SIZE +} + +// --------------------------------------------------------------------------- +// UMA page-pool descriptor handshake +// +// `AGXAcceleratorG16::halLoadUMADesc` publishes one 0x20-byte descriptor at +// `[[accelerator + 0x11360] + 0x60] + HardwareBufferID * 0x20` before the +// command descriptors become visible. `halStoreUMADesc` is the inverse on +// completion. The fourth qword belongs to the firmware and is not touched by +// either load or store. +// --------------------------------------------------------------------------- + +pub(crate) const G17P_UMA_DESC_ENTRY_SIZE: usize = 0x20; +pub(crate) const G17P_UMA_DESC_ENTRY_COUNT: u32 = 0x100; + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct G17PUmaPagePoolState { + pub(crate) page_list: u64, + pub(crate) unit_pages: u32, + pub(crate) counter_a: u32, + pub(crate) counter_b: u32, + pub(crate) empty: u32, + pub(crate) total_pages: u32, +} + +/// Exact `halLoadUMADesc` packing. The source fields live at unaligned +/// offsets `+0x14/+0x1c/+0x20/+0x24/+0x28/+0x2c` in +/// `_AGFIUMAPagePoolState`. +pub(crate) const fn encode_g17p_umadesc( + previous: [u64; 4], + state: G17PUmaPagePoolState, +) -> [u64; 4] { + [ + ((state.unit_pages as u64) << 41) | (state.page_list >> 7), + ((state.counter_b as u64) << 33) + | ((state.counter_a as u64) << 5) + | ((state.empty as u64) << 61), + state.total_pages as u64, + previous[3], + ] +} + +pub(crate) const fn g17p_umadesc_offset(hardware_buffer_id: u32) -> Option { + if hardware_buffer_id >= G17P_UMA_DESC_ENTRY_COUNT { + return None; + } + Some(hardware_buffer_id as usize * G17P_UMA_DESC_ENTRY_SIZE) +} + +/// Recover the mutable pool state before its hardware slot changes owners. +/// The address and allocation unit belong to the retained pool; word three +/// is firmware-owned and is not part of the load/store handshake. +pub(crate) const fn decode_g17p_umadesc( + descriptor: [u64; 4], + mut state: G17PUmaPagePoolState, +) -> G17PUmaPagePoolState { + state.counter_a = ((descriptor[1] >> 5) & 0x0fff_ffff) as u32; + state.counter_b = ((descriptor[1] >> 33) & 0x0fff_ffff) as u32; + state.empty = (descriptor[1] >> 61) as u32; + state.total_pages = descriptor[2] as u32; + state +} + +#[cfg(test)] +mod uma_descriptor_tests { + use super::*; + + #[test] + fn first_render_uma_descriptor_matches_the_native_packer() { + let encoded = encode_g17p_umadesc( + [0, 0, 0, 0xfeed_face_dead_beef], + G17PUmaPagePoolState { + page_list: 0x70_0000_0000, + unit_pages: 0x4_0000, + counter_a: 0, + counter_b: 0x1600, + empty: 0, + total_pages: 0x1600, + }, + ); + assert_eq!( + encoded, + [ + 0x0800_0000_e000_0000, + 0x0000_2c00_0000_0000, + 0x0000_0000_0000_1600, + 0xfeed_face_dead_beef, + ] + ); + assert_eq!(g17p_umadesc_offset(0), Some(0)); + assert_eq!(g17p_umadesc_offset(0xff), Some(0x1fe0)); + assert_eq!(g17p_umadesc_offset(0x100), None); + } +} + +/// Opcode of the USC private-memory free-list device-control record. +/// +/// MISIDENTIFIED ONCE, corrected 2026-08-31. This was briefly taken for the +/// parameter-buffer descriptor-ring publication because it is the only +/// device-control record in a whole process lifetime that carries low VAs, and +/// it appears once per queue right after `allocParameterManagement`. That is +/// adjacency, not provenance. The record is built by +/// `AGXUSCPrivMemFList::updateSubmitInfo(AGXUMADataRec*, int)`, which belongs +/// to the USC/UMA page-pool family (`halLoadUMADesc`/`halStoreUMADesc`), not +/// the parameter-buffer family (`halLoadPBDesc`/`halStorePBDesc`). +/// +/// The real owner of the PBDesc ring is `AGXAccelerator::allocPBDescriptor`, +/// which is the ONLY site in the image that stores to `accelerator + 0x11358` +/// -- the pointer `halLoadPBDesc` dereferences. It allocates the object; it +/// publishes no device-control record, so the ring reaches firmware by some +/// other route we have not yet found. +pub(crate) const G17P_DEVCTL_USC_FREELIST: u32 = 0x20; + +/// Span of the USC free-list ring, as published by the opcode-0x20 record. +/// +/// Stable across four independent captures, so it is a fixed address rather +/// than an allocation. It is NOT the parameter-buffer descriptor ring -- see +/// `G17P_DEVCTL_USC_FREELIST`. Kept because if the USC free list is a third +/// thing this driver does not maintain, this is where it lives. +pub(crate) const G17P_USC_FREELIST_LOW_VA: u64 = 0x70_0020_8000; +pub(crate) const G17P_USC_FREELIST_LOW_END: u64 = 0x70_0020_8440; +pub(crate) const G17P_USC_FREELIST_GROW_DESCRIPTOR_BYTES: u32 = 0x28; + +/// The three fields that move between boots. 256x256 and 1024x1024 gave identical values for `word_04`, and +/// `word_0c` moved by 2 between them, so none of these encode geometry -- they +/// read as per-submission or per-boot sequence counters. +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct G17PUscFreelistSequence { + /// +0x04. Observed 1, 1, 6. + pub(crate) word_04: u32, + /// +0x0c. Observed 0x8df, 0x8e1, 0x65e. + pub(crate) word_0c: u32, + /// +0x30. Observed 2, 1, 3. + pub(crate) word_30: u32, +} + +/// Build the opcode-0x20 (USC free-list) device-control record. +/// +/// Only the first 0x40 bytes of an `AGFIAcceleratorDeviceControlEntry` ever +/// reach firmware: `AGXFirmware::submitDeviceControl` copies exactly four +/// 16-byte SIMD pairs (`ldp q0,q1` / `ldp q2,q3` then `stp`) into a ring whose +/// stride is `1 << 6` and whose bound is `cmp w8, #0x100`. Our +/// `CONTROL_RECORD_SIZE` (0x40) and `CONTROL_RING_ENTRIES` (0x100) already +/// match, so the fields the capture shows at +0x40/+0x44/+0x4c are kext-side +/// bookkeeping that never crosses the interface. +pub(crate) fn encode_g17p_usc_freelist_control( + firmware_half_va: u64, + ring_low_va: u64, + ring_low_end: u64, + sequence: G17PUscFreelistSequence, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != 0x40 { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + if ring_low_end <= ring_low_va { + return Err(SubmissionError::PartialOpeningAddressOverflow); + } + out.fill(0); + put_u32_le(out, 0x00, G17P_DEVCTL_USC_FREELIST); + put_u32_le(out, 0x04, sequence.word_04); + put_u32_le(out, 0x08, 0x3f); // constant across all three captures + put_u32_le(out, 0x0c, sequence.word_0c); + put_u32_le(out, 0x10, 0); + put_u64_le(out, 0x14, firmware_half_va); + put_u64_le(out, 0x1c, ring_low_va); + put_u64_le(out, 0x24, ring_low_end); + put_u32_le(out, 0x2c, G17P_USC_FREELIST_GROW_DESCRIPTOR_BYTES); + put_u32_le(out, 0x30, sequence.word_30); + put_u32_le(out, 0x34, 1); // constant + Ok(()) +} + +#[cfg(test)] +mod usc_freelist_tests { + use super::*; + + fn u32_at(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) + } + + fn u64_at(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) + } + + #[test] + fn linux_owned_first_render_usc_freelist_record_is_exact() { + let mut record = [0xa5; 0x40]; + encode_g17p_usc_freelist_control( + 0xffff_fc20_c082_8000, + G17P_USC_FREELIST_LOW_VA, + G17P_USC_FREELIST_LOW_END, + G17PUscFreelistSequence { + word_04: 1, + word_0c: 0, + word_30: 0, + }, + &mut record, + ) + .unwrap(); + assert_eq!(u32_at(&record, 0x00), G17P_DEVCTL_USC_FREELIST); + assert_eq!(u32_at(&record, 0x04), 1); + assert_eq!(u32_at(&record, 0x08), 0x3f); + assert_eq!(u32_at(&record, 0x0c), 0); + assert_eq!(u64_at(&record, 0x14), 0xffff_fc20_c082_8000); + assert_eq!(u64_at(&record, 0x1c), G17P_USC_FREELIST_LOW_VA); + assert_eq!(u64_at(&record, 0x24), G17P_USC_FREELIST_LOW_END); + assert_eq!( + u32_at(&record, 0x2c), + G17P_USC_FREELIST_GROW_DESCRIPTOR_BYTES + ); + assert_eq!(u32_at(&record, 0x30), 0); + assert_eq!(u32_at(&record, 0x34), 1); + assert!(record[0x38..].iter().all(|byte| *byte == 0)); + } +} + + +#[cfg(test)] +mod exact_prior_tests { + use super::*; + + #[test] + fn absent_prior_preserves_every_legacy_header_form() { + for base in [0, 1, 100, u64::MAX - 10] { + for preceding in 0..5 { + for barrier in [u16::MAX, 0, 1, 3, 5] { + assert_eq!(g17p_render_order_with_prior(base, preceding, barrier, 0), + g17p_render_order(base, preceding, barrier)); + } + } + } + } + + #[test] + fn exact_old_prefix_does_not_grow_with_newer_unrelated_commands() { + for base in [7, 8, 1000] { + for preceding in [0, 1, 17] { + let order = g17p_render_order_with_prior(base, preceding, u16::MAX, 7).unwrap(); + assert_eq!(order.wait_through, Some(7)); + assert_eq!(order.sequence, base + preceding + 1); + assert!(g17p_render_prefix_contains(order.wait_through, 7)); + assert!(!g17p_render_prefix_contains(order.wait_through, 8)); + } + } + } + + #[test] + fn historical_max_preserves_zero_and_positive_relative_prefixes() { + assert_eq!(g17p_render_order_with_prior(100, 4, 0, 7).unwrap().wait_through, Some(100)); + assert_eq!(g17p_render_order_with_prior(100, 4, 3, 100).unwrap().wait_through, Some(103)); + assert_eq!(g17p_render_order_with_prior(100, 4, u16::MAX, 100).unwrap().wait_through, Some(100)); + assert_eq!(g17p_render_order_with_prior(0, 0, u16::MAX, 0).unwrap().wait_through, None); + } + + #[test] + fn earlier_current_ioctl_commands_are_not_historical() { + for prior in [11, 12, 100, u64::MAX] { + assert_eq!(g17p_render_order_with_prior(10, 100, u16::MAX, prior), + Err(SubmissionError::PriorBarrierFuture)); + } + assert_eq!(g17p_render_order_with_prior(0, 1, u16::MAX, 1), + Err(SubmissionError::PriorBarrierFuture)); + } + + #[test] + fn relative_future_and_counter_overflow_still_reject() { + assert_eq!(g17p_render_order_with_prior(10, 1, 2, 7), + Err(SubmissionError::RenderBarrierFuture)); + assert_eq!(g17p_render_order_with_prior(u64::MAX, 0, u16::MAX, 1), + Err(SubmissionError::RenderBarrierSequenceOverflow)); + assert_eq!(g17p_render_order_with_prior(u64::MAX - 1, 1, 1, 1), + Err(SubmissionError::RenderBarrierSequenceOverflow)); + } + + #[test] + fn split_ioctl_prefix_has_same_members_as_original_relative_prefix() { + let original = g17p_render_order_with_prior(20, 6, 3, 0).unwrap(); + let later_slice = g17p_render_order_with_prior(25, 1, u16::MAX, 23).unwrap(); + assert_eq!(original, later_slice); + for sequence in 0..30 { + assert_eq!(g17p_render_prefix_contains(original.wait_through, sequence), + g17p_render_prefix_contains(later_slice.wait_through, sequence)); + } + } + + #[test] + fn physical_siblings_remain_independent_prefix_members() { + let order = g17p_render_order_with_prior(50, 0, u16::MAX, 12).unwrap(); + let mut deps = [g17p_render_barrier_from_timestamp(0, 0).unwrap(); 4]; + let mut seqs = [0; 4]; + let mut count = 0; + for (sequence, qid, timestamp) in [(10, 2, 99), (11, 3, 55), (12, 2, 100)] { + assert!(g17p_render_prefix_contains(order.wait_through, sequence)); + let dep = g17p_render_barrier_from_timestamp(timestamp, qid).unwrap(); + g17p_insert_prefix_dependency(&mut deps, &mut seqs, &mut count, sequence, dep).unwrap(); + } + assert_eq!(count, 2); + assert_eq!(&seqs[..count], &[12, 11]); + assert_eq!((deps[0].queue_id, deps[1].queue_id), (2, 3)); + // Consuming one destination does not change another's exact prefix. + assert_eq!(g17p_render_order_with_prior(51, 0, u16::MAX, 12).unwrap().wait_through, + order.wait_through); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/uma.rs b/drivers/gpu/drm/asahi/g18/fw/uma.rs new file mode 100644 index 0000000000000..ca27f807db358 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/uma.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! T8152 / 26A428 HAL400 UMA hardware descriptor serialization. +//! +//! Native `halLoadUMADesc` at 0xfffffe0008c40980 writes sixteen u64s per +//! hardware slot (0x80 bytes). This differs from M5's four-word format. +//! The caller must supply the owned firmware pool state and the four +//! device configuration words, and validate all addresses and lifetimes. +//! Encoding parity does not establish a usable pool or execute GPU work. + +#![allow(dead_code)] + +pub(crate) const G18_UMA_STATE_BYTES: usize = 0x7c; +pub(crate) const G18_UMA_DESCRIPTOR_BYTES: usize = 0x80; +pub(crate) const G18_POOL_STATE_BYTES: usize = 0xa0; + +/// Resolved inputs to the full native FList firmware state. The directory +/// address includes its sparse page-end adjustment. Accessor/policy values +/// and resource ownership are the caller's responsibility. +pub(crate) struct G18UmaPoolState { + pub(crate) statistics_id: u64, + pub(crate) grow_engine: u32, + pub(crate) priority: u32, + pub(crate) directory_gpu: u64, + pub(crate) directory_stride: u64, + /// Original FList +0xac/+0xb0 order; native rev64 swaps the output lanes. + pub(crate) chunk_counters: [u32; 2], + pub(crate) block_list_gpu: u64, + pub(crate) populated_blocks: u32, + pub(crate) block_units_64k: u32, + pub(crate) sma_mode: u32, + pub(crate) preempt_counter_gpu: u64, + pub(crate) preempt_counter: u64, + pub(crate) compute: bool, + pub(crate) policy_value: u32, +} + +/// Native state initialization at 0xfffffe0008c8d404 and serialization at +/// 0xfffffe0008c8c004. This is the 0xa0-byte firmware state, not the 0x80-byte +/// hardware UMA descriptor. Zero padding matches a fresh native allocation. +pub(crate) fn encode_g18_pool_state(state: &G18UmaPoolState) -> [u8; G18_POOL_STATE_BYTES] { + let mut out = [0; G18_POOL_STATE_BYTES]; + for (offset, value) in [ + (0x00, state.statistics_id), + (0x14, state.directory_gpu), + (0x1c, state.directory_stride), + (0x4c, state.block_list_gpu), + (0x88, state.preempt_counter_gpu), + (0x90, state.preempt_counter), + ] { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [ + (0x08, u32::MAX), + (0x0c, state.grow_engine), + (0x10, state.priority), + (0x44, state.chunk_counters[1]), + (0x48, state.chunk_counters[0]), + (0x54, state.populated_blocks), + (0x58, state.block_units_64k), + (0x7c, state.sma_mode), + (0x98, u32::from(state.compute)), + (0x9c, state.policy_value), + ] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + out +} + +/// Dimensions written by the M6 pool's `computeAndSetUMAPoolVariables`. +pub(crate) struct G18UmaGeometry { + pub(crate) pool_bytes: u64, + pub(crate) block_bytes: u64, + pub(crate) block_count: u32, + pub(crate) directory_bytes: u64, + pub(crate) initial_committed_bytes: u64, + pub(crate) total_1k_chunks: u64, + pub(crate) directory_bytes_per_usc: u64, + pub(crate) chunks_per_usc: u32, +} + +/// Derive allocation geometry after resolving the device's native accessors. +/// +/// Native arithmetic uses 32-bit intermediates for chunk counts and directory +/// sizes. Reject overflow instead of wrapping an allocation. The caller must +/// supply the actual mode and topology; this helper does not select defaults. +pub(crate) fn g18_uma_geometry( + max_bytes: u64, + block_bytes: u64, + uscs: u32, + chunks64k: bool, + page_aligned: bool, + page_shift: u32, +) -> Option { + if uscs == 0 + || uscs > 128 + || !block_bytes.is_power_of_two() + || block_bytes < 0x10000 + || !(12..=16).contains(&page_shift) + { + return None; + } + let pool_bytes = max_bytes & !(block_bytes - 1); + if pool_bytes == 0 { + return None; + } + let units = u32::try_from(pool_bytes >> 16).ok()?; + let per_usc = units.checked_add(uscs - 1)? / uscs; + let raw_directory = per_usc.checked_mul(if chunks64k { 4 } else { 512 })?; + let align = if page_aligned { 1u64 << page_shift } else { 1024 }; + let directory_bytes_per_usc = (u64::from(raw_directory) + align - 1) & !(align - 1); + Some(G18UmaGeometry { + pool_bytes, + block_bytes, + block_count: u32::try_from(pool_bytes / block_bytes).ok()?, + directory_bytes: directory_bytes_per_usc.checked_mul(u64::from(uscs))?, + initial_committed_bytes: 0, + total_1k_chunks: u64::from(units.checked_mul(64)?), + directory_bytes_per_usc, + chunks_per_usc: (units / uscs).checked_mul(64)?, + }) +} + +/// Initial sparse directory layout for the native G18 pool flags value 3. +/// +/// The virtual extent is NOT a request for that much physical backing. Only +/// `initial_page_count` pages are committed initially, at `page_offset(i)`; +/// the owner must reserve the intervening unmapped addresses for later growth. +pub(crate) struct G18UmaSparseDirectory { + pub(crate) reservation_bytes: u64, + pub(crate) page_bytes: u64, + pub(crate) initial_page_count: u32, + pub(crate) gpu_base_offset: u64, + directory_bytes_per_usc: u64, +} + +impl G18UmaSparseDirectory { + pub(crate) fn page_offset(&self, index: u32) -> Option { + if index >= self.initial_page_count { + return None; + } + let offset = self.directory_bytes_per_usc.checked_mul(u64::from(index))?; + if offset.checked_add(self.page_bytes)? > self.reservation_bytes { + return None; + } + Some(offset) + } +} + +/// Native FList init reserves one extra page and backs one page per USC plus +/// the final page. `prepareBufferResources` exposes the directory beginning +/// one KiB before the end of the first page. Other pool flag modes are outside +/// this contract; the caller must first resolve the device's actual flags. +pub(crate) fn g18_uma_sparse_directory( + geometry: &G18UmaGeometry, + uscs: u32, + page_shift: u32, +) -> Option { + if uscs == 0 || uscs > 128 || !(12..=16).contains(&page_shift) { + return None; + } + let page_bytes = 1u64 << page_shift; + let stride = geometry.directory_bytes_per_usc; + if stride < page_bytes + || stride & (page_bytes - 1) != 0 + || stride.checked_mul(u64::from(uscs))? != geometry.directory_bytes + { + return None; + } + Some(G18UmaSparseDirectory { + reservation_bytes: geometry.directory_bytes.checked_add(page_bytes)?, + page_bytes, + initial_page_count: uscs + 1, + gpu_base_offset: page_bytes - 0x400, + directory_bytes_per_usc: stride, + }) +} + +/// Native `halInitSMAEnableMask` at 0xfffffe0008c41bdc splits the active +/// 128-bit mask into sixteen groups, each stored in a 16-bit output lane. +/// The primary mask is accelerator+0x470; +0x480 is used only when it is zero. +/// `group_width` is accelerator+0x4ec. This helper admits widths fitting the +/// output lanes; wider native profiles need a separate qualified contract. +pub(crate) fn pack_g18_sma_masks( + primary: [u64; 2], + fallback: [u64; 2], + group_width: u32, +) -> Option<[u64; 4]> { + if !(1..=16).contains(&group_width) { + return None; + } + let selected = if primary == [0; 2] { fallback } else { primary }; + let mut remaining = u128::from(selected[0]) | (u128::from(selected[1]) << 64); + let mask = (1u128 << group_width) - 1; + let mut words = [0; 4]; + for group in 0..16 { + words[group / 4] |= ((remaining & mask) as u64) << ((group % 4) * 16); + remaining >>= group_width; + } + Some(words) +} + +fn read64(state: &[u8; G18_UMA_STATE_BYTES], offset: usize) -> u64 { + let mut bytes = [0; 8]; + bytes.copy_from_slice(&state[offset..offset + 8]); + u64::from_le_bytes(bytes) +} + +fn read32(state: &[u8; G18_UMA_STATE_BYTES], offset: usize) -> u32 { + let mut bytes = [0; 4]; + bytes.copy_from_slice(&state[offset..offset + 4]); + u32::from_le_bytes(bytes) +} + +/// Pack the exact M6 hardware format. Preserve native masks and truncation; +/// do not substitute M5 directory geometry or infer opaque state values. +/// +/// `device_config` is native accelerator+0x11af8..0x11b10. These four +/// values are supplied by the caller; zero is not a qualified M6 default. +pub(crate) fn pack_g18_uma_descriptor( + state: &[u8; G18_UMA_STATE_BYTES], + device_config: [u64; 4], +) -> [u64; G18_UMA_DESCRIPTOR_BYTES / 8] { + [ + (read64(state, 0x1c) << 40 & 0xfffc_0000_0000_0000) | (read64(state, 0x14) >> 10), + read64(state, 0x24), + read64(state, 0x2c), + device_config[0], + device_config[1], + device_config[2], + device_config[3], + read64(state, 0x34) << 7, + read64(state, 0x3c) << 7, + read64(state, 0x44), + (u64::from(read32(state, 0x58)) << 44) | (read64(state, 0x4c) >> 6), + u64::from(read32(state, 0x54)), + read64(state, 0x5c), + read64(state, 0x64), + read64(state, 0x6c), + read64(state, 0x74), + ] +} + +/// Fixed first-dispatch policy, recovered with the original SMAV1 sizing code. +/// This is deliberately limited to the saved addition workload, not a general +/// replacement for native SMA sizing. Hardware execution remains to be tested. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18ComputeScratchPlan { + pub(crate) block_bytes: u64, + pub(crate) block_count: u32, + pub(crate) register_pair: [u64; 2], + pub(crate) chunk_counters: [u32; 2], +} + +pub(crate) fn g18_addition_scratch(state: &[u8]) -> Option { + const WORDS: [u32; 23] = [ + 32, 10, 0, 0, 192, 16, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 128, 4, 0, 0, + ]; + if state.len() != 0x268 { return None; } + for (index, expected) in WORDS.iter().enumerate() { + let offset = 0x198 + index * 4; + let value = u32::from_le_bytes(state[offset..offset + 4].try_into().ok()?); + if value != *expected { return None; } + } + // Twelve USCs, one compute job, engine/priority 2, flags 3, no preemption, + // 2 GiB maximum and 1 MiB blocks: ideal 5 MiB, minimum 3 MiB. + // Native allocatePagesEnd supplies d550/d558 and FList+0xac/+0xb0 below. + Some(G18ComputeScratchPlan { + block_bytes: 0x10_0000, block_count: 5, + register_pair: [16, 6], chunk_counters: [0, 350], + }) +} + +/// Native 8c8bfc8..8c8bfe0: preserve metadata while encoding a GPU address. +/// Reject truncation; current backing uses one entry per 1 MiB allocation. +pub(crate) fn g18_uma_block_entry(gpu_address: u64, previous: u32) -> Option { + if gpu_address == 0 || gpu_address >= (1u64 << 48) + || gpu_address & 0x0f_ffff != 0 { + return None; + } + Some((previous & 0xf000_0000) | ((gpu_address >> 20) as u32)) +} + +/// Whether the current fixed first-dispatch policy avoids sparse-directory +/// growth. Native 8c8b8f8..8c8b9ac admits 28 one-MiB blocks on twelve USCs; +/// 29 requires new backing and is intentionally refused by this owner. +pub(crate) fn g18_initial_directory_fits( + blocks: u32, block_bytes: u64, uscs: u32, flags: u32, page_shift: u32, +) -> Option { + if blocks == 0 || !matches!(block_bytes, 0x10_0000 | 0x20_0000) + || uscs != 12 || flags != 3 || page_shift != 14 { + return None; + } + let units = (block_bytes >> 16).checked_mul(u64::from(blocks))?; + let per_usc = units.checked_add(u64::from(uscs) - 1)? / u64::from(uscs); + let initial_units = (((1u64 << page_shift) - 0x400) >> 9) + 8; + Some(per_usc <= initial_units) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/vm_layout.rs b/drivers/gpu/drm/asahi/g18/fw/vm_layout.rs new file mode 100644 index 0000000000000..5088901447589 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/vm_layout.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Application VA contract shared with Mesa's agx_m5_va.h. + +use core::ops::Range; + +pub(crate) const DRIVER_BASE: u64 = 0x10_0000_0000; +pub(crate) const DRIVER_END: u64 = DRIVER_BASE + (32u64 << 30); +pub(crate) const COMPACT_END: u64 = DRIVER_BASE + (1u64 << 32); +pub(crate) const DATA_START: u64 = 0x40_0000_0000; + +/// TA deflake fields encode an unsigned 32-bit offset from DRIVER_BASE. +/// Keep the private render arena inside that aperture and the USC backing +/// after it, within the same VM_CREATE reservation. A client may leave a +/// prefix of the compact aperture for its own command data. Mesa's default +/// reservation still begins at DRIVER_BASE and data begins at DATA_START. +pub(crate) fn compact_range(reserved: &Range) -> Option> { + if reserved.start < DRIVER_BASE || reserved.start >= COMPACT_END + || reserved.start & 0x3fff != 0 || reserved.end <= COMPACT_END + || reserved.end > DATA_START || reserved.end & 0x3fff != 0 { + return None; + } + Some(reserved.start..COMPACT_END) +} + +pub(crate) fn auxiliary_range(reserved: &Range) -> Option> { + compact_range(reserved)?; + Some(COMPACT_END..reserved.end) +} + +/// Command pointers must not reach driver-owned mappings, including when +/// the client leaves application data below its private arena reservation. +pub(crate) fn user_range_contains( + reserved: &Range, address: u64, size: u64, user_end: u64, +) -> bool { + let Some(end) = address.checked_add(size) else { return false; }; + size != 0 && address >= DRIVER_BASE && end <= user_end + && (end <= reserved.start || address >= reserved.end) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mesa_reservation_separates_compact_and_auxiliary_storage() { + let reserved = 0x10_0000_0000..0x18_0000_0000; + assert_eq!(compact_range(&reserved), Some(0x10_0000_0000..0x11_0000_0000)); + assert_eq!(auxiliary_range(&reserved), Some(0x11_0000_0000..0x18_0000_0000)); + assert_eq!(COMPACT_END - 1 - DRIVER_BASE, u32::MAX as u64); + } + + #[test] + fn incompatible_reservations_are_rejected() { + for reserved in [ + 0x3f7_ffff_8000..0x3ff_ffff_8000, + DRIVER_BASE..COMPACT_END, + (DRIVER_BASE - 0x4000)..DRIVER_END, + (DRIVER_BASE + 1)..DRIVER_END, + COMPACT_END..DRIVER_END, + DRIVER_BASE..(DATA_START + 0x4000), + DRIVER_BASE..0x18_0000_0001, + ] { + assert_eq!(compact_range(&reserved), None); + assert_eq!(auxiliary_range(&reserved), None); + } + } + + #[test] + fn compact_application_prefix_stays_outside_driver_storage() { + let reserved = (DRIVER_BASE + 0x400000)..DRIVER_END; + assert_eq!(compact_range(&reserved), Some(reserved.start..COMPACT_END)); + assert_eq!(auxiliary_range(&reserved), Some(COMPACT_END..DRIVER_END)); + assert!(user_range_contains(&reserved, DRIVER_BASE, 0x400000, 1 << 42)); + assert!(user_range_contains(&reserved, reserved.end, 0x4000, 1 << 42)); + for (address, size) in [ + (DRIVER_BASE - 1, 1), (DRIVER_BASE, 0), + (reserved.start - 1, 2), (reserved.start, 0x4000), + (reserved.end - 1, 2), (DRIVER_BASE, DRIVER_END), + ((1 << 42) - 1, 2), (u64::MAX, 2), + ] { + assert!(!user_range_contains(&reserved, address, size, 1 << 42)); + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/hw.rs b/drivers/gpu/drm/asahi/g18/hw.rs new file mode 100644 index 0000000000000..cc6728b8583c2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/hw.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18G revision and identity types, and the T8152 SoC entry. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. The SoC entry is in `hw::t8152`. +use kernel::prelude::*; +pub(crate) use crate::g18::identity::{GpuGen, GpuHalGeneration, GpuVariant}; + +/// The `hw::agx3` paths used by the carried module-1134 code. +#[allow(unused_imports)] +pub(crate) mod agx3 { + pub(crate) use super::{G18RegistryFacts, SocConfig, T8152, T8152_REGISTRY}; +} + +/// GPU revision enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevision { + A0 = 0x00, + A1 = 0x01, + B0 = 0x10, + B1 = 0x11, + C0 = 0x20, + C1 = 0x21, +} + +/// GPU revision ID. Note: Part of the firmware ABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevisionID { + // Unknown = 0, + A0 = 1, + A1 = 2, + B0 = 3, + B1 = 4, + C0 = 5, + C1 = 6, +} + +/// Specific GPU ID configuration fetched from SGX MMIO registers. +#[derive(Debug)] +pub(crate) struct GpuIdConfig { + /// GPU generation (should match static config). + pub(crate) gpu_gen: GpuGen, + /// GPU variant type (should match static config). + pub(crate) gpu_variant: GpuVariant, + /// Unified shader core generation. + pub(crate) usc_generation: u32, + /// Apple GPU HAL generation (0 for the pre-numbered generations). + pub(crate) gpu_hal_generation: GpuHalGeneration, + /// GPU silicon revision. + pub(crate) gpu_rev: GpuRevision, + /// GPU silicon revision ID (firmware enum). + pub(crate) gpu_rev_id: GpuRevisionID, + /// Number of GPU dies read from the ID register. + pub(crate) num_dies: u32, + /// Total number of GPU clusters. + pub(crate) num_clusters: u32, + /// Maximum number of GPU cores per cluster. + pub(crate) num_cores: u32, + /// Number of frags per cluster. + pub(crate) num_frags: u32, + /// Number of GPs per cluster. + pub(crate) num_gps: u32, + /// Total number of active cores for the whole GPU. + pub(crate) total_active_cores: u32, + /// Mask of active cores per cluster. + pub(crate) core_masks: KVec, + /// Packed mask of all active cores. + pub(crate) core_masks_packed: KVec, + /// GPC performance-state map nibble sampled during the safe SGX ID probe. + /// G17P emits this cached high nibble in fragment register 0x1c838. The + /// 3D descriptor tail at +0x2110 has an independent render-setup source. + pub(crate) gpc_perf_state_map: u32, + /// Low nibble of the same SGX 0xe01480 word, written to 3D tail +0x210c. + pub(crate) gpc_perf_state_map_low: u32, + /// Bit 0 of SGX 0xe0141c, written to 3D tail +0x2104. + pub(crate) gpc_perf_state_control: u32, +} + +// The T8152 SoC entry and its registry facts live with the other SoCs. +pub(crate) use crate::hw::t8152::{G18RegistryFacts, SocConfig, T8152, T8152_REGISTRY}; diff --git a/drivers/gpu/drm/asahi/g18/identity.rs b/drivers/gpu/drm/asahi/g18/identity.rs new file mode 100644 index 0000000000000..ab25cc7803b5d --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/identity.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU identity decoding (family, variant, die count) and the G18 UABI enums. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. + +/// GPU generation enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuGen { + G13 = 13, + G14 = 14, + G15 = 15, + G16 = 16, + G17 = 17, + /// M6 family (T8152 G18G). AGX·PI_400 host HAL. + G18 = 18, +} + +/// GPU variant enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuVariant { + P = 'P' as u32, + G = 'G' as u32, + S = 'S' as u32, + C = 'C' as u32, + D = 'D' as u32, + A = 'A' as u32, +} + +/// Apple's GPU HAL generation. +/// +/// G13 through G15 predate the numbered HAL namespace. This is separate from +/// both the design generation and the USC generation: G17P/G17A use HAL200, +/// while the other G17 parts use HAL300. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuHalGeneration { + Legacy = 0, + Hal200 = 200, + Hal300 = 300, + /// `AGX·PI_400` (26A428 AGXG18G, `__ZTV31AGX·PI_400·G·A0·Accelerator`). + Hal400 = 400, +} + +/// Firmware-role topology presented by Apple's RTBuddy driver. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub(crate) enum FirmwareRoleTopology { + /// One GFX firmware role. + Single, + /// Separate GFX and GFX1 firmware roles. + Dual, +} + +/// Host-to-firmware submission transport present in Apple's driver. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub(crate) enum SubmissionTransport { + /// Classic device-control ring and SKU channels only. + Classic, + /// Both classic and 128-queue SKSM stacks are linked. + ClassicAndSksm, + /// 128-queue SKSM transport only. + Sksm, +} + +/// Identity and userspace-visible architecture axes decoded from GPU ID MMIO. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub(crate) struct GpuIdentity { + pub(crate) gpu_gen: GpuGen, + pub(crate) gpu_variant: GpuVariant, + pub(crate) usc_generation: u32, + pub(crate) gpu_hal_generation: GpuHalGeneration, + pub(crate) firmware_roles: FirmwareRoleTopology, + pub(crate) submission_transport: SubmissionTransport, + /// Input-address bits per UAT root. + pub(crate) uat_input_address_bits: u8, +} + +/// Decode the hardware family/variant/die tuple reported at 0xd04000/0xd04010. +/// +/// This deliberately does not imply that firmware, initdata, command +/// submission, or a core-mask register layout is implemented for the returned +/// identity. +pub(crate) fn decode_gpu_identity(family: u8, variant: u8, num_dies: u8) -> Option { + use GpuGen::*; + use GpuHalGeneration::*; + use GpuVariant::*; + + let (gpu_gen, gpu_variant, usc_generation, gpu_hal_generation) = match (family, variant) { + (0x4, 0) => (G13, P, 2, Legacy), + (0x4, 2) => (G13, G, 2, Legacy), + (0x4, 3) => (G13, S, 2, Legacy), + // G13 has no D core identity; C covers both Max and Ultra. + (0x4, 4) => (G13, C, 2, Legacy), + + (0x5, 0) => (G14, P, 2, Legacy), + (0x5, 2) => (G14, G, 2, Legacy), + (0x6, 3) => (G14, S, 2, Legacy), + (0x6, 4) => match num_dies { + 1 => (G14, C, 2, Legacy), + 2 => (G14, D, 2, Legacy), + _ => return None, + }, + + // G15P is A16, not an M3 GPU. Apple's compiler routes + // applegpu_g15p through the AGX2 backend even though the design + // generation is 15. Keep USC generation independent from gpu_gen. + (0x7, 0) => (G15, P, 2, Legacy), + (0x7, 2) => (G15, G, 3, Legacy), + (0x7, 3) => (G15, S, 3, Legacy), + (0x7, 4) => match num_dies { + 1 => (G15, C, 3, Legacy), + 2 => (G15, D, 3, Legacy), + _ => return None, + }, + + (0x8, 0) => (G16, P, 3, Hal200), + + // Family 0xa crosses the design-generation boundary. + (0xa, 0) => (G17, P, 3, Hal200), + (0xa, 2) => (G16, G, 3, Hal200), + (0xa, 3) => (G16, S, 3, Hal200), + (0xa, 4) => (G16, C, 3, Hal200), + (0xa, 5) => (G17, A, 3, Hal200), + + (0xb, 2) => (G17, G, 3, Hal300), + (0xb, 3) => (G17, S, 3, Hal300), + (0xb, 4) => (G17, C, 3, Hal300), + + // G18 (M6). 26A428 `AGX·PI_400·G·A0·Accelerator::readChipInfo` + // @ 0xfffffe0008c3d3a0 accepts only family 0xc (`lsr w8,w23,#24; + // cmp w8,#0xc` @ 0xfffffe0008c3d498) and decodes variants 0, 2 and 5 + // (@ 0xfffffe0008c3d4a4..0x4b8). Variant 2 stores 12 at +0x78, which + // matches the J873 AGXAcceleratorG18G registry (num_cores=12, + // gpu_var=G, usc_gen=3). Variants 0 and 5 have no named target here + // and stay undecoded. + (0xc, 2) => (G18, G, 3, Hal400), + _ => return None, + }; + + let firmware_roles = if matches!(gpu_gen, G17 | G18) { + FirmwareRoleTopology::Dual + } else { + FirmwareRoleTopology::Single + }; + + let submission_transport = match (family, variant) { + // G16X and G17P/G17A carry both stacks. The runtime selector is not + // yet known, so callers must not silently choose one. + (0xa, 0 | 3 | 4 | 5) => SubmissionTransport::ClassicAndSksm, + // HAL300 removes the classic SKU channel stack. + (0xb, 2 | 3 | 4) => SubmissionTransport::Sksm, + // PI_400 links only `AGX·PI_400·G·A0·{3D,CL,TA}ChannelSKSM`; the + // 26A428 AGXG18G symbol table has no SKU channel class. + (0xc, 2) => SubmissionTransport::Sksm, + _ => SubmissionTransport::Classic, + }; + + let uat_input_address_bits = if matches!(gpu_gen, G13 | G14) { 39 } else { 42 }; + + Some(GpuIdentity { + gpu_gen, + gpu_variant, + usc_generation, + gpu_hal_generation, + firmware_roles, + submission_transport, + uat_input_address_bits, + }) +} diff --git a/drivers/gpu/drm/asahi/g18/initdata/application.rs b/drivers/gpu/drm/asahi/g18/initdata/application.rs new file mode 100644 index 0000000000000..9db38c60458a9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/initdata/application.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +use super::*; +use crate::g18::uapi::application_runtime::Frame; +use crate::g18::runtime::graphics_job::application::ApplicationJob; + +impl G18FirmwareMemory { + pub(crate) fn application_aliases(&mut self, vm: &mmu::Vm) -> Result> { + self.uma_pool.application_aliases(&self.uat,vm) + } + + pub(crate) fn audit_application(&mut self, dev: &AsahiDrmDevice, job: &mut ApplicationJob) -> Result { + let frame=job.frame();let context=u32::from(job.job.context_id()); + let pb=job.job.parameter_buffer().verify_resident(&mut self.pbdesc,context,frame.generation)?; + let uma=self.uma_pool.verify_application_descriptor(&mut self.uma,j873_sma_device_words(2,6)?,frame.generation==1)?; + let pages=self.uma_pool.audit_render_mappings(&self.uat,job.job.vm())?; + job.audit(dev,&self.uat,&pb,&uma,pages)?; + if frame.generation==1 { + let state=self.uma_pool.render_state()?;let mut row=[0u8;128]; + for (out,word) in row.chunks_exact_mut(8).zip(uma) {out.copy_from_slice(&word.to_le_bytes());} + for (name,bytes) in [("uma-state",state.as_slice()),("uma-row",row.as_slice())] { + dev_info!(dev.as_ref(),"G18_RENDER_DIGEST name={} fnv={:#018x}\n",name,crate::g18::fw::graphics_payload::fnv64(bytes)); + } + } + self.completion_masks=[0;2];Ok(()) + } + + pub(crate) fn publish_application_outer(&mut self, records: &[[u8;24];2], frame: Frame) -> Result { + for (index,record) in records.iter().enumerate() { + let qid=if index==0 {10} else {9}; + if record[22]!=qid || record[23]!=1 || record[16..20]!=2u32.to_le_bytes() + || record[20..22]!=(frame.inner_after() as u16).to_le_bytes() + || record[8..16]!=(crate::g18::address_map::GRAPHICS_QUEUE_HIGH+if qid==9 {0} else {0x100}).to_le_bytes() {return Err(EINVAL);} + } + self.publish_application_records(records) + } + + /// Graphics and compute share this outer ring but have independent inner + /// sequences. Only publish from a fully retired observed outer position. + pub(crate) fn publish_application_records(&mut self, records: &[[u8;24]]) -> Result { + if records.is_empty() || records.len()>2 || self.control_counters()?!=[[2;3],[1;3]] { + return Err(EINVAL); + } + let counters=self.compute_outer_counters()?;let before=counters[0]; + if before>=256 || counters!=[before;3] {return Err(EBUSY);} + for record in records { + let address=u64::from_le_bytes(record[8..16].try_into().map_err(|_|EINVAL)?); + let expected=match record[22] { + 9=>crate::g18::address_map::GRAPHICS_QUEUE_HIGH, + 10=>crate::g18::address_map::GRAPHICS_QUEUE_HIGH+0x100, + 11=>crate::g18::address_map::COMPUTE_GRAPH, + _=>return Err(EINVAL), + }; + if address!=expected || record[..8]!=[0;8] || record[23]!=1 + || record[16..20]!=2u32.to_le_bytes() + || u16::from_le_bytes(record[20..22].try_into().map_err(|_|EINVAL)?)>=256 { + return Err(EINVAL); + } + } + let after=(before+records.len() as u32)%256; + self.shared.with_bytes_mut(|out| { + for (index,record) in records.iter().enumerate() { + let start=init::G18G_26A428_WORK_RING_BUNDLE_OFFSETS[8]+((before as usize+index)%256)*24; + out.get_mut(start..start+24).ok_or(EINVAL)?.copy_from_slice(record); + } Ok(()) + })?; + if self.compute_outer_counters()?!=[before;3] {return Err(EBUSY);} + fence(Ordering::SeqCst); + self.private.write_u32(PRIVATE_BASE+(init::G18G_26A428_PRIVATE_STATE_GRIDS[0] + +init::WORK_STATE_GRID_OFFSETS[8]+0x20) as u64,after)?; + fence(Ordering::SeqCst);Ok(after) + } + + pub(crate) fn publish_application_compute_outer(&mut self, inner: u32) -> Result { + if inner>=256 {return Err(EINVAL);} + let mut record=[0;24]; + record[8..16].copy_from_slice(&crate::g18::address_map::COMPUTE_GRAPH.to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&(inner as u16).to_le_bytes()); + record[22]=11;record[23]=1; + self.publish_application_records(&[record]) + } +} diff --git a/drivers/gpu/drm/asahi/g18/initdata/application_compute.rs b/drivers/gpu/drm/asahi/g18/initdata/application_compute.rs new file mode 100644 index 0000000000000..7dba393568828 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/initdata/application_compute.rs @@ -0,0 +1,534 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded application CDM owner, separate from the captured addition fixture. +use super::*; +use crate::{g18::uapi::command::TranslatedComputeCommand, g18::address_map as va, g18::fw::compute::*}; +use core::sync::atomic::Ordering; + +// Application-compute-only knobs K1-K5 (compute diagnostics) and +// g18_app_d62a (case 10). Unlike +// kernel::module! parameters (perm 0, set once at load), these are 0644 with +// the C uint ops, so /sys/module/aurora/parameters/g18_app_* may be changed +// between submissions. prepare_application_compute reads each exactly once +// per submission; nothing else reads them, so graphics and the stage8 fixture +// are unaffected. Registration otherwise follows g18_render_policy.rs. +macro_rules! app_knob { + ($module:ident, $name:literal, $default:expr, $description:literal) => { + mod $module { + use kernel::{bindings, module_param::KernelParam}; + pub(super) static VALUE: core::sync::atomic::AtomicU32 = + core::sync::atomic::AtomicU32::new($default); + #[link_section = "__param"] + #[used(compiler)] + static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + kernel::c_str!($name) + } else { kernel::c_str!(concat!("aurora.", $name)) }), + // Module lifetime and parameter serialization are managed by the + // kernel, identically to the kernel::module! registration. A raw + // address avoids a reference to the mutable extern static. + #[cfg(MODULE)] + mod_: &raw mut bindings::__this_module, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + // SAFETY: exported vmlinux ops; they access `arg` as an unsigned + // int under the parameter lock, which AtomicU32 represents exactly. + ops: unsafe { core::ptr::from_ref(&bindings::param_ops_uint) }, + perm: 0o644, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { + arg: core::ptr::from_ref(&VALUE).cast_mut().cast(), + }, + }); + const TYPE: &[u8] = concat!("parmtype=", $name, ":uint\0").as_bytes(); + #[link_section = ".modinfo"] + #[used(compiler)] + static TYPE_INFO: [u8; TYPE.len()] = super::modinfo(TYPE); + const TEXT: &[u8] = concat!("parm=", $name, ":", $description, "\0").as_bytes(); + #[link_section = ".modinfo"] + #[used(compiler)] + static TEXT_INFO: [u8; TEXT.len()] = super::modinfo(TEXT); + } + }; +} + +const fn modinfo(bytes: &[u8]) -> [u8; N] { + let mut out = [0; N]; + let mut i = 0; + while i < N { out[i] = bytes[i]; i += 1; } + out +} + +app_knob!(knob_d514, "g18_app_d514", 0xffff_ffff, + "Application compute command +0xf28 (native desc+0x514). Default 0xffffffff = native, fixture and M5; 0 = candidate160. Read at every compute submission"); +// Defaults are the values that retire on hardware (run 162d: exact integer +// compute and atomics readbacks). The native 0xa731=1 alone stalls the +// dispatch silently, with no fault or watchdog. +app_knob!(knob_a731, "g18_app_a731", 0, + "Application compute first 0xa731: 0 (default) = fixture/M5 (emits 0), proven on hardware; 1 = native integer compute (d420=1,d418=2 emits 1), which stalls the dispatch on Linux. Other values refuse the submission"); +app_knob!(knob_sma_pair, "g18_app_sma_pair", 0x6_0010, + "Application compute 0xa5e1 SMA pair as d550|d558<<16: 0x60010 (16,6) fixture (default, proven on hardware), 0xb001e (30,11) native integer compute, 0xc001e (30,12) candidate160. Must fit the pool plan (30,12); other values refuse the submission"); +app_knob!(knob_sma_units, "g18_app_sma_units", 0, + "Application compute 0x10241[63:16] SMA result (AGXUMADataRec+0x100). Default 0 (proven on hardware); 0xae = native integer compute. Independent of the pool binding generation"); +app_knob!(knob_prepare_test, "g18_app_prepare_test", 0, + "Unpublished preparation rollback E2E: 0 disabled, 1 inject four allocation failures before the next compute command is encoded, 2 do this for the next two commands to cover reuse after retirement. Verify rollback then prepare each original command. Other values refuse before preparation"); +app_knob!(knob_pool_test, "g18_app_pool_test", 0, + "Pool replacement E2E: arm 1 before the first compute. Starts with the qualified 8-MiB pool, then runs original non-spill commands with 30,40,30-MiB shared pools; internal phases 2-4 replace only after verified retirement. Checks staged-owner rollback and reused addresses. Does not admit scratch shaders"); +app_knob!(knob_preempt_mode, "g18_app_preempt_mode", 1, + "Application compute 0x1a510 object: 1 (default) = fresh zeroed 0xc000 bytes mapped only in the job VM at 0x1100000000 (driver arena); 0 = candidate160 16 KiB page at 0x7050000000. Other values refuse the submission"); +app_knob!(knob_d62a, "g18_app_d62a", 1, + "Application compute desc+0x62a (0x1a458 0x14c08a60 texture mode instead of 0x10c08860): 1 (default) = from the command's DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE flag, as native; 0 = never; 2 = always. Other values refuse the submission"); + +/// Every SMA pair witnessed in a completed native or fixture compute command. +const SMA_PAIRS: [[u64; 2]; 3] = [[30, 11], [30, 12], [16, 6]]; + +/// One submission's knob values, read once and logged with its kick. +#[derive(Clone, Copy)] +pub(crate) struct ApplicationComputeKnobs { + d514: u32, + a731: u32, + sma_pair: u32, + sma_units: u32, + metadata_plan: bool, + preempt_mode: u32, + d62a_mode: u32, + preempt: u64, + preempt_bytes: u64, + pool_slot: u32, + /// The command's USES_TEXTURE_STATE flag and the d62a it produced. + texture_state: bool, + d62a: u8, +} + +impl ApplicationComputeKnobs { + fn read() -> Self { + Self { + d514: knob_d514::VALUE.load(Ordering::Relaxed), + a731: knob_a731::VALUE.load(Ordering::Relaxed), + sma_pair: knob_sma_pair::VALUE.load(Ordering::Relaxed), + sma_units: knob_sma_units::VALUE.load(Ordering::Relaxed), + metadata_plan: false, + preempt_mode: knob_preempt_mode::VALUE.load(Ordering::Relaxed), + d62a_mode: knob_d62a::VALUE.load(Ordering::Relaxed), + preempt: 0, preempt_bytes: 0, pool_slot: 0, + texture_state: false, d62a: 0, + } + } + + /// Native run112 sets desc+0x62a (0x1a458 = 0x14c08a60) on every compute + /// that uses texture or image instructions and on none of the + /// buffer-only ones; Mesa reports that per command. + fn resolve_d62a(&mut self, texture_state: bool) { + self.texture_state = texture_state; + self.d62a = match self.d62a_mode { + 0 => 0, + 2 => 1, + _ => u8::from(texture_state), + }; + } + + fn pair(&self) -> [u64; 2] { + [u64::from(self.sma_pair & 0xffff), u64::from(self.sma_pair >> 16)] + } + + fn apply_plan(&mut self, plan: crate::g18::uapi::compute_plan::G18ComputePlan, + dev: &AsahiDrmDevice) -> Result { + // Fixed knobs remain available for legacy diagnostics. Never combine + // them with shader-derived ownership or silently ignore an override. + if self.sma_pair != 0x6_0010 || self.sma_units != 0 { + dev_err!(dev.as_ref(), "G18_APP_COMPUTE_REFUSED metadata with SMA overrides\n"); + return Err(EINVAL); + } + self.sma_pair = plan.ideal.register_pair[0] | plan.ideal.register_pair[1] << 16; + self.sma_units = plan.sma_units; + self.metadata_plan = true; + Ok(()) + } + + /// Refuse, before any owner changes, values that would make the command + /// inconsistent. Nothing is published for a refused submission. + fn validate(&self, dev: &AsahiDrmDevice) -> Result { + let refusal = if self.a731 > 1 { + Some(("g18_app_a731", self.a731, "admitted 0 (fixture) or 1 (native)")) + } else if !SMA_PAIRS.contains(&self.pair()) { + Some(("g18_app_sma_pair", self.sma_pair, + "admitted 0xb001e (30,11), 0xc001e (30,12) or 0x60010 (16,6)")) + } else if self.preempt_mode > 1 { + Some(("g18_app_preempt_mode", self.preempt_mode, + "admitted 0 (0x7050000000 page) or 1 (driver arena)")) + } else if self.d62a_mode > 2 { + Some(("g18_app_d62a", self.d62a_mode, + "admitted 0 (never), 1 (from the command flag) or 2 (always)")) + } else { None }; + if let Some((knob, value, reason)) = refusal { + dev_err!(dev.as_ref(), + "G18_APP_COMPUTE_REFUSED knob={} value={:#x}: {}; command not sent\n", + knob, value, reason); + return Err(EINVAL); + } + Ok(()) + } +} + +impl kernel::fmt::Display for ApplicationComputeKnobs { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let [d550, d558] = self.pair(); + write!(f, "d514={:#x} a731={} sma_pair=({},{}) reg_a5e1={:#x} sma_units={:#x} reg_10241={:#x} preempt_mode={} preempt={:#x}+{:#x} texture_state={} d62a_mode={} d62a={} metadata_plan={}", + self.d514, self.a731, d550, d558, d550 | d558 << 16, self.sma_units, + (self.pool_slot as i32 as i64 as u64) | u64::from(self.sma_units) << 16, + self.preempt_mode, self.preempt, self.preempt_bytes, + u8::from(self.texture_state), self.d62a_mode, self.d62a, u8::from(self.metadata_plan)) + } +} + +pub(super) struct ApplicationCompute { + guard: Option, + aliases: KVec, + // g18_app_preempt_mode=1 object, mapped only in the submitting job VM. + // Replaced only after the previous generation retired. + driver_preempt: Option, + pool: G18UmaPoolMemory, + binding: G18UmaPoolBinding, + previous_pool: Option, + previous_binding: Option, + bound_context: u32, + successful_retirement: bool, + /// Distinguish production shader plans from the legacy replacement probe. + metadata_pool: bool, + preempt: MappedObject, + generation: u32, + retired: bool, +} + +impl G18FirmwareMemory { + pub(crate) fn prepare_application_compute(&mut self, dev: &AsahiDrmDevice, + vm: &mmu::Vm, command: &TranslatedComputeCommand) + -> Result<(G18PreparedComputeCommand, G18UmaPoolBinding, u32, ApplicationComputeKnobs)> { + if command.sampler_heap != 0 || command.sampler_count != 0 + || command.g17p_add3_buffers.is_some() + || command.control_stream_end.checked_sub(command.control_stream_base) + .map_or(true, |n| n > crate::g18::drm::G18_COMPUTE_STREAM_BYTES) { + pr_err!("G18 compute prepare rejected: sampler_heap={:#x} samplers={} add3={} stream {:#x}..{:#x} (admits at most {} bytes)\n", + command.sampler_heap, command.sampler_count, command.g17p_add3_buffers.is_some(), + command.control_stream_base, command.control_stream_end, + crate::g18::drm::G18_COMPUTE_STREAM_BYTES); + return Err(ENOTSUPP); + } + let mut knobs = ApplicationComputeKnobs::read(); + knobs.validate(dev)?; + knobs.resolve_d62a(command.g18_texture_state); + let diagnostic = knob_prepare_test::VALUE.swap(0, Ordering::Relaxed); + if diagnostic > 2 { return Err(EINVAL); } + let pool_test = knob_pool_test::VALUE.swap(0, Ordering::Relaxed); + if pool_test > 4 || (pool_test != 0 && diagnostic != 0) { return Err(EINVAL); } + let metadata_plan = match command.g18_resource_descriptor.as_ref() { + Some(words) => Some(crate::g18::uapi::compute_plan::G18ComputePlan::from_descriptor(words) + .ok_or(EINVAL)?), + None => None, + }; + if let Some(plan) = metadata_plan { + if diagnostic != 0 || pool_test != 0 { return Err(EINVAL); } + knobs.apply_plan(plan, dev)?; + } + if self.application_compute.as_ref().is_some_and(|job| job.metadata_pool) + && (diagnostic != 0 || pool_test != 0) { return Err(EINVAL); } + let pool_plan = if pool_test != 0 { + let mut words = [48,12,0,0,2304,24,0,0,2,1,0,0,0,0,0,0,1,0,0,128,4,0,0]; + if pool_test == 3 { words[4] = 3072; } + Some(crate::g18::uapi::compute_plan::G18ComputePlan::from_descriptor(&words).ok_or(EINVAL)?) + } else { None }; + if pool_test == 1 && self.application_compute.is_some() + || pool_test > 1 && self.application_compute.is_none() { return Err(EINVAL); } + if self.application_compute.is_none() { + let high = va::DYNAMIC_HIGH_START..va::DYNAMIC_HIGH_END; + let mut pool = if let Some(plan) = metadata_plan { + G18UmaPoolMemory::new_shared_compute(dev, &self.uat, high, plan)? + } else { + G18UmaPoolMemory::new_application_compute(dev, &self.uat, high)? + }; + let binding = pool.prepare_binding()?; + let preempt = MappedObject::new_wc_in_range(dev, self.uat.kernel_lower_vm(), + 0x70_5000_0000..0x70_5100_0000, 0x4000, 0x4000, + mmu::PROT_GPU_FW_SHARED_RW)?; + let mut owner = KBox::new(ApplicationCompute { + guard: None, aliases: KVec::new(), driver_preempt: None, pool, binding, + previous_pool: None, previous_binding: None, bound_context: 0, + successful_retirement: true, + metadata_pool: metadata_plan.is_some(), + preempt, generation: 0, retired: true, + }, GFP_KERNEL)?; + self.scheduler.prepare(0, 1)?; + owner.pool.load_hardware_descriptor(dev, &mut self.uma, binding, + j873_sma_device_words(2, 6)?)?; + self.application_compute = Some(owner); + } + if diagnostic != 0 { + let before_generation = self.application_compute_generation(); + let before = self.scheduler.snapshot(0)?; + for checkpoint in 1..=4 { + let result = self.prepare_application_compute_attempt(dev, vm, command, knobs, checkpoint, None); + if !matches!(result, Err(error) if error == ENOMEM) { return Err(EIO); } + let owner = self.application_compute.as_ref().ok_or(EIO)?; + let after = self.scheduler.snapshot(0)?; + if !owner.retired || owner.guard.is_some() || !owner.aliases.is_empty() + || owner.driver_preempt.is_some() || owner.generation != before_generation + || !self.compute.unpublished_idle() + || (before.submitted, before.completed, before.linked) + != (after.submitted, after.completed, after.linked) { return Err(EIO); } + dev_info!(dev.as_ref(), + "G18_COMPUTE_PREPARE_E2E checkpoint={} rollback=pass generation={} scheduler={}/{}/{} published=0\n", + checkpoint, before_generation, after.submitted, after.completed, after.linked); + } + } + if pool_test > 1 { + let job = &mut **self.application_compute.as_mut().ok_or(EIO)?; + let scheduler = self.scheduler.snapshot(0)?; + if !job.retired || !job.successful_retirement || job.previous_pool.is_some() + || job.generation == 0 || scheduler.submitted != job.generation + || scheduler.completed != job.generation || scheduler.linked != 0 { + return Err(EBUSY); + } + job.pool.verify_replacement_rollback(dev, &self.uat, pool_plan.ok_or(EINVAL)?)?; + } + let replacement = if let Some(plan) = metadata_plan { + let job = self.application_compute.as_ref().ok_or(EIO)?; + (job.pool.shared_compute_plan() != Some(plan)).then_some(plan) + } else if pool_test > 1 { pool_plan } else { None }; + let prepared = self.prepare_application_compute_attempt(dev, vm, command, knobs, 0, replacement)?; + if pool_test != 0 && pool_test < 4 { + let _ = knob_pool_test::VALUE.compare_exchange(0, pool_test + 1, + Ordering::Relaxed, Ordering::Relaxed); + } + if diagnostic == 2 { + // The next submission still requires this command to retire. + // Preserve a concurrent explicit parameter change, and never add + // a fallible operation after the prepared owner was committed. + let _ = knob_prepare_test::VALUE.compare_exchange(0, 1, + Ordering::Relaxed, Ordering::Relaxed); + } + Ok(prepared) + } + + fn prepare_application_compute_attempt(&mut self, dev: &AsahiDrmDevice, + vm: &mmu::Vm, command: &TranslatedComputeCommand, + mut knobs: ApplicationComputeKnobs, fail_checkpoint: u32, + replacement: Option) + -> Result<(G18PreparedComputeCommand, G18UmaPoolBinding, u32, ApplicationComputeKnobs)> { + let job = &mut **self.application_compute.as_mut().ok_or(EIO)?; + if !job.retired || job.guard.is_some() || !self.compute.unpublished_idle() + || job.previous_pool.is_some() || job.previous_binding.is_some() { return Err(EBUSY); } + let generation = job.generation.checked_add(1).ok_or(EOVERFLOW)?; + if generation > 0x00ff_ffff { return Err(EOVERFLOW); } + let previous = self.scheduler.snapshot(0)?; + if previous.submitted != job.generation || previous.completed != job.generation + || previous.linked != 0 { return Err(EBUSY); } + // The pool is sized once for its plan pair; a larger pair would name + // backing that does not exist. The slot must not reach the SMA half. + let mut candidate = if let Some(plan) = replacement { + if !job.successful_retirement || job.generation == 0 { return Err(EBUSY); } + let mut pool = job.pool.replacement_shared_compute(dev, &self.uat, + va::DYNAMIC_HIGH_START..va::DYNAMIC_HIGH_END, plan)?; + let mut binding = pool.prepare_binding()?; + binding.generation = job.binding.generation.checked_add(1).ok_or(EOVERFLOW)?; + if binding.generation == u64::MAX { return Err(EOVERFLOW); } + Some((pool, binding)) + } else { None }; + let (pool, binding) = match candidate.as_mut() { + Some((pool, binding)) => (pool, *binding), + None => (&mut job.pool, job.binding), + }; + let plan = pool.scratch_plan().ok_or(EINVAL)?; + let pair = knobs.pair(); + if let Some(expected) = command.g18_resource_descriptor.as_ref() { + let expected = crate::g18::uapi::compute_plan::G18ComputePlan::from_descriptor(expected) + .ok_or(EINVAL)?; + if pool.shared_compute_plan() != Some(expected) + || pair != expected.ideal.register_pair.map(u64::from) + || knobs.sma_units != expected.sma_units || !knobs.metadata_plan { + return Err(EIO); + } + crate::g18_kick_info!(dev.as_ref(), + "G18_COMPUTE_RESOURCE_PLAN bytes={} blocks={} pair={:?} counters={:?} sma_units={} replacement={}\n", + expected.ideal.bytes, expected.ideal.block_count, pair, + expected.ideal.chunk_counters, expected.sma_units, u8::from(replacement.is_some())); + } else if job.metadata_pool + && (plan.block_bytes != 0x20_0000 || plan.block_count < 4 + || plan.register_pair[0] < 30 || plan.register_pair[1] < 12 + || plan.chunk_counters[1] < 260) + { + dev_err!(dev.as_ref(), "G18_APP_COMPUTE_REFUSED legacy command exceeds retained native pool\n"); + return Err(ENOTSUPP); + } + if pair[0] > plan.register_pair[0] || pair[1] > plan.register_pair[1] { + dev_err!(dev.as_ref(), + "G18_APP_COMPUTE_REFUSED knob=g18_app_sma_pair value={:#x}: exceeds pool plan {:?}; command not sent\n", + knobs.sma_pair, plan.register_pair); + return Err(EINVAL); + } + if binding.slot >= 0x1_0000 { + dev_err!(dev.as_ref(), + "G18_APP_COMPUTE_REFUSED knob=g18_app_sma_units value={:#x}: pool slot {} overlaps 0x10241[63:16]; command not sent\n", + knobs.sma_units, binding.slot); + return Err(EINVAL); + } + knobs.pool_slot = binding.slot; + let driver = va::APPLICATION_COMPUTE_PREEMPT_DRIVER; + let driver_bytes = va::APPLICATION_COMPUTE_PREEMPT_DRIVER_BYTES; + let driver_range = driver..driver + driver_bytes + mmu::UAT_PGSZ as u64; + let reserved = vm.kernel_range(); + if knobs.preempt_mode == 1 + && (driver_range.start < reserved.start || driver_range.end > reserved.end) { + dev_err!(dev.as_ref(), + "G18_APP_COMPUTE_REFUSED knob=g18_app_preempt_mode value=1: VM_CREATE driver range {:#x}..{:#x} does not contain {:#x}..{:#x}; command not sent\n", + reserved.start, reserved.end, driver_range.start, driver_range.end); + return Err(EINVAL); + } + let guard = vm.retain_t8140_job()?; + job.aliases.clear(); + job.driver_preempt = None; + let attempt = (|| { + if fail_checkpoint == 1 { return Err(ENOMEM); } + let context = self.compute.bind_process(dev, &self.uat, vm)?; + if fail_checkpoint == 2 { return Err(ENOMEM); } + let mut aliases = pool.compute_application_aliases(dev, &self.uat, vm)?; + if fail_checkpoint == 3 { return Err(ENOMEM); } + let mut driver_preempt = None; + let preempt = if knobs.preempt_mode == 0 { + let preempt = job.preempt.iova(); + job.preempt.with_bytes_mut(|bytes| { bytes.fill(0); Ok(()) })?; + let alias = job.preempt.map_alias_at(vm, preempt, mmu::PROT_GPU_SHARED_RW)?; + aliases.push(alias, GFP_KERNEL)?; + if vm.translate_iova(preempt)? != self.uat.kernel_lower_vm().translate_iova(preempt)? { + return Err(EFAULT); + } + knobs.preempt_bytes = 0x4000; + preempt + } else { + // Native keeps this object in the process VM only (0x100_003f_0000, + // 48 KiB). The driver arena is refused to userspace binds, and the + // range admits exactly one placement: the object plus one guard. + let object = MappedObject::new_wc_in_range_with_guards(dev, vm, driver_range, + driver_bytes as usize, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, 1)?; + let preempt = object.iova(); + driver_preempt = Some(object); + if preempt != driver { return Err(EFAULT); } + for offset in (0..driver_bytes).step_by(mmu::UAT_PGSZ) { + vm.translate_iova(preempt + offset)?; + } + if vm.translate_iova(preempt + driver_bytes).is_ok() { return Err(EFAULT); } + knobs.preempt_bytes = driver_bytes; + preempt + }; + knobs.preempt = preempt; + if fail_checkpoint == 4 { return Err(ENOMEM); } + let inputs = G18ComputeCommandInputs { + sequence: u64::from(generation), d100: 1, + completion: G18ComputeCompletionState { + stamp: generation << 8, index: 11, entries: 256, generation: 0, base: 1, + }, + d4e0: command.control_stream_terminator, d7b8: binding.state, + ..Default::default() + }; + // G18 preemption pointers are +0x1b80..+0x1b98 (native gallery), + // not the M5 +0x1480 layout. These are fresh driver-owned zero pages. + let registers = G18ComputeRegisterInputs { + program: [preempt, command.control_stream_base, preempt + 0x1b80, + preempt + 0x1b88, preempt + 0x1b90, preempt + 0x1b98], + d550: pair[0], d558: pair[1], d774: 0x1c, d5c4: 8, + d420: knobs.a731 as u8, d418: 2, d50c: 1, d514: knobs.d514, + d62a: knobs.d62a, + context_id: context, pool_slot: binding.slot, + sma_units: u64::from(knobs.sma_units), + a10aac: 1, a6d0: 1, clusters: 2, cluster_width: 6, + ..Default::default() + }; + let prepared = self.compute.prepare(self.scheduler.binding(0)?, &inputs, ®isters)?; + Ok((prepared, aliases, driver_preempt, knobs)) + })(); + let (prepared, aliases, driver_preempt, knobs) = match attempt { + Ok(value) => value, + Err(error) => { + // The local VM guard outlives process/alias cleanup. A cleanup + // failure remains fatal; it is never called a clean rollback. + self.compute.abort_unpublished_process()?; + return Err(error); + } + }; + job.guard = Some(guard); + job.aliases = aliases; + job.driver_preempt = driver_preempt; + job.retired = false; + job.generation = generation; + if command.g18_resource_descriptor.is_some() { job.metadata_pool = true; } + if let Some((pool, binding)) = candidate { + job.previous_pool = Some(core::mem::replace(&mut job.pool, pool)); + job.previous_binding = Some(job.binding); + job.binding = binding; + } + self.completion_masks = [0; 2]; + Ok((prepared, job.binding, generation, knobs)) + } + + /// The binding the firmware must still own before the next tag15. + pub(crate) fn application_compute_previous_binding(&self) -> Result<(G18UmaPoolBinding, u32)> { + let job = self.application_compute.as_ref().ok_or(EIO)?; + Ok((job.previous_binding.unwrap_or(job.binding), job.bound_context)) + } + + /// Called only after the session checked the old firmware binding and the + /// retired scheduler. Both views remain owned if any later step fails. + pub(crate) fn install_application_compute_pool(&mut self, dev: &AsahiDrmDevice) -> Result { + let job = &mut **self.application_compute.as_mut().ok_or(EIO)?; + if let Some(previous) = job.previous_pool.as_mut() { + let old = job.previous_binding.ok_or(EIO)?; + let scheduler = self.scheduler.snapshot(0)?; + if !job.successful_retirement || job.generation < 2 + || scheduler.submitted != job.generation - 1 + || scheduler.completed != job.generation - 1 || scheduler.linked != 0 { + return Err(EBUSY); + } + job.pool.replace_hardware_descriptor(dev, &mut self.uma, job.binding, + previous, old, j873_sma_device_words(2, 6)?)?; + crate::g18_kick_info!(dev.as_ref(), + "G18_COMPUTE_POOL_SWITCH compute={} old_state={:#x} new_state={:#x} old_generation={} new_generation={} slot={} old_retained=1\n", + job.generation, old.state, job.binding.state, old.generation, + job.binding.generation, job.binding.slot); + } + Ok(()) + } + + /// The caller must first verify the new firmware binding, including both + /// directions of queue membership. A completion stamp alone is insufficient. + pub(crate) fn retire_application_compute(&mut self, generation: u32, context: u32) -> Result { + let job = &mut **self.application_compute.as_mut().ok_or(EIO)?; + if job.retired || job.generation != generation { return Err(EINVAL); } + self.compute.retire_application(generation << 8)?; + job.bound_context = context; + job.successful_retirement = true; + job.retired = true; + job.guard = None; + job.previous_pool = None; + job.previous_binding = None; + Ok(()) + } + + /// The last compute stamp kicked to QID 11 (0 before the first dispatch). + pub(crate) fn application_compute_generation(&self) -> u32 { + self.application_compute.as_ref().map_or(0, |job| job.generation) + } + + /// As [`Self::retire_application_compute`] for a dispatch a recovery ended. + pub(crate) fn retire_recovered_application_compute(&mut self, generation: u32) -> Result { + let job = &mut **self.application_compute.as_mut().ok_or(EIO)?; + if job.retired || job.generation != generation { return Err(EINVAL); } + self.compute.retire_recovered()?; + job.successful_retirement = false; + job.retired = true; + job.guard = None; + Ok(()) + } + + pub(crate) fn quarantine_application_compute(&mut self) { + if let Some(job) = &self.application_compute { + if let Some(guard) = &job.guard { guard.quarantine(); } + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/initdata/mod.rs b/drivers/gpu/drm/asahi/g18/initdata/mod.rs new file mode 100644 index 0000000000000..50b8901e252a4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/initdata/mod.rs @@ -0,0 +1,949 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![allow(dead_code)] + +//! Retained RAM for the J873 / T8152 / G18G / 26A428 initialization graph +//! (plan M1, steps 1.1-1.7). +//! +//! Construction only. Nothing here starts a processor, publishes on a +//! mailbox, reads or writes MMIO, requests PMGR/PSD power, touches the ASC +//! wrappers or the post-office bank, or submits work: every write lands in +//! host-allocated GEM memory or in UAT page tables. Register windows are +//! mapped for the firmware (page-table writes only); their contents are never +//! accessed from the host. +//! +//! The G18 live-boot session owns this object. It must build the UAT with +//! `mmu::Uat::new_t8152` (plan step 0.6) before calling +//! [`G18FirmwareMemory::new_with_uat`], and must retain both the UAT and this +//! owner until BOTH firmware processors are stopped, including on failure. +//! +//! Derived from `m5_resources.rs` with the M2/M3 compute and graphics owners +//! removed (they depend on the unported HAL400 submission ABI). The firmware +//! VA plan is the J813 one: the addresses are host-chosen and both shared +//! allocations keep their 25G76 page totals (0x28000 / 0x178000) in 26A428. +//! Every object inside the allocations follows the 26A428 kext layout from +//! `g17_initdata` (`G18G_26A428_*`). Values that are not proven for 26A428 +//! are listed in `g17_initdata::G18G_26A428_UNPROVEN`. +//! +//! Integration (outside this file): include it from `g17_resources.rs` next +//! to the M5 module, so it can use the private `MappedObject`: +//! +//! ```ignore +//! #[cfg(not(test))] +//! #[path = "g18_resources.rs"] +//! pub(crate) mod g18; +//! ``` + +#[path = "application.rs"] +mod application; +#[path = "application_compute.rs"] +mod application_compute; +use application_compute::ApplicationCompute; + +use crate::g18::buffer::object::MappedObject; +use crate::g18::buffer::pool::{G18UmaPoolBinding, G18UmaPoolMemory}; +use crate::g18::queue::scheduler::G18SchedulerMemory; +use crate::g18::runtime::compute::{G18ComputeMemory, G18PreparedComputeCommand}; +use crate::g18::runtime::compute_payload::G18ComputePayloadMemory; +use crate::{driver::AsahiDrmDevice, g18::fw::initdata as init, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; +use kernel::sync::Arc; + +const PROFILE: init::RootProfile = init::RootProfile::J873G18G26A428; + +const KERNEL_BASE: u64 = 0xffff_fc20_0000_0000; +/// Shared allocation #2 (the private cluster, 0x178000 bytes). +const PRIVATE_BASE: u64 = KERNEL_BASE + 0x2_0000; +const REGION_C_BASE: u64 = KERNEL_BASE + 0x1a_0000; +/// The secondary root sits [`init::SECONDARY_ROOT_DELTA`] above the primary. +const ROOTS_BASE: u64 = KERNEL_BASE + 0x1a_8000; +const REGION_A_BASE: u64 = KERNEL_BASE + 0x1b_8000; +/// ArmFirmware fwctl allocation: 0x20-byte state, then the ring at +0x40. +const FWCTL_BASE: u64 = KERNEL_BASE + 0x1c_0000; +const FWCTL_SIZE: usize = 0x4000; +/// Shared allocation #1 (hw_data, policy blocks, rings, main configs). +const SHARED_BASE: u64 = KERNEL_BASE + 0xc078_8000; +/// Main `+0x2d0` blank sentinel; unmapped at handoff, as on J813. +const SENTINEL_BASE: u64 = KERNEL_BASE + 0xc07e_0000; +/// PB descriptor and UMA pool tables, aliased at [`init::REGION_VIEW_LOW_ADDRS`]. +const PBDESC_BASE: u64 = KERNEL_BASE + 0x15e_0000; +const UMA_BASE: u64 = KERNEL_BASE + 0x15e_8000; +const PBDESC_TABLE_SIZE: usize = 0x4000; +/// 26A428 createHardwareMappings at 0xfffffe0008c41978 requests 0x8000 +/// bytes: 256 native 0x80-byte UMA descriptors, rather than the M5 page. +const UMA_TABLE_SIZE: usize = 0x8000; +/// The four KSM completion queues (low context-0 alias / firmware-high). +const COMPLETION_LOW_BASE: u64 = 0x70_0184_8000; +const COMPLETION_HIGH_BASE: u64 = KERNEL_BASE + 0xc07e_8000; +const COMPLETION_STRIDE: u64 = 0x2_8000; +/// Allocator-selected objects (QoS block, AON PTD) stay between the PB/UMA +/// tables and the fixed high graph. +const DYNAMIC_START: u64 = KERNEL_BASE + 0x168_0000; +const DYNAMIC_END: u64 = KERNEL_BASE + 0x1000_0000; + +const _: () = { + assert!(PBDESC_BASE + PBDESC_TABLE_SIZE as u64 <= UMA_BASE); + assert!(UMA_BASE + UMA_TABLE_SIZE as u64 <= DYNAMIC_START); + assert!(init::REGION_VIEW_LOW_ADDRS[0] + PBDESC_TABLE_SIZE as u64 + <= init::REGION_VIEW_LOW_ADDRS[1]); + assert!(init::REGION_VIEW_LOW_ADDRS[1] + UMA_TABLE_SIZE as u64 <= COMPLETION_LOW_BASE); +}; + +/// Report rings keep their 25G76 shape (FW_PROVEN: event producer b000 +/// 0x7f14, 256 x 0x48; trace producer b000 0x19984, 512 x 0x48; host owns the +/// cursor at +0, firmware the producer at +0x20). +const REPORT_RECORD_SIZE: usize = 0x48; +const REPORT_PRODUCER_OFFSET: u64 = 0x20; +const REPORT_CAPACITY: [u32; 2] = [256, 512]; +const REPORT_DRAIN_BUDGET: usize = 128; +const TRACE_PRINT_BUDGET: [u32; 2] = [512, 16]; +/// Host-only storage; do not change the native firmware ring geometry. +/// Roughly 5 MiB for both roles, allocated before publication. +const TRACE_JOURNAL_CAPACITY: usize = 32768; +/// b000 0x7de8 constructs type4 with counter at +4 and stamp index at +12. +const EVENT_TYPE_RECOVERY: u32 = 4; + +/// `halLoadUMADesc` device words (accelerator+0x11af8..0x11b10) as native +/// `halInitSMAEnableMask` 0xfffffe0008c41bdc derives them on J873. +/// getProbeScore stores readChipInfo's 0xe01500 core-enable word as the +/// accelerator+0x480 fallback (0xfffffe0008c3d6c4, 0xfffffe0008c002d8); the +/// +0x470 override stays zero without gpu-fstp-downgrade properties, which the +/// J873 ADT lacks. Width +0x4ec is ID_COUNTS_1[7:0] (0xfffffe0008c3d554) and +/// +0x4d4 the cluster count; generateRegisterList reads the same two fields, +/// so they must equal the command topology. getPropertyCoreConfigImpl splits +/// the same selected mask into the native registry core_mask_list, so every +/// 16-bit lane must equal that list (zero beyond it). +fn j873_sma_device_words(clusters: u32, width: u32) -> Result<[u64; 4]> { + let facts = crate::g18::hw::T8152_REGISTRY; + if clusters != facts.num_mgpus || facts.core_mask_list.len() != clusters as usize + || !(1..=16).contains(&width) || width.checked_mul(clusters) != Some(facts.num_cores) { + return Err(EINVAL); + } + let mut mask = 0u64; + for (index, &cores) in facts.core_mask_list.iter().enumerate() { + if u64::from(cores) >> width != 0 { return Err(EINVAL); } + mask |= u64::from(cores) << (index as u32 * width); + } + let words = crate::g18::fw::uma::pack_g18_sma_masks([0; 2], [mask, 0], width).ok_or(EINVAL)?; + for group in 0..16 { + let lane = (words[group / 4] >> ((group % 4) * 16)) & 0xffff; + if lane != facts.core_mask_list.get(group).map_or(0, |&cores| u64::from(cores)) { + return Err(EINVAL); + } + } + Ok(words) +} + +struct G18TraceRecord { + ordinal: u32, + raw: [u8; REPORT_RECORD_SIZE], +} + +/// Owning mappings are declared before the UAT so they drop first. Aliases +/// retain the SAME GEM backing; no firmware pointer names a copied snapshot. +pub(crate) struct G18FirmwareMemory { + primary_root: MappedObject, + secondary_root: MappedObject, + shared: MappedObject, + private: MappedObject, + region_a: MappedObject, + region_c: MappedObject, + fwctl: MappedObject, + aon: MappedObject, + qos: MappedObject, + pbdesc: MappedObject, + uma: MappedObject, + uma_pool: G18UmaPoolMemory, + scheduler: G18SchedulerMemory, + compute: G18ComputeMemory, + compute_payload: Option, + application_compute: Option>, + completions: KVec, + aliases: KVec, + registers: KVec, + report_counts: [[u32; 2]; 2], + completion_masks: [u64; 2], + trace_printed: [u32; 2], + trace_journal: [KVVec; 2], + trace_dropped: [u32; 2], + uat: Arc, +} + +/// Read-back of the new 26A428 status cross-links (plan step 1.3 validation): +/// status B +0xe714 header, +0xe71c word and +0xe720 pointer; status config +/// +0x6c4 word and +0x6c8 pointer. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18StatusLinks { + pub(crate) status_b_header: u32, + pub(crate) status_b_word: u32, + pub(crate) status_b_pointer: u64, + pub(crate) status_config_word: u32, + pub(crate) status_config_pointer: u64, +} + +/// The status-B recovery handshake words. +#[derive(Debug, Copy, Clone, Default)] +pub(crate) struct G18RecoveryWord { + pub(crate) epoch: u64, + pub(crate) state: u32, + pub(crate) host: u32, +} + +/// What the firmware published about one recovery: the status-B cause words, +/// the valid recovery-info entries, and the shared recovery-request block. +#[derive(Debug, Copy, Clone, Default)] +pub(crate) struct G18RecoveryCause { + pub(crate) emitted: u32, + pub(crate) source_count: u32, + pub(crate) command: u32, + pub(crate) reason: u32, + pub(crate) detail: u64, + pub(crate) slot_mask: u64, + pub(crate) valid_entries: u32, + pub(crate) first_entry: Option<(u32, u64)>, + /// The first valid entries as (index, raw). Upstream's 13.5 layout for + /// the same 256 x 8 table (`pending_stamps`) is info u32 (bit 0 valid, + /// flags 3:0, stamp slot 10:4) then the awaited stamp value u32. + pub(crate) pending: [(u16, u64); G18_RECOVERY_PENDING_LOGGED], + pub(crate) request: [u32; init::G18G_26A428_RECOVERY_REQUEST_WORDS], +} + +pub(crate) const G18_RECOVERY_PENDING_LOGGED: usize = 8; + +impl G18FirmwareMemory { + /// Allocate, map and encode the complete G18G init-data graph while both + /// firmware processors are stopped. All fallible work happens before the + /// caller publishes anything. The caller keeps its own `Arc` of the UAT + /// until both processors stop, including when this construction fails. + #[inline(never)] + pub(crate) fn new_with_uat( + dev: &AsahiDrmDevice, + uat: Arc, + boot_timestamp: u64, + render: bool, + ) -> Result> { + if !uat.is_t8152_geometry() { + return Err(ENODEV); + } + match crate::g18::params::g18_compute_owner_test.value() { + 0 => {}, + 1 => G18UmaPoolMemory::verify_shared_compute_owner(dev, &uat)?, + _ => return Err(EINVAL), + } + let trace_journal = [ + KVVec::with_capacity(TRACE_JOURNAL_CAPACITY, GFP_KERNEL)?, + KVVec::with_capacity(TRACE_JOURNAL_CAPACITY, GFP_KERNEL)?, + ]; + let mut registers = KVec::new(); + for window in init::G18G_26A428_REGISTER_WINDOWS { + let (physical, virtual_address, span) = window.mapped_pages().map_err(|_| EINVAL)?; + // Flag 2 windows are firmware read-write; flag 0 windows are + // mapped read-only, the J813 rule (INFERRED for 26A428). + let protection = match window.flag { + 0 => mmu::PROT_FW_MMIO_RO, + 2 => mmu::PROT_FW_MMIO_RW, + _ => return Err(EINVAL), + }; + // Page-table writes only. No register contents are accessed. + registers.push( + uat.kernel_vm().map_io(dev.as_ref().as_ref(), virtual_address, physical.try_into()?, span.try_into()?, + protection)?, + GFP_KERNEL, + )?; + } + + let fixed = |address, size, protection| { + MappedObject::new_wc_at(dev, uat.kernel_vm(), address, size, protection) + }; + // Install every fixed address before allocator-selected objects. + let mut private = fixed(PRIVATE_BASE, init::G18G_26A428_PRIVATE_CLUSTER_SIZE, + mmu::PROT_FW_SHARED_RW)?; + // Both 26A428 roots are 0xc8 bytes. Separate mappings keep the empty + // page between them. + let mut primary_root = fixed(ROOTS_BASE, init::G18G_26A428_ROOT_SIZE, mmu::PROT_FW_SHARED_RO)?; + let mut secondary_root = fixed(ROOTS_BASE + init::SECONDARY_ROOT_DELTA, + init::G18G_26A428_ROOT_SIZE, mmu::PROT_FW_SHARED_RO)?; + let mut shared = fixed(SHARED_BASE, init::G18G_26A428_SHARED_CLUSTER_SIZE, mmu::PROT_FW_PRIV_RW)?; + let region_a = fixed(REGION_A_BASE, init::REGION_A_SIZE, mmu::PROT_FW_SHARED_RO)?; + let mut region_c = fixed(REGION_C_BASE, init::REGION_C_SIZE, mmu::PROT_FW_SHARED_RO)?; + let fwctl = fixed(FWCTL_BASE, FWCTL_SIZE, mmu::PROT_FW_SHARED_RW)?; + // Zero-backed at handoff; owned by later (M2/M3) publication. + let mut pbdesc = fixed(PBDESC_BASE, PBDESC_TABLE_SIZE, mmu::PROT_FW_SHARED_RW)?; + let mut uma = fixed(UMA_BASE, UMA_TABLE_SIZE, mmu::PROT_FW_SHARED_RW)?; + let mut aliases = KVec::new(); + for (object, address, size) in [ + (&mut pbdesc, init::REGION_VIEW_LOW_ADDRS[0], PBDESC_TABLE_SIZE), + (&mut uma, init::REGION_VIEW_LOW_ADDRS[1], UMA_TABLE_SIZE), + ] { + aliases.push(object.map_alias_at(uat.kernel_lower_vm(), address, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + for offset in (0..size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(object.iova() + offset as u64)? + != uat.kernel_lower_vm().translate_iova(address + offset as u64)? + { + return Err(EFAULT); + } + } + } + dev_info!(dev.as_ref(), "G18 UMA descriptor table: bytes={:#x} slots=256 stride=0x80 alias_pages={} verified\n", + UMA_TABLE_SIZE, UMA_TABLE_SIZE / mmu::UAT_PGSZ); + + let mut completions = KVec::new(); + let mut completion_bindings = + [init::G17PKsmCompletionAliases { low: 0, high: 0 }; init::M5_25G76_COMPLETION_QUEUE_COUNT]; + for (ordinal, binding) in completion_bindings.iter_mut().enumerate() { + // Host-chosen pairs, the J813 placement (INFERRED for 26A428). + let low = COMPLETION_LOW_BASE + ordinal as u64 * COMPLETION_STRIDE; + let high = COMPLETION_HIGH_BASE + ordinal as u64 * COMPLETION_STRIDE; + let mut backing = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, + init::M5_25G76_COMPLETION_BACKING_SIZE, mmu::PROT_FW_PRIV_RW)?; + aliases.push(backing.map_alias_at(uat.kernel_lower_vm(), low, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + for offset in (0..init::M5_25G76_COMPLETION_BACKING_SIZE).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset as u64)? + != uat.kernel_lower_vm().translate_iova(low + offset as u64)? + { + return Err(EFAULT); + } + } + *binding = init::G17PKsmCompletionAliases { low, high }; + completions.push(backing, GFP_KERNEL)?; + } + let dynamic = DYNAMIC_START..DYNAMIC_END; + let qos = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), dynamic.clone(), + init::G17P_QOS_RESOURCE_SIZE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)?; + let mut aon = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), dynamic, + init::G18G_26A428_AON_PTD_SIZE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_PRIV_RW)?; + let uma_pool = if render { G18UmaPoolMemory::new_render(dev, &uat)? } else { + G18UmaPoolMemory::new(dev, &uat, DYNAMIC_START..DYNAMIC_END, + crate::g18::hw::T8152_REGISTRY.num_cores)? + }; + let scheduler = G18SchedulerMemory::new(dev, &uat, DYNAMIC_START..DYNAMIC_END)?; + let facts = crate::g18::hw::T8152_REGISTRY; + let compute = G18ComputeMemory::new(dev, &uat, DYNAMIC_START..DYNAMIC_END, + facts.num_mgpus, facts.num_gps)?; + + aon.with_bytes_mut(|out| init::encode_g18g_26a428_aon_ptd(out).map_err(|_| EINVAL))?; + region_c.with_bytes_mut(|out| { + init::encode_region_c_for_profile(PROFILE, out).map_err(|_| EINVAL) + })?; + private.with_bytes_mut(|out| { + init::encode_g18g_26a428_private_cluster(PRIVATE_BASE, FWCTL_BASE, out).map_err(|_| EINVAL) + })?; + + let layout = init::G18G_26A428_PRIVATE_LAYOUT; + let addresses = init::SharedAddresses { + region_a: region_a.iova(), + region_c: region_c.iova(), + hw_data_bundle: SHARED_BASE, + hw_data_bundle_alloc: init::G18G_26A428_SHARED_CLUSTER_SIZE, + }; + let primary = init::InstanceAddresses { + root: ROOTS_BASE, + main_config: SHARED_BASE + init::G18G_26A428_MAIN_BUNDLE_OFFSETS[0] as u64, + status_a: PRIVATE_BASE + init::G18G_26A428_PRIVATE_STATUS_A[0] as u64, + status_b: PRIVATE_BASE + layout.status_b as u64, + secondary_extras: [0, 0], + }; + let secondary = init::InstanceAddresses { + root: ROOTS_BASE + init::SECONDARY_ROOT_DELTA, + main_config: SHARED_BASE + init::G18G_26A428_MAIN_BUNDLE_OFFSETS[1] as u64, + status_a: PRIVATE_BASE + init::G18G_26A428_PRIVATE_STATUS_A[1] as u64, + status_b: 0, + // Secondary root +0xb8 = status config, +0xc0 = the 0x60 object + // (KEXT_PROVEN, g18g_key.c:3748-3812). + secondary_extras: [ + PRIVATE_BASE + layout.status_config as u64, + PRIVATE_BASE + layout.secondary_extra_1 as u64, + ], + }; + init::validate_pair(&primary, &secondary).map_err(|_| EINVAL)?; + let views = init::PrimaryRegionViews { + sentinel_high: SENTINEL_BASE, + alias_high: [PBDESC_BASE, UMA_BASE], + }; + // GFX traces remain disabled here. Stage8 explicitly enables only + // re-derived b000 command/scheduler classes before root publication. + shared.with_bytes_mut(|out| { + init::encode_g18g_26a428_hw_data(completion_bindings, qos.iova(), + &mut out[..init::G18G_26A428_HW_DATA_SIZE]).map_err(|_| EINVAL)?; + init::encode_g18g_26a428_bundle_policy(init::M5PolicyBindings { + state: PRIVATE_BASE + layout.policy_state as u64, + aon_ptd: aon.iova(), + boot_timestamp, + }, out).map_err(|_| EINVAL)?; + for (role, instance, views) in [ + (init::InstanceRole::Primary, &primary, Some(&views)), + (init::InstanceRole::Secondary, &secondary, None), + ] { + let grid = PRIVATE_BASE + init::G18G_26A428_PRIVATE_STATE_GRIDS[role as usize] as u64; + let channels = init::derive_channel_table_for_profile(PROFILE, role, grid, + instance.status_a, instance.main_config, SHARED_BASE); + let offset = (instance.main_config - SHARED_BASE) as usize; + init::encode_main_config_for_profile(PROFILE, role, &addresses, instance, &channels, + views, &mut out[offset..offset + init::MAIN_CONFIG_SIZE]).map_err(|_| EINVAL)?; + } + Ok(()) + })?; + for (root, role, instance) in [ + (&mut primary_root, init::InstanceRole::Primary, &primary), + (&mut secondary_root, init::InstanceRole::Secondary, &secondary), + ] { + root.with_bytes_mut(|out| { + init::encode_root_for_profile(PROFILE, role, &addresses, instance, out) + .map_err(|_| EINVAL)?; + // The two firmware gates (version word, root+0x2c) must pass + // before anything is published. + init::check_firmware_root_gates_for_profile(PROFILE, out).map_err(|_| EINVAL) + })?; + } + // Unlike PB/UMA, the blank sentinel is unmapped at native handoff. + if uat.kernel_vm().translate_iova(SENTINEL_BASE).is_ok() { + return Err(EINVAL); + } + // Writes to all CPU WC mappings precede any future handoff. Only the + // two opening control records are staged; work rings stay empty. + fence(Ordering::SeqCst); + // Keep the retained graph off the kernel stack. + KBox::init(try_init!(Self { + primary_root, secondary_root, shared, private, region_a, region_c, fwctl, aon, qos, + pbdesc, uma, uma_pool, scheduler, compute, compute_payload: None, + application_compute: None, + completions, aliases, registers, + report_counts: [[0; 2]; 2], completion_masks: [0; 2], trace_printed: [0; 2], uat, + trace_journal, trace_dropped: [0; 2], + }), GFP_KERNEL) + } + + pub(crate) fn uma_pool(&self) -> &G18UmaPoolMemory { &self.uma_pool } + + pub(crate) fn prepare_render(&mut self, dev: &AsahiDrmDevice, + job: &mut crate::g18::runtime::graphics_job::G18GraphicsJob) -> Result { + self.uma_pool.prepare_render_backing(dev, &self.uat, job.vm())?; + let pool = self.uma_pool.prepare_binding()?; + self.uma_pool.load_hardware_descriptor(dev, &mut self.uma, pool, j873_sma_device_words(2, 6)?)?; + job.parameter_buffer().load_row(&mut self.pbdesc)?; + // Without a capture (stage 10 only) the first application frame encodes. + if !job.layout_only_payload() { job.encode_h1(dev)?; } + Ok(pool) + } + + pub(crate) fn audit_render(&mut self, dev: &AsahiDrmDevice, + job: &mut crate::g18::runtime::graphics_job::G18GraphicsJob) -> Result { + let pb = job.parameter_buffer().verify_before_kick(&mut self.pbdesc)?; + if pb.counter != 1 { return Err(EIO); } + let uma = self.uma_pool.verify_render_descriptor(&mut self.uma, j873_sma_device_words(2, 6)?)?; + let pages = self.uma_pool.audit_render_mappings(&self.uat, job.vm())?; + job.audit_h1(dev, &self.uat, &pb, &uma)?; + let state = self.uma_pool.render_state()?; + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=uma-state fnv={:#018x}\n", + crate::g18::fw::graphics_payload::fnv64(&state)); + dev_info!(dev.as_ref(), "G18_RENDER_AUDIT_UMA pages={} row=match state-slot=0\n", pages); + Ok(()) + } + + /// Batched CL_2 publication. Both queue records become visible through one + /// producer write, after every byte and every counter has been validated. + /// Used only by stage 9; the existing compute publication stays unchanged. + pub(crate) fn publish_outer_records(&mut self, records: &[[u8; 24]]) -> Result { + if records.len() != 2 || self.control_counters()? != [[2; 3], [1; 3]] { return Err(EINVAL); } + let before = self.compute_outer_counters()?; + let head = before[0]; + if before != [head; 3] || head > 6 || head & 1 != 0 { return Err(EBUSY); } + for (i, record) in records.iter().enumerate() { + let qid = record[22]; + let address = u64::from_le_bytes(record[8..16].try_into().map_err(|_| EINVAL)?); + if qid != if i == 0 {10} else {9} + || address != crate::g18::address_map::GRAPHICS_QUEUE_HIGH + if qid == 9 {0} else {0x100} + || record[16..20] != 2u32.to_le_bytes() || record[23] != 1 { return Err(EINVAL); } + } + self.shared.with_bytes_mut(|out| { + let start = init::G18G_26A428_WORK_RING_BUNDLE_OFFSETS[8] + head as usize * 24; + let slots = out.get_mut(start..start + 48).ok_or(EINVAL)?; + if slots.iter().any(|&b| b != 0) { return Err(EBUSY); } + for (out, record) in slots.chunks_exact_mut(24).zip(records) { out.copy_from_slice(record); } + Ok(()) + })?; + // No producer moved while the batch was being stored. + if self.compute_outer_counters()? != before { return Err(EBUSY); } + fence(Ordering::SeqCst); + let address = PRIVATE_BASE + (init::G18G_26A428_PRIVATE_STATE_GRIDS[0] + + init::WORK_STATE_GRID_OFFSETS[8] + 0x20) as u64; + self.private.write_u32(address, head + 2)?; + fence(Ordering::SeqCst); + Ok(head + 2) + } + + pub(crate) fn scheduler(&mut self) -> &mut G18SchedulerMemory { &mut self.scheduler } + + pub(crate) fn compute(&mut self) -> &mut G18ComputeMemory { &mut self.compute } + + /// Select the firmware-owned KSMFE queue path before root publication. + /// b000 0x216f8..0x21700 forms x10=global+0x1093e0, then 0x21884 + /// copies hwdata+0x2724 to [x10+0xb7c] (global+0x109f5c); + /// 0x12cdc..0x12de8 skips tag15 hardware queue setup when it is nonzero. + /// Native macOS uses host queue programming. This first Linux submission + /// sends tag15/tag14 and performs no host KSMFE MMIO, so choose one owner. + pub(crate) fn use_firmware_queue_programming(&mut self, dev: &AsahiDrmDevice) -> Result { + let policy = SHARED_BASE + 0x2724; + let trace = PRIVATE_BASE + init::G18G_26A428_PRIVATE_STATUS_A[0] as u64; + let progress = REGION_C_BASE + 0x9ac; + if self.shared.read_u32(policy)? != 1 || self.private.read_u32(trace)? != 0 + || self.region_c.read_u32(progress)? != 0x333 { + return Err(EINVAL); + } + self.shared.write_u32(policy, 0)?; + // b000 0x216b8 obtains GFX status-A; 0x3508 loads its first word + // into global 0x1093e4. Bit0 gates commands/completion; bit1 gates + // scheduling/recovery. audit-g18-trace-mask-native.py executes these + // exact branches. Opt-in graphics diagnosis additionally enables + // bit9 (global 0x1093e5 bit1), the native stall diagnostic category. + // audit-g18-graphics-stall.py executes that original predicate. + let graphics_stall_trace = crate::g18::params::g18_graphics_stall_trace.value() == 1; + let trace_mask = if graphics_stall_trace { 0x203 } else { 3 }; + self.private.write_u32(trace, trace_mask)?; + // b000 refresh 0x352c/0x3688 copies region-C+0x9ac into + // global 0x109488. Its compute nibble (bits8..11) selects native + // progress sampling: mode3 only checks busy; mode2 takes the + // detailed sampler. TA (nibble 0) also takes mode2: b000 0x890c + // then enters the original sampler at 0x8924 instead of the + // busy-bit-only path. The busy-only path recovers any TA busy for + // region-C+0x9a0 (=0xa) one-tick scans, ~12.8 ms after the kick, + // which legitimate single-invocation vertex/geometry work reaches + // (image-op-rootcause-1113). Mode2 still recovered a real TA hang + // (linux-workloads-123, ~32 ms). Keep 3D and all timeout bounds. + let progress_policy = 0x232; + self.region_c.write_u32(progress, progress_policy)?; + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), + "G18_QUEUE_OWNER firmware: hwdata+0x2724=0 before root publication; no host KSMFE writes\n"); + dev_info!(dev.as_ref(), "G18_TRACE_MASK GFX status-A={:#x} before root publication\n", trace_mask); + dev_info!(dev.as_ref(), "G18_PROGRESS_POLICY region-C+0x9ac={:#x} before publication; timeout bounds unchanged\n", progress_policy); + Ok(()) + } + + /// Retain a private shader/input/output VM. This does not select a process + /// context or make its roots visible to firmware, and never queues work. + pub(crate) fn prepare_compute_payload(&mut self, dev: &AsahiDrmDevice) + -> Result<&mut G18ComputePayloadMemory> { + if self.compute_payload.is_some() { return Err(EBUSY); } + self.compute_payload = Some(G18ComputePayloadMemory::load(dev, &self.uat)?); + Ok(self.compute_payload.as_mut().unwrap()) + } + + /// Connect the retained payload VM to the command/channel owner. Context + /// roots remain unpublished until the complete command has been prepared. + pub(crate) fn bind_compute_process(&mut self, dev: &AsahiDrmDevice) -> Result { + let payload = self.compute_payload.as_ref().ok_or(EINVAL)?; + self.compute.bind_process(dev, &self.uat, payload.vm()) + } + + /// Allocate the captured job's scratch pool while its state is unpublished. + /// The returned register inputs are valid only with this retained owner. + pub(crate) fn prepare_compute_scratch(&mut self, dev: &AsahiDrmDevice) + -> Result { + let plan = self.compute_payload.as_ref().ok_or(EINVAL)?.scratch_plan()?; + self.uma_pool.prepare_compute_backing(dev, &self.uat, DYNAMIC_START..DYNAMIC_END, plan)?; + Ok(plan) + } + + /// Assemble the single captured addition with retained Linux resources. + /// Every step is still unpublished: the caller must subsequently bind the + /// pool, publish the process roots, encode the owned SKSM entry and only + /// then publish a kick. Errors leave any partial owners retained here. + pub(crate) fn prepare_addition(&mut self, dev: &AsahiDrmDevice) + -> Result<(G18PreparedComputeCommand, G18UmaPoolBinding)> { + use crate::g18::fw::compute::G18ComputeCompletionState; + use crate::g18::queue::probe::QUEUE_ID; + + let (mut command, mut registers) = self.prepare_compute_payload(dev)?.addition_inputs()?; + let context_id = self.bind_compute_process(dev)?; + let scratch = self.prepare_compute_scratch(dev)?; + // The hardware reads the pool through the job's context (stage27 + // GMMU fault in vm_slot 3), so the pool must be mapped in that VM. + let payload_vm = self.compute_payload.as_ref().ok_or(EINVAL)?.vm(); + self.uma_pool.alias_into_process(dev, &self.uat, payload_vm)?; + let pool = self.prepare_uma_pool_binding()?; + // Native prepareLocked loads the hardware descriptor for the same slot + // before the submit's tag15 names the pool; bind_pool publishes later. + let sma = j873_sma_device_words(registers.clusters, registers.cluster_width)?; + self.uma_pool.load_hardware_descriptor(dev, &mut self.uma, pool, sma)?; + let scheduler = self.scheduler.prepare(0, 1)?; + + // One fresh CL queue, first work ordinal 0, hardware stamp 1. Native + // event-machine conversion with 256 entries and base 1 yields 1 for + // event stamp 0x100; no previous completion is imported from macOS. + command.sequence = 1; + command.d100 = 1; + command.d7b8 = pool.state; + command.completion = G18ComputeCompletionState { + stamp: 0x100, index: u32::from(QUEUE_ID), entries: 256, + generation: 0, base: 1, + }; + registers.context_id = context_id; + registers.pool_slot = pool.slot; + // The passing fixture ran with a zero SMA result (0x10241 = slot). + // The pool binding generation is the separate FList field (tag15). + registers.sma_units = 0; + registers.d550 = scratch.register_pair[0]; + registers.d558 = scratch.register_pair[1]; + let prepared = self.compute.prepare(scheduler, &command, ®isters)?; + dev_info!(dev.as_ref(), + "G18 addition prepared: command={:#x}/{:#x} context={} scheduler={:#x} stamp={:#x} pool={}/{} registers={:?}; unpublished\n", + prepared.gpu_address, prepared.firmware_address, context_id, + scheduler.record, prepared.expected_stamp, pool.slot, pool.generation, + prepared.registers); + Ok((prepared, pool)) + } + + pub(crate) fn addition_output(&mut self) -> Result { + self.compute_payload.as_mut().ok_or(EINVAL)?.output_snapshot() + } + + pub(crate) fn triangle_output(&mut self) -> Result { + self.compute_payload.as_mut().ok_or(EINVAL)?.triangle_output_snapshot() + } + + pub(crate) fn prepare_uma_pool_binding(&mut self) -> Result { + self.uma_pool.prepare_binding() + } + + /// FW VA of one role's root, for the EP 0x20 0x81 doorbell. + pub(crate) fn root(&self, secondary: bool) -> u64 { + if secondary { self.secondary_root.iova() } else { self.primary_root.iova() } + } + + pub(crate) fn compute_outer_counters(&mut self) -> Result<[u32; 3]> { + let base = PRIVATE_BASE + (init::G18G_26A428_PRIVATE_STATE_GRIDS[0] + + init::WORK_STATE_GRID_OFFSETS[8]) as u64; + Ok([self.private.read_u32(base)?, self.private.read_u32(base + 0x10)?, + self.private.read_u32(base + 0x20)?]) + } + + /// G18 native 24-byte CL_2 record, limited to the registration and + /// scheduler-bind records. Each new producer is published exactly once. + pub(crate) fn publish_queue_probe(&mut self, address: u64, queue_id: u8, producer: u32) -> Result { + if !(1..=3).contains(&producer) { return Err(EINVAL); } + self.publish_compute_outer(address, queue_id, producer, producer) + } + + /// After the three setup records, publish one outer item covering the + /// command, context configuration and kick (inner producer 6). + pub(crate) fn publish_compute_queue(&mut self, address: u64, queue_id: u8) -> Result { + self.publish_compute_outer(address, queue_id, 4, 6) + } + + fn publish_compute_outer(&mut self, address: u64, queue_id: u8, + producer: u32, inner_producer: u32) -> Result { + if address < KERNEL_BASE || address & 0x3f != 0 || queue_id > 127 + || !(1..=4).contains(&producer) { + return Err(EINVAL); + } + if self.compute_outer_counters()? != [producer - 1; 3] + || self.control_counters()? != [[2; 3], [1; 3]] { + return Err(EBUSY); + } + self.shared.with_bytes_mut(|out| { + let start = init::G18G_26A428_WORK_RING_BUNDLE_OFFSETS[8] + (producer as usize - 1) * 24; + let record = out.get_mut(start..start + 24).ok_or(EINVAL)?; + if record.iter().any(|byte| *byte != 0) { return Err(EBUSY); } + record[8..16].copy_from_slice(&address.to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&(inner_producer as u16).to_le_bytes()); + record[22] = queue_id; + record[23] = 1; + Ok(()) + })?; + fence(Ordering::SeqCst); + let producer_address = PRIVATE_BASE + (init::G18G_26A428_PRIVATE_STATE_GRIDS[0] + + init::WORK_STATE_GRID_OFFSETS[8] + 0x20) as u64; + self.private.write_u32(producer_address, producer)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// The live device-control counters (consumer, CFI, producer) of both + /// roles, read from the shared state the firmware consumes. + pub(crate) fn control_counters(&mut self) -> Result<[[u32; 3]; 2]> { + self.private.with_bytes_mut(|bytes| { + let mut counters = [[0; 3]; 2]; + for (role, grid) in init::G18G_26A428_PRIVATE_STATE_GRIDS.into_iter().enumerate() { + let base = grid + init::WORK_STATE_GRID_OFFSETS[init::CONTROL_CHANNEL_INDEX]; + for (index, value) in counters[role].iter_mut().enumerate() { + let offset = base + index * init::CHANNEL_STATE_SPACING; + let word = bytes.get(offset..offset + 4).ok_or(EINVAL)?; + // SAFETY: the grid offsets are u32 aligned inside this + // bounded WC mapping, which is retained while firmware runs. + *value = unsafe { core::ptr::read_volatile(word.as_ptr().cast::()) }; + } + } + fence(Ordering::Acquire); + Ok(counters) + }) + } + + /// Stage device-control record 1 of role 0, IdlePowerOff(0) (opcode 0x0a, + /// FW_PROVEN unchanged: b000 0x4ef0), and publish it. Only valid after + /// both opening records retired, i.e. every counter reads 1. + pub(crate) fn publish_idle_wake(&mut self) -> Result { + if self.control_counters()? != [[1; 3]; 2] { + return Err(EBUSY); + } + let record = init::G18G_26A428_CONTROL_RING_BUNDLE_OFFSETS[0] + init::CONTROL_RECORD_SIZE; + self.shared.with_bytes_mut(|out| { + let record = out.get_mut(record..record + init::CONTROL_RECORD_SIZE).ok_or(EINVAL)?; + if record.iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + init::encode_device_control_record( + init::G18G_26A428_DEVICE_CONTROL_OPCODE_IDLE_POWER_OFF, + init::DEVICE_CONTROL_IDLE_POWER_OFF_ARG_DISABLE_IDLE_AND_WAKE, + record, + ) + .map_err(|_| EINVAL) + })?; + fence(Ordering::SeqCst); + let producer = init::G18G_26A428_PRIVATE_STATE_GRIDS[0] + + init::WORK_STATE_GRID_OFFSETS[init::CONTROL_CHANNEL_INDEX] + + init::CHANNEL_STATE_PRODUCER * init::CHANNEL_STATE_SPACING; + self.private.write_u32(PRIVATE_BASE + producer as u64, init::CONTROL_OPENING_PRIMARY_PRODUCER + 1)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// The four firmware-owned KSM completion cursors (hw_data +0x2734 + + /// ordinal * 0x20 + 0x1c; FW_PROVEN b000 0x1cbc0..0x1cc48). + pub(crate) fn completion_cursors(&mut self) -> Result<[u32; init::M5_25G76_COMPLETION_QUEUE_COUNT]> { + let mut cursors = [0; init::M5_25G76_COMPLETION_QUEUE_COUNT]; + for (ordinal, cursor) in cursors.iter_mut().enumerate() { + *cursor = self.shared.read_u32( + SHARED_BASE + + (init::G18G_26A428_KSM_COMPLETION_TABLE_OFFSET + + ordinal * init::G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE + + init::G18G_26A428_KSM_COMPLETION_CURSOR) as u64, + )?; + } + Ok(cursors) + } + + /// Current recovery handshake words. Host DRAM only; no SGX MMIO. + pub(crate) fn recovery_word(&mut self) -> Result { + let status_b = PRIVATE_BASE + init::G18G_26A428_PRIVATE_LAYOUT.status_b as u64; + fence(Ordering::Acquire); + Ok(G18RecoveryWord { + epoch: self.private.read_u64(status_b + init::G18G_26A428_STATUS_B_RECOVERY_EPOCH as u64)?, + state: self.private.read_u32(status_b + init::G18G_26A428_STATUS_B_RECOVERY_STATE as u64)?, + host: self.private.read_u32(status_b + init::G18G_26A428_STATUS_B_RECOVERY_HOST as u64)?, + }) + } + + /// The cause and info the firmware published for the current recovery. + /// Like macOS, count entries by their valid bit rather than trusting the + /// emitted count. + pub(crate) fn recovery_cause(&mut self) -> Result { + let status_b = PRIVATE_BASE + init::G18G_26A428_PRIVATE_LAYOUT.status_b as u64; + fence(Ordering::Acquire); + let word = |memory: &mut Self, offset: usize| memory.private.read_u32(status_b + offset as u64); + let mut cause = G18RecoveryCause { + emitted: word(self, init::G18G_26A428_STATUS_B_RECOVERY_EMITTED)?, + source_count: word(self, init::G18G_26A428_STATUS_B_RECOVERY_SOURCE_COUNT)?, + command: word(self, init::G18G_26A428_STATUS_B_RECOVERY_COMMAND)?, + reason: word(self, init::G18G_26A428_STATUS_B_RECOVERY_REASON)?, + detail: self.private.read_u64(status_b + init::G18G_26A428_STATUS_B_RECOVERY_DETAIL as u64)?, + slot_mask: self.private.read_u64(status_b + init::G18G_26A428_STATUS_B_RECOVERY_SLOT_MASK as u64)?, + ..Default::default() + }; + for index in 0..init::G18G_26A428_STATUS_B_RECOVERY_INFO_ENTRIES { + let entry = self.private.read_u64(status_b + + (init::G18G_26A428_STATUS_B_RECOVERY_INFO + index * 8) as u64)?; + if entry & 1 != 0 { + if let Some(slot) = cause.pending.get_mut(cause.valid_entries as usize) { + *slot = (index as u16, entry); + } + cause.valid_entries += 1; + cause.first_entry.get_or_insert((index as u32, entry)); + } + } + let request = SHARED_BASE + init::G18G_26A428_SHARED_LAYOUT.block_68 as u64; + for (index, value) in cause.request.iter_mut().enumerate() { + *value = self.shared.read_u32(request + (index * 4) as u64)?; + } + Ok(cause) + } + + /// One QID's last-submitted entry: (valid, stamp). + pub(crate) fn queue_progress(&mut self, qid: usize) -> Result<(u32, u64)> { + if qid >= init::G18G_26A428_QUEUE_PROGRESS_ENTRIES { + return Err(EINVAL); + } + let entry = SHARED_BASE + (init::G18G_26A428_SHARED_LAYOUT.block_800 + + qid * init::G18G_26A428_QUEUE_PROGRESS_STRIDE) as u64; + fence(Ordering::Acquire); + Ok((self.shared.read_u32(entry)?, self.shared.read_u64(entry + 8)?)) + } + + /// Publish one QID's last-submitted entry for the firmware's restart. + pub(crate) fn publish_queue_progress(&mut self, qid: usize, valid: u32, stamp: u64) -> Result { + if qid >= init::G18G_26A428_QUEUE_PROGRESS_ENTRIES { + return Err(EINVAL); + } + let entry = SHARED_BASE + (init::G18G_26A428_SHARED_LAYOUT.block_800 + + qid * init::G18G_26A428_QUEUE_PROGRESS_STRIDE) as u64; + self.shared.write_u32(entry, valid)?; + self.shared.write_u32(entry + 4, 0)?; + self.shared.write_u32(entry + 8, stamp as u32)?; + self.shared.write_u32(entry + 12, (stamp >> 32) as u32)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Take the host's half of one handshake edge: 1 -> 2 lets the halted + /// firmware recover, 3 -> 0 is the final clear. Any other edge, or a + /// state other than `expected`, is refused without writing. + pub(crate) fn acknowledge_recovery(&mut self, expected: u32, next: u32) -> Result { + if !matches!((expected, next), (1, 2) | (3, 0)) { + return Err(EINVAL); + } + let state = PRIVATE_BASE + (init::G18G_26A428_PRIVATE_LAYOUT.status_b + + init::G18G_26A428_STATUS_B_RECOVERY_STATE) as u64; + fence(Ordering::Acquire); + if self.private.read_u32(state)? != expected { + return Err(EBUSY); + } + fence(Ordering::SeqCst); + self.private.write_u32(state, next)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Read back the new status cross-links from the host mapping (plan step + /// 1.3 validation after the 0x81 ACKs). + pub(crate) fn status_links(&mut self) -> Result { + let layout = init::G18G_26A428_PRIVATE_LAYOUT; + let status_b = PRIVATE_BASE + layout.status_b as u64; + let config = PRIVATE_BASE + layout.status_config as u64; + Ok(G18StatusLinks { + status_b_header: self.private.read_u32(status_b + init::G18G_26A428_STATUS_B_CONFIG_HEADER as u64)?, + status_b_word: self.private.read_u32(status_b + init::G18G_26A428_STATUS_B_LINK_WORD as u64)?, + status_b_pointer: self.private.read_u64(status_b + init::G18G_26A428_STATUS_B_LINK_PTR as u64)?, + status_config_word: self.private.read_u32(config + init::G18G_26A428_STATUS_CONFIG_LINK_WORD as u64)?, + status_config_pointer: self.private.read_u64(config + init::G18G_26A428_STATUS_CONFIG_LINK_PTR as u64)?, + }) + } + + /// Drain both roles' event (channel 13) and trace (channel 14) report + /// rings. Records are copied before their slot is released; each drain is + /// bounded. Returns whether an event record of the J813 recovery type was + /// seen. This is the milestone-1 heartbeat (plan step 1.8, stage 4). + pub(crate) fn drain_reports(&mut self, dev: &AsahiDrmDevice) -> Result { + let mut recovery = false; + for (role, status_a) in init::G18G_26A428_PRIVATE_STATUS_A.into_iter().enumerate() { + for channel in 0..2 { + let [cursor_offset, ring_offset, _] = init::REPORT_STATE_STATUS_A_OFFSETS[channel]; + let capacity = REPORT_CAPACITY[channel]; + let cursor = PRIVATE_BASE + (status_a + cursor_offset) as u64; + let ring = status_a + ring_offset; + for _ in 0..REPORT_DRAIN_BUDGET { + let consumer = self.private.read_u32(cursor)?; + let producer = self.private.read_u32(cursor + REPORT_PRODUCER_OFFSET)?; + if consumer >= capacity || producer >= capacity { + return Err(EIO); + } + if consumer == producer { + break; + } + fence(Ordering::Acquire); + let offset = ring + consumer as usize * REPORT_RECORD_SIZE; + let mut raw = [0u8; REPORT_RECORD_SIZE]; + self.private.with_bytes_mut(|bytes| { + raw.copy_from_slice(bytes.get(offset..offset + REPORT_RECORD_SIZE).ok_or(EINVAL)?); + Ok(()) + })?; + let word = |off: usize| u32::from_le_bytes([raw[off], raw[off + 1], raw[off + 2], raw[off + 3]]); + let qword = |off: usize| u64::from(word(off)) | (u64::from(word(off + 4)) << 32); + let count = self.report_counts[role][channel]; + if channel == 1 { + // Preserve all bytes before firmware may reuse this slot. + // No allocation, MMIO, or extra logging in the hot path. + if self.trace_journal[role].len() >= TRACE_JOURNAL_CAPACITY + || self.trace_journal[role] + .push_within_capacity(G18TraceRecord { ordinal: count, raw }) + .is_err() + { + self.trace_dropped[role] = self.trace_dropped[role].saturating_add(1); + } + } + if channel == 0 { + crate::g18_event_info!(role == 0 && word(0) == 1, dev.as_ref(), "G18_EVENT role={} ordinal={} type={} payload={:x?}\n", + role, count, word(0), [qword(4), qword(12), qword(20), qword(28), qword(36)]); + recovery |= word(0) == EVENT_TYPE_RECOVERY; + if role == 0 && word(0) == 1 { + // Original G18 b000 233a8..233d4 uses this exact + // type-1 prefix. Copy/validate before releasing the + // slot; the remaining bytes are not completion proof. + let event = crate::g18::fw::events::decode_g17p_firmware_stamp_event(&raw) + .map_err(|_| EIO)?; + self.completion_masks[0] |= event.masks[0]; + self.completion_masks[1] |= event.masks[1]; + } + } else if self.trace_printed[role] < TRACE_PRINT_BUDGET[role] { + dev_info!(dev.as_ref(), + "G18_TRACE role={} ordinal={} timestamp={} type={} id={:#x} args={:x?}\n", + role, count, qword(4), word(0), word(0x2c), + [qword(12), qword(20), qword(28), qword(36)]); + self.trace_printed[role] += 1; + } + self.report_counts[role][channel] = count.saturating_add(1); + fence(Ordering::SeqCst); + self.private.write_u32(cursor, (consumer + 1) % capacity)?; + } + } + } + Ok(recovery) + } + + /// Report-record totals per role and channel since construction. + pub(crate) fn report_counts(&self) -> [[u32; 2]; 2] { + self.report_counts + } + + /// Only primary firmware type-1 records contribute to these fresh-boot bits. + pub(crate) fn completion_masks(&self) -> [u64; 2] { + self.completion_masks + } + + /// Export after the transaction and stop requests, never while polling a + /// GPU job. Completeness covers consumed records, not a possibly live producer. + pub(crate) fn log_trace_journal(&self, dev: &AsahiDrmDevice) { + for role in 0..2 { + let journal = &self.trace_journal[role]; + let consumed = self.report_counts[role][1]; + dev_info!(dev.as_ref(), + "G18_TRACE_JOURNAL_BEGIN role={} consumed={} stored={} dropped={} capacity={}\n", + role, consumed, journal.len(), self.trace_dropped[role], TRACE_JOURNAL_CAPACITY); + for record in journal.iter() { + let words: [u64; 9] = core::array::from_fn(|index| { + let mut bytes = [0; 8]; + bytes.copy_from_slice(&record.raw[index * 8..index * 8 + 8]); + u64::from_le_bytes(bytes) + }); + dev_info!(dev.as_ref(), "G18_TRACE_JOURNAL role={} ordinal={} words={:016x?}\n", + role, record.ordinal, words); + } + dev_info!(dev.as_ref(), + "G18_TRACE_JOURNAL_END role={} consumed={} stored={} dropped={}\n", + role, consumed, journal.len(), self.trace_dropped[role]); + } + } + + /// One-line construction summary for the bring-up log. + pub(crate) fn log_summary(&self, dev: &AsahiDrmDevice) { + dev_info!(dev.as_ref(), + "G18G initdata: roots {:#x}/{:#x} version {:#x}; shared {:#x}+{:#x}; private {:#x}+{:#x}; {} register windows; {} unproven fields\n", + self.root(false), self.root(true), init::G18G_26A428_INITDATA_VERSION_WORD, + SHARED_BASE, init::G18G_26A428_SHARED_CLUSTER_SIZE, + PRIVATE_BASE, init::G18G_26A428_PRIVATE_CLUSTER_SIZE, + self.registers.len(), init::G18G_26A428_UNPROVEN.len()); + } +} diff --git a/drivers/gpu/drm/asahi/g18/mod.rs b/drivers/gpu/drm/asahi/g18/mod.rs new file mode 100644 index 0000000000000..23e6f2a76a62c --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/mod.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 (T8152, Mac mini M6 / J873) backend. +//! +//! Probe, firmware start and DRM registration follow the J873 module 1134 +//! path; the backend plugs into the shared driver through `gpu::Gpu` and +//! `queue::Queue`. + +#[allow(dead_code)] +pub(crate) mod address_map; +#[allow(dead_code)] +pub(crate) mod boot; +pub(crate) mod buffer; +pub(crate) mod compat; +#[allow(dead_code)] +pub(crate) mod drm; +pub(crate) mod fixture; +pub(crate) mod fw; +#[allow(dead_code)] +pub(crate) mod initdata; +pub(crate) mod params; +pub(crate) mod queue; +pub(crate) mod runtime; +pub(crate) mod uapi; +#[allow(dead_code)] +pub(crate) mod debug; +#[allow(dead_code)] +pub(crate) mod hw; +#[allow(dead_code)] +pub(crate) mod identity; +#[allow(dead_code)] +pub(crate) mod probe; +#[allow(dead_code)] +pub(crate) mod regs; diff --git a/drivers/gpu/drm/asahi/g18/params.rs b/drivers/gpu/drm/asahi/g18/params.rs new file mode 100644 index 0000000000000..555c9959ead2f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/params.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Module parameters of the G18 backend. +//! +//! They are registered exactly as `kernel::module!` registers the driver's +//! own parameters (a `__param` entry with `perm` 0 and the `parmtype`/`parm` +//! modinfo strings, prefixed by `aurora.` when built in), but only when the +//! backend is built, so other configurations keep their parameter set. The +//! names, types, defaults and descriptions are those of the J873 kernel. + +#![allow(non_upper_case_globals)] + +use kernel::module_param::{ModuleParamAccess, PARAM_OPS_U32, PARAM_OPS_U64}; + +/// Copy `bytes` into an array of its exact length, for a `.modinfo` string. +const fn modinfo(bytes: &[u8]) -> [u8; N] { + let mut out = [0; N]; + let mut i = 0; + while i < N { + out[i] = bytes[i]; + i += 1; + } + out +} + +macro_rules! g18_param { + ($name:ident: $ty:ident = $default:expr, $ops:ident, $description:literal) => { + pub(crate) static $name: ModuleParamAccess<$ty> = ModuleParamAccess::new($default); + + const _: () = { + #[link_section = "__param"] + #[used(compiler)] + static PARAM: kernel::module_param::KernelParam = + kernel::module_param::KernelParam::new(kernel::bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + kernel::c_str!(stringify!($name)) + } else { + kernel::c_str!(concat!("aurora.", stringify!($name))) + }), + // `__this_module` is constructed by the kernel at load time and is not + // freed until the module is unloaded. A raw address avoids a reference to + // the mutable extern static. + #[cfg(MODULE)] + mod_: &raw mut kernel::bindings::__this_module, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&$ops), + perm: 0, // Will not appear in sysfs + level: -1, + flags: 0, + __bindgen_anon_1: kernel::bindings::kernel_param__bindgen_ty_1 { + arg: $name.as_void_ptr(), + }, + }); + + #[cfg(MODULE)] + const TYPE: &[u8] = + concat!("parmtype=", stringify!($name), ":", stringify!($ty), "\0").as_bytes(); + #[cfg(not(MODULE))] + const TYPE: &[u8] = + concat!("aurora.parmtype=", stringify!($name), ":", stringify!($ty), "\0") + .as_bytes(); + #[cfg_attr(not(target_os = "macos"), link_section = ".modinfo")] + #[used(compiler)] + static TYPE_INFO: [u8; TYPE.len()] = modinfo(TYPE); + + #[cfg(MODULE)] + const TEXT: &[u8] = + concat!("parm=", stringify!($name), ":", $description, "\0").as_bytes(); + #[cfg(not(MODULE))] + const TEXT: &[u8] = + concat!("aurora.parm=", stringify!($name), ":", $description, "\0").as_bytes(); + #[cfg_attr(not(target_os = "macos"), link_section = ".modinfo")] + #[used(compiler)] + static TEXT_INFO: [u8; TEXT.len()] = modinfo(TEXT); + }; + }; +} + +g18_param!(g18_initdata_test: u32 = 10, PARAM_OPS_U32, + "T8152/J873 G18G bring-up stage. 0 refuses before MMIO/DMA. 1=read-only powered-resource probe. 2=RTKit. 3=initdata ACKs. 4=openings, idle policy, 60 s observation. 5=empty queue. 6=queue plus scheduler binding. 7=owned empty UMA pool binding. 8=compute diagnostic. 9=H1 render replay. 10=firmware plus application DRM render node. Stages 5/6/7 then observe without GPU jobs. Firmware stages require prior power, loader receipt and apple_mailbox.ascwrap_v8_cpu_control=1; all owners remain retained until reboot. 90=SGX power-on; 92=GFX1 power-on; 91=GFX power-on; 93=idle requests. Power stages are never implied. Mutating stages require user-authorized bring-up. Other values refused"); +g18_param!(g18_compute_variant: u32 = 0, PARAM_OPS_U32, + "Stage 8 compute diagnostic. 0=captured compute-add. 1=empty stream: captured barrier and terminate replace the launch; all output must remain poisoned. 2=64x64 compute triangle using a separately verified private shader patch; no TA/3D or Mesa. Variants 1 and 2 require g18_initdata_test=8. Other values are refused"); +g18_param!(g18_compute_owner_test: u32 = 0, PARAM_OPS_U32, + "J873 allocation-only E2E before firmware publication: 1 verifies shared-compute backing, guarded directory growth, allocator rollback and process aliases. Never submits GPU work. Other nonzero values refused"); +g18_param!(g18_render_variant: u32 = 0, PARAM_OPS_U32, + "Stage 9 H1 diagnostic: 0=colour, 2=terminated VDM plus background clear. Other variants refused before MMIO. One asynchronous worker pass, captured/M5 H1 policy, all owners retained; see the H1 render-runtime notes"); +g18_param!(g18_render_profile: u32 = 1, PARAM_OPS_U32, + "Stage 9 G18GFX01 v2 profile: 1=colour-a, 2=colour-b, 4=fragment-witness (colour only). Gears profile3 is offline only and refused by H1. Every other stage, including the default stage10, requires 1"); +g18_param!(g18_admit_compute: u32 = 1, PARAM_OPS_U32, + "G18 DRM adapter: 1 admits single-dispatch compute commands (K1). 0 refuses them"); +g18_param!(g18_admit_msaa4: u32 = 1, PARAM_OPS_U32, + "G18 DRM adapter (g18_initdata_test=10 only): 1 (default) admits exactly the native 4x MSAA render (4 samples, 32x16 utiles, sample positions 0xeaa26e26, 8-byte samples, one layer, no depth/stencil) with its native 4x command fields; Mesa needs AGX_G18_MSAA4=1 to submit it. 0 refuses every multisampled render"); +g18_param!(g18_admit_graphics_scratch: u32 = 1, PARAM_OPS_U32, + "G18 DRM adapter: 1 (default) advertises DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH and admits single-sample renders carrying the G18 render resource suffix; the kernel derives both TA/3D private-memory launch pairs and selects the native ideal or minimum result that fits the owned render pool. 0 refuses the suffix"); +g18_param!(g18_usc_statistics: u32 = 0, PARAM_OPS_U32, + "Stage 9 colour fixture only: 1 enables owned TA/3D SMAPP statistics pages. Raw memory statistics, not shader invocation counts. Default 0"); +g18_param!(g18_sma_minimum: u32 = 1, PARAM_OPS_U32, + "Stage9/profile4 or stage10/profile1, variant0: 1 probes fixed native minimum TA/fragment resource words with unchanged owned backing. Default 1"); +g18_param!(g18_graphics_stall_trace: u32 = 0, PARAM_OPS_U32, + "Stage9/profile4 or stage10/profile1, variant0: 1 enables firmware category9 diagnostics and the native TA mode2 progress sampler. No timeout changes. Default 0"); +g18_param!(g18_kick_log: u32 = 0, PARAM_OPS_U32, + "Stage10 application DRM log volume: 0 (default) omits the per-kick, frame-pass and per-job resource lines (G18_APP_KICK, G18_APP_FRAME_PASS, G18_APP_COMPUTE_KNOBS, G18_COMPUTE_*, G18_SHARED_COMPUTE_*, G18 UMA/compute-context lines, G18_POOL_DESCRIPTOR_CURSOR, role-0 type-1 G18_EVENT) once G18_DRM_READY is logged; 1 prints them as before. Startup, fault, recovery, refusal, wait, stall, admission and sampled summary lines are always printed"); +g18_param!(g18_vdm_high_alias: u32 = 0, PARAM_OPS_U32, + "Stage9/profile4/variant0 with minimum SMA and stall tracing: 1 fetches unchanged native VDM bytes through the Gallium-address alias. Diagnostic only. Default 0"); +g18_param!(g18_ppp_high_alias: u32 = 0, PARAM_OPS_U32, + "With the stage9 high-VDM control: 1 aliases unchanged native PPP pages into the Gallium range and relocates only their VDM pointers. Diagnostic only. Default 0"); +g18_param!(g18_sma_ideal: u32 = 0, PARAM_OPS_U32, + "Stage9 variant0 profile4 only: 1 probes native ideal graphics resource policy with 40MiB backing and precommitted directory growth. Default 0"); +g18_param!(g18_native_barrier: u32 = 0, PARAM_OPS_U32, + "Stage9 variant0 profile4 with ideal SMA only: 1 probes the native same-group TA-to-3D barrier kind3. Default 0"); +g18_param!(g17p_simplefb_iova: u64 = 0, PARAM_OPS_U64, + "Diagnostic simplefb GPU address override; 0 disables it"); +g18_param!(g17p_fw_log: u32 = 0, PARAM_OPS_U32, + "Enable the GPU firmware log ring"); +g18_param!(g17p_frg_hwdsid_clear: u32 = 0, PARAM_OPS_U32, + "Diagnostic: 1 writes FRG descriptor +0x222d as 0 (native light captures); 0 preserves 1"); +g18_param!(g17p_trace_depth: u32 = 0, PARAM_OPS_U32, + "Trace the first N compute submissions per queue; 0 disables success-path tracing"); +g18_param!(g17p_completion_trace: u32 = 0, PARAM_OPS_U32, + "Completion logging level: 0 off, 1 summary, 2 detailed"); diff --git a/drivers/gpu/drm/asahi/g18/probe.rs b/drivers/gpu/drm/asahi/g18/probe.rs new file mode 100644 index 0000000000000..42c3988597e34 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/probe.rs @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! T8152 probe: the G18 platform driver, its parameter gates, device +//! preparation and the stage-10 DRM path. +//! +//! Items copied from the J873 module-1134 driver.rs (provenance in +//! the port's extraction ledger), then adapted to this driver's core: G18 is +//! its own platform driver, like G16; the shared probe body +//! ([`AsahiPlatformDriver::probe_g18`]) owns the DRM device and its +//! registration, and the G18 adapter is a `gpu::Gpu`. The parameter gates and +//! every log line are unchanged. + +use kernel::{ + device::Core, + dma::{Device, DmaMask}, + of, + platform, + prelude::*, + sync::{aref::ARef, Arc}, +}; + +use crate::g18::boot as g18_live_boot; +use crate::g18::{hw, identity, regs}; +use crate::{ + driver::{AsahiDrmDevice, AsahiPlatformDriver, AsahiPlatformDriverData}, + gpu, +}; + +/// Name of the G18 platform driver. +pub(crate) const DRIVER_NAME: &CStr = c"aurora-g18"; + +kernel::of_device_table!( + OF_TABLE, + ::IdInfo, + [(of::DeviceId::new(c"apple,agx-t8152"), &crate::hw::t8152::T8152)] +); + +/// The G18 platform driver: the shared [`AsahiPlatformDriver`] probed through +/// [`AsahiPlatformDriver::probe_g18`]. +pub(crate) struct Driver; + +/// What a bound G18 device holds until unbind. +pub(crate) struct G18Bound<'bound> { + /// The firmware owner, shut down on unbind before devres releases its resources. + pub(crate) runtime: Arc, + /// The DMA binding of a diagnostic stage, which has no DRM device. + pub(crate) binding: Option, + /// The shared driver data of the application stage's render node. + pub(crate) driver: Option>, +} + +impl platform::Driver for Driver { + type IdInfo = &'static crate::hw::t8152::SocConfig; + type Data<'bound> = G18Bound<'bound>; + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + /// The firmware session is retained until reboot once started. + fn suppress_bind_attrs() -> bool { + true + } + + fn probe<'bound>( + pdev: &'bound platform::Device>, + info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound { + let soc = *info.ok_or(ENODEV)?; + AsahiPlatformDriver::probe_g18(pdev, soc) + } + + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + this.runtime.shutdown(); + if let Some(binding) = &this.binding { + binding.close(); + } + if let Some(driver) = &this.driver { + driver.close_binding(); + } + } +} + +/// A started G18 session: its owner and, for the application stage, the GPU +/// the render node is registered over. +pub(crate) struct G18Started { + pub(crate) runtime: Arc, + pub(crate) gpu: Option>, +} + +/// Owner object returned by the G18 live-boot module. +/// +/// The live-boot module implements this for its session type. The driver +/// keeps it for the device's lifetime and calls [`G18Runtime::shutdown`] on +/// unbind, before devres releases the SGX mapping and the mailbox links. +pub(crate) trait G18Runtime: Send + Sync { + /// Quiesce the firmware, both ASC processors and every host mapping. + fn shutdown(&self); +} + +/// Everything the T8152 probe gate hands to the G18 live-boot entry point. +/// +/// Built only by [`prepare_t8152_device`], and only when `g18_initdata_test` +/// is non-zero. Holding one means: +/// - the three GPU PSD services (sgx 3, gfx-asc 81, gfx1-asc 82) were read +/// once, read-only, and their samples are in [`T8152Probe::psd`]; +/// - the SGX window is mapped in `registers`. Its ID block was read only if +/// the sgx PSD sample proved the block powered: `observed` is `Some` +/// exactly then; +/// - for the firmware stages 2..4 the loader's G18G firmware receipt matched +/// and the 42-bit DMA mask is set. Stages 1 and 90 require neither. +/// +/// No firmware, ASC CPU, PSD request, DMA or UAT state has been touched. +#[allow(dead_code)] +pub(crate) struct T8152Probe { + /// The matched SoC entry (`hw::agx3::T8152`). + pub(crate) soc: &'static hw::agx3::SocConfig, + /// Static identity decoded from the SoC entry (G18G, HAL400, SKSM). + pub(crate) identity: identity::GpuIdentity, + /// The `g18_initdata_test` stage, always non-zero here. + pub(crate) stage: u32, + /// Whether /chosen `aurora,g18g-gpu-firmware-profile` matched. + pub(crate) firmware_profile_admitted: bool, + /// Read-only samples of PSD services 3, 81 and 82, in that order. + pub(crate) psd: [regs::T8152PsdSample; 3], + /// SGX mapping. Every further SGX read must again follow a powered + /// `regs::t8152_prove_sgx_powered()` result. + pub(crate) registers: regs::Resources, + /// Hardware identity, only when the SGX PSD proved the block powered. + pub(crate) observed: Option, + /// The DRM device, not yet registered. The live-boot module allocates + /// its kernel objects on it; the shared probe registers it at stage 10. + pub(crate) drm: ARef, +} + +/// Loader receipt the G18G firmware stages require. The loader writes it +/// only after hashing both loaded TEXT images (b000 GFX 0x50000, b010 GFX1 +/// 0x54000) against the 26A428 profile (plan step 0.2). +const G18G_FIRMWARE_PROFILE: &[u8] = b"G18G-RTKit-3514.0.15.release\0"; + +/// T8152 probe gate. With `g18_initdata_test=0` (the default) this is a +/// complete no-op: it logs and refuses with ENODEV before any MMIO, DMA +/// mask, reserved-memory, mailbox or processor access, just as M5 refuses by +/// default. Unlike the M5 default it also skips the read-only SGX identity +/// probe, because J873 cold-boots with the GPU PSD services idle, where an +/// SGX read can raise an SError. +pub(crate) fn admit_t8152(pdev: &platform::Device>) -> Result { + let stage = crate::g18::params::g18_initdata_test.value(); + if stage == 0 { + dev_info!( + pdev.as_ref(), + "G18G: T8152 matched; g18_initdata_test=0 refuses before any MMIO, DMA, mailbox or processor access\n" + ); + return Err(ENODEV); + } + if !g18_live_boot::stage_is_known(stage) { + dev_err!( + pdev.as_ref(), + "G18G: g18_initdata_test={} is not a known stage (1-9, 90-93); refused before any MMIO\n", + stage + ); + return Err(EINVAL); + } + let variant = crate::g18::params::g18_compute_variant.value(); + if variant > 2 || (variant != 0 && stage != g18_live_boot::STAGE_COMPUTE_ADD) { + dev_err!( + pdev.as_ref(), + "G18G: g18_compute_variant={} needs 0, or 1/2 with g18_initdata_test=8; refused before any MMIO\n", + variant + ); + return Err(EINVAL); + } + let render_variant = crate::g18::params::g18_render_variant.value(); + let render_profile = crate::g18::params::g18_render_profile.value(); + let sma_minimum = crate::g18::params::g18_sma_minimum.value(); + let sma_ideal = crate::g18::params::g18_sma_ideal.value(); + let native_barrier = crate::g18::params::g18_native_barrier.value(); + if native_barrier > 1 || (native_barrier != 0 && (sma_ideal != 1 || sma_minimum != 0 || + stage != crate::g18::boot::render_stage::STAGE_RENDER || render_variant != 0 || render_profile != 4)) { + dev_err!(pdev.as_ref(), "G18G: native barrier probe requires ideal SMA stage9 variant0 profile4; refused before MMIO\n"); + return Err(EINVAL); + } + if sma_ideal > 1 || (sma_ideal != 0 && (sma_minimum != 0 || + stage != crate::g18::boot::render_stage::STAGE_RENDER || render_variant != 0 || render_profile != 4)) { + dev_err!(pdev.as_ref(), "G18G: SMA ideal probe requires stage9 variant0 profile4 and minimum=0; refused before MMIO\n"); + return Err(EINVAL); + } + // Explicit fixed-minimum comparison for the retained first application + // draw, as well as the original H1 witness. This is not dynamic sizing. + let minimum_probe = render_variant == 0 && + ((stage == crate::g18::boot::render_stage::STAGE_RENDER && render_profile == 4) || + (stage == g18_live_boot::STAGE_DRM && render_profile == 1)); + let stall_trace = crate::g18::params::g18_graphics_stall_trace.value(); + if stall_trace > 1 || (stall_trace != 0 && !minimum_probe) { + dev_err!(pdev.as_ref(), "G18G: graphics stall tracing requires stage9/profile4 or stage10/profile1, variant0; refused before MMIO\n"); + return Err(EINVAL); + } + let high_vdm = crate::g18::params::g18_vdm_high_alias.value(); + if high_vdm > 1 || (high_vdm != 0 && + (stage != crate::g18::boot::render_stage::STAGE_RENDER || render_variant != 0 || + render_profile != 4 || sma_minimum != 1 || sma_ideal != 0 || stall_trace != 1)) { + dev_err!(pdev.as_ref(), "G18G: high VDM alias requires stage9/profile4/variant0, minimum SMA and stall tracing; refused before MMIO\n"); + return Err(EINVAL); + } + let high_ppp = crate::g18::params::g18_ppp_high_alias.value(); + if high_ppp > 1 || (high_ppp != 0 && high_vdm != 1) { + dev_err!(pdev.as_ref(), "G18G: high PPP aliases require the bounded high VDM control; refused before MMIO\n"); + return Err(EINVAL); + } + if sma_minimum > 1 || (sma_minimum != 0 && !minimum_probe) { + dev_err!(pdev.as_ref(), "G18G: SMA minimum probe requires stage9/profile4 or stage10/profile1, variant0; refused before MMIO\n"); + return Err(EINVAL); + } + let usc_statistics = crate::g18::params::g18_usc_statistics.value(); + if usc_statistics > 1 || (usc_statistics != 0 && + (stage != crate::g18::boot::render_stage::STAGE_RENDER || render_variant != 0)) { + dev_err!(pdev.as_ref(), "G18G: USC statistics require the stage9 colour fixture; refused before MMIO\n"); + return Err(EINVAL); + } + if stage == crate::g18::boot::render_stage::STAGE_RENDER { + crate::g18::boot::render_stage::admit(pdev.as_ref(), render_variant, render_profile)?; + } else if render_variant != 0 || render_profile != 1 { + return Err(EINVAL); + } + let admit_msaa4 = crate::g18::params::g18_admit_msaa4.value(); + if admit_msaa4 > 1 || (admit_msaa4 != 0 && stage != g18_live_boot::STAGE_DRM) { + dev_err!(pdev.as_ref(), "G18G: g18_admit_msaa4={} needs 0, or 1 with g18_initdata_test=10; refused before MMIO\n", admit_msaa4); + return Err(EINVAL); + } + if stage == g18_live_boot::STAGE_DRM && crate::g18::params::g18_admit_compute.value() > 1 { + return Err(EINVAL); + } + Ok(()) +} + +/// Start the stage that [`admit_t8152`] admitted, on the unregistered `drm`. +pub(crate) fn probe_t8152( + pdev: &platform::Device>, + drm: &AsahiDrmDevice, + soc: &'static hw::agx3::SocConfig, +) -> Result { + let stage = crate::g18::params::g18_initdata_test.value(); + if stage == g18_live_boot::STAGE_DRM { + return probe_t8152_drm(pdev, drm, soc, stage); + } + let probe = prepare_t8152_device(pdev, drm, soc, stage)?; + Ok(G18Started { runtime: g18_live_boot_start(pdev, probe)?, gpu: None }) +} + +/// Start the G18 session the render node is registered over. +fn probe_t8152_drm( + pdev: &platform::Device>, + drm: &AsahiDrmDevice, + soc: &'static hw::agx3::SocConfig, + stage: u32, +) -> Result { + let mut probe = prepare_t8152_device(pdev, drm, soc, stage)?; + let observed = probe.observed.take(); + let (owner, runtime) = g18_live_boot::start_drm(pdev, probe)?; + let config = crate::g18::drm::G18DrmConfig { + admit_compute: crate::g18::params::g18_admit_compute.value() != 0, + admit_msaa4: crate::g18::params::g18_admit_msaa4.value() != 0, + admit_graphics_scratch: + crate::g18::params::g18_admit_graphics_scratch.value() != 0, + }; + let gpu: Arc = Arc::new( + crate::g18::drm::G18DrmGpu::new(runtime, observed.as_ref(), config)?, + GFP_KERNEL, + )?; + gpu.init()?; + Ok(G18Started { runtime: owner, gpu: Some(gpu) }) +} + +fn prepare_t8152_device( + pdev: &platform::Device>, + drm: &AsahiDrmDevice, + soc: &'static hw::agx3::SocConfig, + stage: u32, +) -> Result { + let identity = + identity::decode_gpu_identity(soc.hw_family, soc.hw_variant, soc.num_dies as u8) + .ok_or(EINVAL)?; + if soc.chip_id != 0x8152 + || identity.gpu_gen != hw::GpuGen::G18 + || identity.gpu_variant != identity::GpuVariant::G + || identity.uat_input_address_bits != soc.uat_ias + { + return Err(EINVAL); + } + dev_info!( + pdev.as_ref(), + "G18G: T8152 stage {}: {:?}{:?} USC gen {} HAL {:?} {:?} roles {:?} submission, {}-bit UAT roots\n", + stage, + identity.gpu_gen, + identity.gpu_variant, + identity.usc_generation, + identity.gpu_hal_generation, + identity.firmware_roles, + identity.submission_transport, + soc.uat_ias + ); + + let firmware_profile_admitted = match of::chosen() { + Some(chosen) => { + let profile: Result> = + chosen.get_property(c"aurora,g18g-gpu-firmware-profile"); + matches!(profile, Ok(profile) if profile.as_slice() == G18G_FIRMWARE_PROFILE) + } + None => false, + }; + dev_info!( + pdev.as_ref(), + "G18G: firmware receipt {}\n", + (if firmware_profile_admitted { "admitted" } else { "absent or different" }) + ); + if g18_live_boot::stage_runs_firmware(stage) && !firmware_profile_admitted { + dev_err!( + pdev.as_ref(), + "G18G: stage {} needs /chosen aurora,g18g-gpu-firmware-profile = G18G-RTKit-3514.0.15.release\n", + stage + ); + return Err(ENODEV); + } + + // PSD state first, read-only. Nothing below reads SGX unless the sgx + // service proves the block active and acknowledged. + let mut psd = [regs::T8152PsdSample { + service: regs::T8152PsdService::Sgx, + state: 0, + ack: 0, + }; 3]; + for (slot, service) in psd.iter_mut().zip(regs::T8152PsdService::ALL) { + *slot = regs::t8152_read_psd(pdev, service)?; + dev_info!( + pdev.as_ref(), + "G18G: PSD service {} ({:#x}) state={:#x} power state {} ack={:#x} -> {}\n", + service.service_id(), + service.physical_base(), + slot.state, + slot.power_state(), + slot.ack, + (if slot.powered() { "powered" } else { "not powered" }) + ); + } + + let registers = regs::Resources::new(pdev)?; + let observed = match regs::t8152_prove_sgx_powered(pdev) { + Ok(powered) => Some(registers.get_gpu_id_t8152(&powered, soc)?), + Err(error) if error == regs::t8152_not_powered() => { + dev_info!( + pdev.as_ref(), + "G18G: SGX identity not read (sgx PSD idle); identified by chip id {:#x} and compatible\n", + soc.chip_id + ); + None + } + Err(error) => return Err(error), + }; + + if g18_live_boot::stage_runs_firmware(stage) { + // J873 DRAM ends at 0x106_0000_0000, inside a 42-bit output space. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(soc.uat_oas)?)? }; + } + + // The device stays unregistered until the shared probe registers it at + // stage 10; diagnostic stages never register it. + let drm: ARef = drm.into(); + Ok(T8152Probe { + soc, + identity, + stage, + firmware_profile_admitted, + psd, + registers, + observed, + drm, + }) +} + +/// G18 live-boot entry point: the staged bring-up in `g18_live_boot.rs`. +/// +/// `probe.stage` is `g18_initdata_test`, already checked by +/// [`g18_live_boot::stage_is_known`]: +/// - 1: read-only probe (PSD, SGX ID, wrapper CPU state, post-office +/// status), then ENODEV; +/// - 2, 3, 4, 5, 6 (REQUIRE USER APPROVAL, plus the mailbox provider's own +/// `apple_mailbox.ascwrap_v8_cpu_control=1`): ASC CPU start and RTKit, +/// init-data ACKs, openings + idle wake + heartbeat. Stages 2 and 3 end in +/// ENODEV with every firmware owner retained; stages 4/5/6 return the owner. +/// Stage 5 registers one empty queue; stage 6 binds its scheduler. No GPU jobs; +/// - 90 (REQUIRES USER APPROVAL): PMGR2 PSD power-on only, then ENODEV. +/// +/// On success the driver keeps the owner and calls +/// [`G18Runtime::shutdown`] on unbind. +fn g18_live_boot_start( + pdev: &platform::Device>, + probe: T8152Probe, +) -> Result> { + g18_live_boot::start(pdev, probe) +} diff --git a/drivers/gpu/drm/asahi/g18/queue/graphics.rs b/drivers/gpu/drm/asahi/g18/queue/graphics.rs new file mode 100644 index 0000000000000..7917fd9555153 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/queue/graphics.rs @@ -0,0 +1,1077 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! J873 (26A428) TA/3D queue registration, binding and one render pass. +//! +//! Two SKSM kick queues, TA qid 9 (stage 0) and 3D qid 10 (stage 1), follow +//! the proven compute protocol of g18_queue_probe.rs (qid 11) step by step: +//! install (inner producer 1), scheduler bind (2), UMA pool + PB bind (3), +//! then one render pass [command, context config, kick] (6). Both queues share +//! ONE scheduler control record and ONE job list (M5 "Shared graphics +//! completion list"), and every inner command is encoded by +//! g18_render_command.rs, whose bytes equal the original TA/3D submitBuffer. +//! +//! Firmware references (GFX b000, SHA 2c63b343...): tag15 handler 0x12824 +//! (0x98 bytes), queue slot table 0xb90b8 + qid*0x28 (stage word at +0x20), +//! scheduler table 0xb7468 + qid*8, UMA pool slots 0x5f660 + slot*0x100, +//! per-queue pool table 0x6f660 + qid*0x38, PB slot table 0x71420 + slot*0x50. +//! +//! Outer CL_2 records (24 bytes each) are returned, not published: the +//! firmware-memory owner (g18_resources.rs) owns the CL_2 ring and producer. +//! Addresses come from the address map; nothing here chooses a VA. + +#![allow(dead_code)] + +#[cfg(not(test))] +#[path = "graphics_application.rs"] +mod application; + +use crate::g18::fw::render_command::{self as cmd, ChannelForm, G18ChannelConfig, G18ConfigUma, + G18ConfigParameterBuffer, G18RenderStage, G18_CHANNEL_CONFIG_BYTES, G18_KICK_BYTES}; + +pub(crate) const G18_GRAPHICS_QUEUE_IDS: [u8; 2] = [9, 10]; +pub(crate) const G18_GRAPHICS_STAGES: [G18RenderStage; 2] = [G18RenderStage::Ta, G18RenderStage::ThreeD]; +pub(crate) const G18_ENTRY_RING_BYTES: usize = 0x20000; // b000 0x1257c: 256 * 0x200 +pub(crate) const G18_GRAPH_BYTES: usize = 0x8000; +pub(crate) const G18_OUTER_RECORD_BYTES: usize = 24; +const PRIORITY: u32 = 2; +/// AGXChannel::getSubPriority value the compute path proved (config +0x5e). +const SUB_PRIORITY: u32 = 2; + +// Graph layout (one firmware-private object; offsets are this module's own). +const CHANNEL: [usize; 2] = [0x0, 0x100]; // 0x80-byte M6 channel state +const POINTERS: [usize; 2] = [0x200, 0x300]; // pointer state (+0 consumer, +0x40 producer) +const JOB_LIST: usize = 0x400; // shared scheduler job list +const CONTROL: usize = 0x500; // shared scheduler control (0x38) +const ITEMS: [usize; 2] = [0x800, 0x1000]; // u64 command-pointer rings +const CAPACITY: u32 = 0x100; +const CONFIGS: [usize; 2] = [0x1800, 0x2000]; // install/bind/pool/draw tag15 at 0x100 stride +const KICKS: [usize; 2] = [0x2800, 0x2c00]; // 0x40 stride + +const _: () = assert!(ITEMS[1] + CAPACITY as usize * 8 <= CONFIGS[0]); +const _: () = assert!(CONFIGS[1] + 0x400 <= KICKS[0] && KICKS[1] + 0x400 <= G18_GRAPH_BYTES); + +/// Addresses from the address map (see the G18 render-command notes, section 5). +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18GraphicsQueueAddresses { + /// SKSM entry rings: firmware VA (kernel_vm, PRIV_RW) and the kernel low + /// alias (kernel_lower_vm, GPU_SHARED_RW) of the same pages, per stage. + pub entry_high: [u64; 2], + pub entry_low: [u64; 2], + /// Queue graph, firmware VA (kernel_vm, PRIV_RW), G18_GRAPH_BYTES. + pub graph: u64, +} + +/// One 24-byte outer CL_2 record: b000 walkers 0x274cc..0x27598 read +8 +/// queue address, +0x10 ring DM (2), +0x14 inner producer, +0x16 qid, +0x17 +/// cache flag. Both stage queues travel on the CL_2 ring. +pub(crate) fn encode_g18_outer_record(queue_address: u64, queue_id: u8, inner_head: u16) + -> [u8; G18_OUTER_RECORD_BYTES] { + let mut record = [0; G18_OUTER_RECORD_BYTES]; + record[8..16].copy_from_slice(&queue_address.to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&inner_head.to_le_bytes()); + record[22] = queue_id; + record[23] = 1; + record +} + +/// Everything one render pass needs beyond the queues themselves. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18QueuedPass { + /// TA and 3D command firmware VAs (SKSM payload[0], inner item 0). + pub command_high: [u64; 2], + pub context_id: u32, + pub scheduler_control: u64, + pub scheduler_index: u32, + pub uma: G18ConfigUma, + /// TA only; `None` disables the PB tuple (3D never carries one). + pub parameter_buffer: Option, + /// RCE lists returned by encode_g18_ta_command / encode_g18_3d_command. + pub rce: [[m5_rce::M5SksmRce; 4]; 2], + /// Event stamp of each stage (0x100 per generation) and the queues' + /// event-machine generation for it. + pub stamps: [u32; 2], + pub event_generation: [u64; 2], + /// TA barrier record flag for the 3D's dependency (0: kind 1, the + /// M5-proven choice; 1: HAL400 3D dependency type 3). + pub ta_barrier_flag: u32, +} + +use crate::g18::fw::sksm as m5_rce; + +/// Kick-queue state the kernel mirrors (native queue +0x48/+0x98/+0xa0/+0x90). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18KickQueueState { + pub current_stamp: u64, + pub previous_stamp: Option, + pub base: u64, + /// Passes submitted on this queue (host bookkeeping only). + pub submitted_kicks: u16, +} + +/// Build one stage's SKSM entry and kick for a pass. Pure; the owner stores +/// them. `queue_address` is the stage's channel state (SKSM payload[1], the +/// native channel +0x80). +pub(crate) fn g18_stage_kick( + stage: usize, + pass: &G18QueuedPass, + queue: G18KickQueueState, + queue_address: u64, +) -> Result<(m5_rce::PreparedM5SksmEntry, [u8; G18_KICK_BYTES]), cmd::G18RenderCommandError> { + let qid = G18_GRAPHICS_QUEUE_IDS[stage]; + let mut barriers = [cmd::G18Barrier { stamp: 0, queue_index: 0, flag: 0, generation: 0 }; 1]; + let barrier_count = if stage == 1 { + barriers[0] = cmd::G18Barrier { + stamp: pass.stamps[0], queue_index: G18_GRAPHICS_QUEUE_IDS[0], + flag: pass.ta_barrier_flag, generation: pass.event_generation[0], + }; + cmd::g18_order_3d_barriers(&mut barriers, u32::from(G18_GRAPHICS_QUEUE_IDS[0])); + 1 + } else { 0 }; + let entry = cmd::encode_g18_kick_entry(&cmd::G18KickEntryInputs { + stage: G18_GRAPHICS_STAGES[stage], + queue: m5_rce::M5SksmQueue { id: qid, priority: PRIORITY as u8, + current_stamp: queue.current_stamp, submitted: 0, previous_stamp: queue.previous_stamp }, + entries: 256, base: queue.base, stamp: pass.stamps[stage], + generation: pass.event_generation[stage], barriers: &barriers[..barrier_count], + pipeline_dependency: 0, scheduler_index: pass.scheduler_index as u8, qos_word: 0xffff, + payload: [pass.command_high[stage], queue_address], rce: pass.rce[stage], + flag_bit1: false, flag_bit61: false, flag_bit63: false, event_flags: 0, + auxiliary: 0x3f_ffff_ffff_ffff, + })?; + // Native kick count = this descriptor's new entries (NOPs + 1); Linux + // never needs NOPs because every stamp is consecutive (encodeNops 0). + let kick = cmd::encode_g18_kick(G18_GRAPHICS_STAGES[stage], qid, PRIORITY as u8, + queue.current_stamp, 1)?; + Ok((entry, kick)) +} + +/// The tag15 configuration for a stage and protocol step (compute-proven +/// staging: 0 install, 1 scheduler bind, 2 pool/PB bind, 3 pass context). +pub(crate) fn g18_stage_config( + stage: usize, + step: u32, + entry: (u64, u64), + pass: &G18QueuedPass, +) -> [u8; G18_CHANNEL_CONFIG_BYTES] { + let mut out = [0u8; G18_CHANNEL_CONFIG_BYTES]; + let bound = step >= 2; + let config = G18ChannelConfig { + ring_low: entry.0, ring_high: entry.1, queue_id: u16::from(G18_GRAPHICS_QUEUE_IDS[stage]), + install: step == 0, priority: PRIORITY, data_master: stage as u32, + state_update_flag: false, state_update: 0, context_id: pass.context_id, + uma: if bound { pass.uma } else { G18ConfigUma { state: 0, generation: None, slot: u32::MAX } }, + channel_control: pass.scheduler_control, update_scheduler: step == 1, + scheduler_index: 0, pid: 0, sub_priority: SUB_PRIORITY, + update_context: step == 3, gart_flag: step == 3, + extension_flag: false, extension: 0, + parameter_buffer: if bound && stage == 0 { pass.parameter_buffer } else { None }, + }; + // Infallible: the output is exactly one M6 record. + let _ = cmd::encode_channel_config(ChannelForm::M6, &config, &mut out); + out +} + +// ---- Kernel owner ------------------------------------------------------------- + +#[cfg(not(test))] +use crate::g18::buffer::object::MappedObject; +#[cfg(not(test))] +use crate::{driver::AsahiDrmDevice, mmu}; +#[cfg(not(test))] +use core::sync::atomic::{fence, Ordering}; +#[cfg(not(test))] +use kernel::{c_str, io, prelude::*}; + +#[cfg(not(test))] +const DATA_SIZE: usize = 0xbc000; +#[cfg(not(test))] +const DATA_BASE: usize = 0x50000; +#[cfg(not(test))] +const SLOT_TABLE: usize = 0xb90b8 - DATA_BASE; +#[cfg(not(test))] +const SCHEDULER_TABLE: usize = 0xb7468 - DATA_BASE; +#[cfg(not(test))] +const POOL_TABLE: usize = 0x5f660 - DATA_BASE; +#[cfg(not(test))] +const QUEUE_POOL_TABLE: usize = 0x6f660 - DATA_BASE; +#[cfg(not(test))] +const PB_TABLE: usize = 0x71420 - DATA_BASE; + +/// Read-only view of reserved GFX DATA (same rules as g18_queue_probe.rs: +/// invalidate before reading, never clean firmware lines). +#[cfg(not(test))] +struct DataView { + map: io::mem::Mem, + cache_line: usize, +} +// SAFETY: bounded volatile reads of a fixed reserved mapping only. +#[cfg(not(test))] +unsafe impl Send for DataView {} +// SAFETY: concurrent reads cannot mutate host-owned state. +#[cfg(not(test))] +unsafe impl Sync for DataView {} + +#[cfg(not(test))] +impl DataView { + fn new(dev: &AsahiDrmDevice) -> Result { + let resource = dev.as_ref().as_ref().of_node().ok_or(EINVAL)? + .reserved_mem_region_to_resource_byname(c_str!("gfx-data"))?; + if resource.size() != DATA_SIZE as u64 || resource.start() < 0x100_0000_0000 + || resource.start() & 0x3fff != 0 + || resource.start().checked_add(resource.size()).ok_or(EOVERFLOW)? > 0x104_0000_0000 { + return Err(EINVAL); + } + let ctr: u64; + // SAFETY: architectural read-only cache geometry register. + unsafe { core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)) }; + let cache_line = 4usize << ((ctr >> 16) & 0xf); + if cache_line > 0x4000 { return Err(EINVAL); } + // SAFETY: WB matches reserved System RAM; the host never dirties it. + let map = unsafe { io::mem::Mem::try_new(resource, io::mem::MemFlag::WB.into()) }?; + Ok(Self { map, cache_line }) + } + + fn words(&self, offset: usize) -> Result<[u64; N]> { + let start = offset & !(self.cache_line - 1); + let end = (offset + N * 8 + self.cache_line - 1) & !(self.cache_line - 1); + if offset % 8 != 0 || end > DATA_SIZE { return Err(EINVAL); } + for line in (start..end).step_by(self.cache_line) { + // SAFETY: bounds checked above; invalidate only. + unsafe { core::arch::asm!("dc ivac, {addr}", + addr = in(reg) self.map.ptr().add(line), options(nostack, preserves_flags)) }; + } + // SAFETY: complete invalidation before observing firmware stores. + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; + let mut out = [0; N]; + for (index, word) in out.iter_mut().enumerate() { + // SAFETY: aligned reads inside the checked extent. + *word = unsafe { core::ptr::read_volatile( + self.map.ptr().add(offset + index * 8).cast::()) }; + } + fence(Ordering::Acquire); + Ok(out) + } +} + +/// Firmware-side observations for both queues. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18GraphicsFirmwareStatus { + /// Queue slot (5 words, +0x20 low half is the stage word) per stage. + pub slots: [[u64; 5]; 2], + /// Scheduler table entry per stage (the control pointer). + pub schedulers: [u64; 2], + /// Control header, membership words. + pub control: [u64; 3], +} + +#[cfg(not(test))] +pub(crate) struct G18GraphicsQueues { + aliases: KVec, + entries: KVec, + graph: MappedObject, + firmware: DataView, + addresses: G18GraphicsQueueAddresses, + step: u32, + queues: [G18KickQueueState; 2], +} + +#[cfg(not(test))] +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +#[cfg(not(test))] +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +#[cfg(not(test))] +impl G18GraphicsQueues { + /// Allocate both entry rings (dual-mapped, same pages proven) and the + /// graph; initialize channel/pointer states (AGXChannel::resetChannelState + /// 0xfffffe0008bdd178), the shared job list and the shared control + /// (AGXCommandQueueReclaimableState::init 0xfffffe0008bf9014). Nothing is + /// published: all inner producers stay 0. + pub(crate) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, a: G18GraphicsQueueAddresses) -> Result { + if !uat.is_t8152_geometry() { return Err(ENODEV); } + let mut entries = KVec::new(); + let mut aliases = KVec::new(); + for stage in 0..2 { + let mut entry = MappedObject::new_wc_at(dev, uat.kernel_vm(), a.entry_high[stage], + G18_ENTRY_RING_BYTES, mmu::PROT_FW_PRIV_RW)?; + aliases.push(entry.map_alias_at(uat.kernel_lower_vm(), a.entry_low[stage], + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..G18_ENTRY_RING_BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(a.entry_high[stage] + offset as u64)? + != uat.kernel_lower_vm().translate_iova(a.entry_low[stage] + offset as u64)? { + return Err(EFAULT); + } + } + entries.push(entry, GFP_KERNEL)?; + } + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), a.graph, G18_GRAPH_BYTES, + mmu::PROT_FW_PRIV_RW)?; + graph.with_bytes_mut(|out| { + for stage in 0..2 { + let channel = CHANNEL[stage]; + put64(out, channel, a.graph + POINTERS[stage] as u64); + put64(out, channel + 8, a.graph + ITEMS[stage] as u64); + // One job list for both queues (M5 lesson). + put64(out, channel + 0x10, a.graph + JOB_LIST as u64); + put32(out, channel + 0x24, u32::MAX); + put32(out, channel + 0x28, 4); + put32(out, channel + 0x40, u32::MAX); + put32(out, channel + 0x44, 0); + put64(out, channel + 0x70, a.graph + CONTROL as u64); + put32(out, POINTERS[stage] + 0x50, u32::MAX); + put32(out, POINTERS[stage] + 0x60, CAPACITY); + } + put64(out, JOB_LIST + 8, a.graph + JOB_LIST as u64); + out[CONTROL..CONTROL + 2].fill(0xff); + out[CONTROL + 5] = 1; + out[CONTROL + 0x33] = 0xff; + out[CONTROL + 0x26] = 2; + Ok(()) + })?; + fence(Ordering::SeqCst); + Ok(Self { aliases, entries, graph, firmware: DataView::new(dev)?, addresses: a, step: 0, + queues: [G18KickQueueState { current_stamp: 1, previous_stamp: None, base: 1, + submitted_kicks: 0 }; 2] }) + } + + /// Channel-state firmware VA: outer record +8 and SKSM payload[1]. + pub(crate) fn queue_address(&self, stage: usize) -> u64 { + self.addresses.graph + CHANNEL[stage & 1] as u64 + } + + pub(crate) fn scheduler_control(&self) -> u64 { self.addresses.graph + CONTROL as u64 } + + pub(crate) fn job_list(&self) -> u64 { self.addresses.graph + JOB_LIST as u64 } + + pub(crate) fn counters(&mut self) -> Result<[[u32; 3]; 2]> { + let mut result = [[0; 3]; 2]; + for stage in 0..2 { + let base = self.addresses.graph + POINTERS[stage] as u64; + result[stage] = [self.graph.read_u32(base)?, self.graph.read_u32(base + 0x30)?, + self.graph.read_u32(base + 0x40)?]; + } + Ok(result) + } + + pub(crate) fn completion_list(&mut self) -> Result<[u64; 2]> { + let list = self.job_list(); + Ok([self.graph.read_u64(list)?, self.graph.read_u64(list + 8)?]) + } + + pub(crate) fn firmware_status(&mut self) -> Result { + let mut slots = [[0; 5]; 2]; + let mut schedulers = [0; 2]; + for stage in 0..2 { + let qid = G18_GRAPHICS_QUEUE_IDS[stage] as usize; + slots[stage] = self.firmware.words(SLOT_TABLE + qid * 0x28)?; + schedulers[stage] = self.firmware.words::<1>(SCHEDULER_TABLE + qid * 8)?[0]; + } + let control = self.scheduler_control(); + Ok(G18GraphicsFirmwareStatus { slots, schedulers, + control: [self.graph.read_u64(control)?, self.graph.read_u64(control + 0x12)?, + self.graph.read_u64(control + 0x1a)?] }) + } + + /// Both firmware slots name our rings with the right stage words + /// (b000 0x12cb4..0x12cd4; +0x20 = tag15 +0x22). + pub(crate) fn installed(status: &G18GraphicsFirmwareStatus, a: &G18GraphicsQueueAddresses) -> bool { + (0..2).all(|stage| { + let slot = status.slots[stage]; + slot[0] & 1 != 0 && slot[1] == a.entry_high[stage] && slot[2] == a.entry_low[stage] + && slot[3] == 4u64 << 32 && slot[4] as u32 == stage as u32 + }) + } + + /// Scheduler bound for both queues: shared control published in both + /// scheduler entries, allocated indices, state 4, membership bits 9/10. + pub(crate) fn scheduler_bound(&self, status: &G18GraphicsFirmwareStatus) -> bool { + let header = status.control[0]; + let members = (1u64 << G18_GRAPHICS_QUEUE_IDS[0]) | (1u64 << G18_GRAPHICS_QUEUE_IDS[1]); + status.schedulers.iter().all(|&pointer| pointer == self.scheduler_control()) + && header & 0xff != 0xff && (header >> 8) & 0xff != 0xff + && (header >> 16) & 0xff == 4 && status.control[1] == members && status.control[2] == 0 + } + + /// Firmware UMA-pool and PB slot entries, for the pool/PB bind gate. + pub(crate) fn pool_and_pb_status(&mut self, uma_slot: u32, pb_slot: u32) -> Result<([u64; 5], [u64; 6])> { + let pool = self.firmware.words::<3>(POOL_TABLE + uma_slot as usize * 0x100)?; + let members = self.firmware.words::<2>(POOL_TABLE + uma_slot as usize * 0x100 + 0x78)?; + let pb = self.firmware.words::<6>(PB_TABLE + pb_slot as usize * 0x50)?; + Ok(([pool[0], pool[1], pool[2], members[0], members[1]], pb)) + } + + pub(crate) fn stage_gate(&mut self, phase: u32, pass: &G18QueuedPass) -> Result { + let status = self.firmware_status()?; + if self.counters()? != [[phase; 3]; 2] + || !Self::installed(&status, &self.addresses) { return Ok(false); } + if phase >= 2 && !self.scheduler_bound(&status) { return Ok(false); } + if phase >= 3 { + let pb = pass.parameter_buffer.ok_or(EINVAL)?; + let (pool, slot) = self.pool_and_pb_status(pass.uma.slot, pb.slot)?; + // b000 0x13708..0x137c0 sets the queue bit at slot +0x18 and the + // manager bit at +0x28; +0x20 stays clear. Stage33b hardware read + // exactly [state, 1, 3, 0x200, 0, 0x400] after the pool records. + if pool != [pass.uma.state, pass.uma.generation.ok_or(EINVAL)?, (2u64 << 32) | 3, + (1 << 9) | (1 << 10), 0] + || slot != [pb.state, pb.generation, u64::from(pass.context_id), 1 << 9, 0, 1 << 10] { + return Ok(false); + } + } + Ok(true) + } + + /// Gate before preparing any work item. Both rows, context and resource + /// fields are checked by the job/table owners; these are the queue checks. + pub(crate) fn audit_before_pass(&mut self, uat: &mmu::Uat, pass: &G18QueuedPass) -> Result { + if self.step != 3 || !self.stage_gate(3, pass)? { return Err(EIO); } + let a = self.addresses; + for stage in 0..2 { + self.graph.with_bytes_mut(|out| { + if out[ITEMS[stage]+24..ITEMS[stage]+48].iter().any(|&b| b != 0) { return Err(EBUSY); } + for phase in 0..3 { + let expected = g18_stage_config(stage, phase, (a.entry_low[stage], a.entry_high[stage]), pass); + let off = CONFIGS[stage]+phase as usize*0x100; + if out[off..off+G18_CHANNEL_CONFIG_BYTES] != expected { return Err(EIO); } + } + Ok(()) + })?; + self.entries[stage].with_bytes_mut(|out| { + if out[0x200..0x400].iter().any(|&b| b != 0) { return Err(EBUSY); } + Ok(()) + })?; + for off in (0..G18_ENTRY_RING_BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(a.entry_high[stage] + off as u64)? + != uat.kernel_lower_vm().translate_iova(a.entry_low[stage] + off as u64)? { return Err(EFAULT); } + } + } + for off in (0..G18_GRAPH_BYTES).step_by(mmu::UAT_PGSZ) { + uat.kernel_vm().translate_iova(a.graph + off as u64)?; + } + Ok(()) + } + + /// Compare the retained command records to the shared pure encoders and + /// print their actual bytes' digest before producer 6 can be published. + pub(crate) fn audit_prepared_pass(&mut self, dev: &AsahiDrmDevice, pass: &G18QueuedPass) -> Result { + if self.step != 4 || self.counters()? != [[3; 3]; 2] { return Err(EBUSY); } + let a = self.addresses; + for stage in 0..2 { + for phase in 0..4 { + let expected = g18_stage_config(stage, phase, (a.entry_low[stage], a.entry_high[stage]), pass); + self.graph.with_bytes_mut(|out| { + let off = CONFIGS[stage] + phase as usize*0x100; + let raw = &out[off..off+G18_CHANNEL_CONFIG_BYTES]; + if raw != expected { return Err(EIO); } + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=tag15-{}-{} fnv={:#018x}\n", stage, phase, + crate::g18::fw::graphics_payload::fnv64(raw)); + Ok(()) + })?; + let outer = encode_g18_outer_record(self.queue_address(stage), G18_GRAPHICS_QUEUE_IDS[stage], + [1,2,3,6][phase as usize]); + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=outer-{}-{} fnv={:#018x}\n", stage, phase, + crate::g18::fw::graphics_payload::fnv64(&outer)); + } + let (entry, kick) = g18_stage_kick(stage, pass, + G18KickQueueState { current_stamp: 1, previous_stamp: None, base: 1, submitted_kicks: 0 }, + self.queue_address(stage)).map_err(|_| EINVAL)?; + self.entries[stage].with_bytes_mut(|out| { + let raw = &out[0x200..0x400]; + if raw != entry.bytes { return Err(EIO); } + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=sksm-{} fnv={:#018x}\n",stage, + crate::g18::fw::graphics_payload::fnv64(raw)); + Ok(()) + })?; + self.graph.with_bytes_mut(|out| { + let raw = &out[KICKS[stage]..KICKS[stage]+G18_KICK_BYTES]; + if raw != kick { return Err(EIO); } + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=kick-{} fnv={:#018x}\n",stage, + crate::g18::fw::graphics_payload::fnv64(raw)); + Ok(()) + })?; + } + Ok(()) + } + + fn stage_step(&mut self, step: u32, pass: &G18QueuedPass) -> Result<[[u8; G18_OUTER_RECORD_BYTES]; 2]> { + if self.step != step || step > 2 { return Err(EBUSY); } + let expected = [step; 3]; + if self.counters()? != [expected; 2] { return Err(EBUSY); } + let a = self.addresses; + let mut records = [[0; G18_OUTER_RECORD_BYTES]; 2]; + for stage in 0..2 { + let config = g18_stage_config(stage, step, (a.entry_low[stage], a.entry_high[stage]), pass); + let at = CONFIGS[stage] + step as usize * 0x100; + let item = ITEMS[stage] + step as usize * 8; + self.graph.with_bytes_mut(|out| { + if out[at..at + 0x100].iter().any(|b| *b != 0) + || out[item..item + 8].iter().any(|b| *b != 0) { return Err(EBUSY); } + out[at..at + G18_CHANNEL_CONFIG_BYTES].copy_from_slice(&config); + put64(out, item, a.graph + at as u64); + Ok(()) + })?; + records[stage] = encode_g18_outer_record(self.queue_address(stage), + G18_GRAPHICS_QUEUE_IDS[stage], step as u16 + 1); + } + fence(Ordering::SeqCst); + for stage in 0..2 { + self.graph.write_u32(a.graph + POINTERS[stage] as u64 + 0x40, step + 1)?; + } + fence(Ordering::SeqCst); + self.step = step + 1; + Ok([records[1], records[0]]) + } + + /// Step 1: install both queues (tag15 install=1, stage words 0/1). + pub(crate) fn install_records(&mut self, pass: &G18QueuedPass) -> Result<[[u8; G18_OUTER_RECORD_BYTES]; 2]> { + self.stage_step(0, pass) + } + + /// Step 2 (after `installed`): bind both queues to the shared control. + pub(crate) fn scheduler_records(&mut self, pass: &G18QueuedPass) -> Result<[[u8; G18_OUTER_RECORD_BYTES]; 2]> { + self.stage_step(1, pass) + } + + /// Step 3 (after `scheduler_bound`): UMA pool on both, PB tuple on TA. + /// The UMA descriptor and PB row must already be loaded and read back. + pub(crate) fn pool_records(&mut self, pass: &G18QueuedPass) -> Result<[[u8; G18_OUTER_RECORD_BYTES]; 2]> { + if pass.uma.slot == u32::MAX || pass.uma.state == 0 { return Err(EINVAL); } + if let Some(pb) = pass.parameter_buffer { + if pb.generation == 0 || pb.state == 0 { return Err(EINVAL); } + } + self.stage_step(2, pass) + } + + /// Step 4: store both SKSM entries and [command, context config, kick] + /// per queue, unpublished. The job must have encoded both commands, + /// incremented the PB counter, loaded its roots and passed every + /// pre-kick check before `publish_pass`. + pub(crate) fn prepare_pass(&mut self, pass: &G18QueuedPass) -> Result { + if self.step != 3 || self.counters()? != [[3; 3]; 2] { return Err(EBUSY); } + let a = self.addresses; + for stage in [1, 0] { + let (entry, kick) = g18_stage_kick(stage, pass, self.queues[stage], + self.queue_address(stage)).map_err(|_| EINVAL)?; + let offset = entry.offset as usize; + self.entries[stage].with_bytes_mut(|out| { + let slot = out.get_mut(offset..offset + entry.bytes.len()).ok_or(EINVAL)?; + if slot.iter().any(|b| *b != 0) { return Err(EBUSY); } + slot.copy_from_slice(&entry.bytes); + Ok(()) + })?; + let config = g18_stage_config(stage, 3, (a.entry_low[stage], a.entry_high[stage]), pass); + let at = CONFIGS[stage] + 0x300; + self.graph.with_bytes_mut(|out| { + if out[at..at + 0x100].iter().any(|b| *b != 0) + || out[KICKS[stage]..KICKS[stage] + 0x40].iter().any(|b| *b != 0) + || out[ITEMS[stage] + 24..ITEMS[stage] + 48].iter().any(|b| *b != 0) { + return Err(EBUSY); + } + out[at..at + G18_CHANNEL_CONFIG_BYTES].copy_from_slice(&config); + out[KICKS[stage]..KICKS[stage] + G18_KICK_BYTES].copy_from_slice(&kick); + for (index, address) in [pass.command_high[stage], a.graph + at as u64, + a.graph + KICKS[stage] as u64].into_iter().enumerate() { + put64(out, ITEMS[stage] + (index + 3) * 8, address); + } + Ok(()) + })?; + self.queues[stage] = G18KickQueueState { + current_stamp: (self.queues[stage].current_stamp + 1) & ((1 << 40) - 1), + previous_stamp: Some(self.queues[stage].current_stamp), + base: self.queues[stage].base, + submitted_kicks: self.queues[stage].submitted_kicks + 1, + }; + } + fence(Ordering::SeqCst); + self.step = 4; + Ok(()) + } + + /// Publish both inner producers (6) and return the two outer records. + /// A later failure never authorizes resetting rings or entries. + pub(crate) fn publish_pass(&mut self) -> Result<[[u8; G18_OUTER_RECORD_BYTES]; 2]> { + if self.step != 4 || self.counters()? != [[3; 3]; 2] { return Err(EBUSY); } + fence(Ordering::SeqCst); + let mut records = [[0; G18_OUTER_RECORD_BYTES]; 2]; + for stage in [1, 0] { + self.graph.write_u32(self.addresses.graph + POINTERS[stage] as u64 + 0x40, 6)?; + records[stage] = encode_g18_outer_record(self.queue_address(stage), + G18_GRAPHICS_QUEUE_IDS[stage], 6); + } + fence(Ordering::SeqCst); + self.step = 5; + Ok([records[1], records[0]]) + } +} + +// ---- Parameter-buffer kernel owner ------------------------------------------- +// +// The PM object graph below keeps the layout native AGXParameterManagement +// init (0xfffffe0008c5eee4) builds and that M5 proved on hardware +// (m5_graphics_job.rs PM_BLOCKS/PM_SHARED/PM_COUNTER/SCENES): a per-block page +// base table (state +0x44 = PM +0xc0, u32 at 8-byte stride = PM +0x90), a +// shared header (state +0x4c = PM +0xc8; scene records link to its +0x40 = +// PM +0xa0), the submitted-operations counter (state +0x64) and 0x80-byte scene +// records (TA +0x28 = PM +0xa8[scene], TA +0x30 = PM +0xb8 = the record after +// the last). Scene record fields follow the native init loop (0x8c5f850.. +// 0x8c5f874) and setupSceneState (0x8c61ba8..0x8c61bc8); the shared header +// words are M5-derived and marked [OPEN] in the G18 render-command notes. + +use crate::g18::fw::parameter_buffer::*; + +const PM_GRAPH_BYTES: usize = 0x8000; +const BLOCK_TABLE: usize = 0; +const SHARED: usize = 0x800; +const COUNTER: usize = 0x900; +const SCENES: usize = 0x1000; +const SCENE_BYTES: usize = 0x80; + +fn pb_put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn pb_put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} +/// Native metrics spacing: device PM config word 1 (0x40) rounded (0x8c5f7b0). +const METRICS_STRIDE: u64 = 0x40; +const METRICS_BYTES: usize = 0x4000; +const SCENE_SCRATCH_BYTES: usize = 0x4000; +const DISCARD_BYTES: usize = 0x8000; +pub(crate) const G18_PB_MAX_SCENES: u32 = ((PM_GRAPH_BYTES - SCENES) / SCENE_BYTES - 1) as u32; + +/// Fixed addresses from the address map. `*_high` are firmware VAs (global, +/// kernel_vm); `*_low` are GPU VAs that the TA/3D hardware dereferences in +/// the job's context: they are created in the caller's low VM (the job VM) +/// and may be re-mapped into further windows with `alias_low_window`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18PbAddresses { + /// PB state object (0x4000 allocation, 0xc0 used). Firmware writes +0x08. + pub state_high: u64, + /// PM graph: block table, shared header, counter, scene records (0x8000). + pub graph_high: u64, + pub page_list_high: u64, + pub page_list_low: u64, + /// The page-list VA written to PB state +0x28 and the PB row: the job + /// context's view the PM decodes (see `g18_pb_row_address_valid`). + pub page_list_row: u64, + pub metrics_high: u64, + pub metrics_low: u64, + pub scene_scratch_low: u64, + /// Block 0; blocks are contiguous at 0x80000 stride. + pub blocks_low: u64, + pub discard_low: u64, + /// Added (wrapping) to a low VA to form the user-process view native + /// uses via getGPUVirtualAddressForUserProcess (block page bases, discard + /// page, scene cycle). 0 when the job maps the PB unbiased; the address + /// map's COMPACT_BIAS when the page list must name the biased window. + pub user_view_delta: u64, +} + +/// Host policy for one PB. Every field is a documented choice, not a default +/// silently inherited from M5 (see the G18 render-command notes, section 4). +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18PbPolicy { + /// Backed blocks (native initial allocation); must be < ring capacity. + pub blocks: u32, + /// PM +0x148 ring capacity, +0x154/+0x15c, +0x1d4. + pub capacity: u32, + pub pm_154: u32, + pub pm_15c: u32, + pub pm_1d4: u32, + /// Scene records and the selected scene (descriptor PM +0x2c). + pub scene_count: u32, + pub scene: u32, + /// PB table slot (TA HardwareBufferID). + pub slot: u32, + /// tag15 +0x82. Must differ from the firmware slot's cached generation + /// (0 on a fresh boot) or firmware never fetches the PB state. + pub generation: u64, +} + +/// Values the TA/3D commands take from the PB. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18PbCommandFields { + pub pb_slot: u32, + pub pm_1d4: u32, + pub pb_state: u64, + pub pm_scene: u64, + pub pm_b8: u64, +} + +/// Pre-kick readback of the published PB objects. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18PbReadback { + pub row: [u8; G18_PB_ROW_BYTES], + pub state: [u8; G18_PB_STATE_FIRMWARE_BYTES], + pub counter: u32, +} + +#[cfg(not(test))] +pub(crate) struct G18ParameterBuffer { + // Window aliases go first: that VM's roots may still be live. + window_aliases: KVec, + low_aliases: KVec, + state: MappedObject, + graph: MappedObject, + page_list: MappedObject, + metrics: MappedObject, + scene_scratch: MappedObject, + blocks: KVec, + discard: MappedObject, + addresses: G18PbAddresses, + policy: G18PbPolicy, + inputs: G18PbStateInputs, + row: Option<[u8; G18_PB_ROW_BYTES]>, +} + +#[cfg(not(test))] +impl G18ParameterBuffer { + /// Allocate, map and encode every PB object. Nothing is published to + /// firmware: the PB-table row is written separately by `load_row`, and + /// the tag15 that names the PB comes later still. + pub(crate) fn new( + dev: &AsahiDrmDevice, + uat: &mmu::Uat, + low_vm: &mmu::Vm, + a: G18PbAddresses, + policy: G18PbPolicy, + ) -> Result { + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 { + return Err(ENODEV); + } + if policy.blocks == 0 || policy.blocks >= policy.capacity + || policy.scene_count == 0 || policy.scene_count > G18_PB_MAX_SCENES + || policy.scene >= policy.scene_count || policy.slot >= G18_PB_TABLE_ROWS + || policy.generation == 0 + || u64::from(policy.scene_count) * METRICS_STRIDE > METRICS_BYTES as u64 + || a.blocks_low & (G18_PB_BLOCK_BYTES - 1) != 0 || a.discard_low & 0x7fff != 0 + || a.user_view_delta & 0x7fff != 0 + || !g18_pb_row_address_valid(a.page_list_row) { + return Err(EINVAL); + } + let list_entries = usize::try_from(policy.capacity)? + .checked_mul(G18_PB_PAGES_PER_BLOCK as usize).ok_or(EOVERFLOW)?; + let list_bytes = list_entries.checked_mul(4).ok_or(EOVERFLOW)? + .checked_add(mmu::UAT_PGSZ - 1).ok_or(EOVERFLOW)? & !(mmu::UAT_PGSZ - 1); + let high = |address, size| MappedObject::new_wc_at(dev, uat.kernel_vm(), address, size, + mmu::PROT_FW_SHARED_RW); + let low = |address, size| MappedObject::new_wc_at(dev, low_vm, address, + size, mmu::PROT_GPU_SHARED_RW); + let state = high(a.state_high, mmu::UAT_PGSZ)?; + let graph = high(a.graph_high, PM_GRAPH_BYTES)?; + let mut page_list = high(a.page_list_high, list_bytes)?; + let mut metrics = high(a.metrics_high, METRICS_BYTES)?; + let mut low_aliases = KVec::new(); + for (object, address, size) in [(&mut page_list, a.page_list_low, list_bytes), + (&mut metrics, a.metrics_low, METRICS_BYTES)] { + low_aliases.push(object.map_alias_at(low_vm, address, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(object.iova() + offset as u64)? + != low_vm.translate_iova(address + offset as u64)? { + return Err(EFAULT); + } + } + } + let scene_scratch = low(a.scene_scratch_low, SCENE_SCRATCH_BYTES)?; + let discard = low(a.discard_low, DISCARD_BYTES)?; + let mut blocks = KVec::new(); + for index in 0..u64::from(policy.blocks) { + let address = a.blocks_low + index * G18_PB_BLOCK_BYTES; + blocks.push(low(address, G18_PB_BLOCK_BYTES as usize)?, GFP_KERNEL)?; + } + let config = G18PmConfig { + pm_148: policy.capacity, + pm_138: u64::from(policy.blocks) << 19, + pm_1d8: G18_PB_PAGES_PER_BLOCK, + pm_70: G18_PB_BLOCK_BYTES, + pm_c0: a.graph_high + BLOCK_TABLE as u64, + pm_c8: a.graph_high + SHARED as u64, + pm_154: policy.pm_154, + pm_15c: policy.pm_15c, + pm_1d4: policy.pm_1d4, + }; + let inputs = G18PbStateInputs { + config, + page_list_firmware: a.page_list_high, + page_list_gpu: a.page_list_row, + page_list_bytes: u32::try_from(list_bytes)?, + counter_gpu: a.graph_high + COUNTER as u64, + discard_gpu: a.discard_low.wrapping_add(a.user_view_delta), + slot: policy.slot, + start_block: 0, + end_block: policy.blocks, + }; + let mut owner = Self { + window_aliases: KVec::new(), low_aliases, state, graph, page_list, metrics, + scene_scratch, blocks, discard, addresses: a, policy, inputs, row: None, + }; + owner.encode()?; + dev_info!(dev.as_ref(), + "G18 PB: state={:#x} graph={:#x} list={:#x}/{:#x} blocks={}x{:#x}@{:#x} slot={} gen={}; unpublished\n", + a.state_high, a.graph_high, a.page_list_high, a.page_list_low, policy.blocks, + G18_PB_BLOCK_BYTES, a.blocks_low, policy.slot, policy.generation); + Ok(owner) + } + + fn page_bases(&self) -> Result> { + let mut bases = KVec::with_capacity(self.policy.capacity as usize, GFP_KERNEL)?; + for slot in 0..u64::from(self.policy.capacity) { + // Unbacked ring slots are never reached before growth, which is + // not implemented; they repeat block 0 so no stray page appears. + let block = if slot < u64::from(self.policy.blocks) { slot } else { 0 }; + let va = (self.addresses.blocks_low + block * G18_PB_BLOCK_BYTES) + .wrapping_add(self.addresses.user_view_delta); + let base = u32::try_from(va >> 15).map_err(|_| ERANGE)?; + if base >> 25 != 0 { return Err(ERANGE); } + bases.push(base, GFP_KERNEL)?; + } + Ok(bases) + } + + fn scene_cycle_at(&self, scene: u32) -> u64 { + // Scene scratch in the user-process view (setupSceneState 0x8c61bb8). + self.addresses.scene_scratch_low.wrapping_add(self.addresses.user_view_delta) + 0x40 + + u64::from(scene) * 0x40 + } + + fn encode(&mut self) -> Result { + let bases = self.page_bases()?; + let inputs = self.inputs; + let blocks = self.policy.blocks as usize; + self.page_list.with_bytes_mut(|out| { + g18_pb_page_list(out, &inputs.config, &bases, 0, inputs.end_block) + .map_err(|_| EINVAL)?; + Ok(()) + })?; + let a = self.addresses; + let policy = self.policy; + let mut cycles = KVec::with_capacity(policy.scene_count as usize, GFP_KERNEL)?; + for scene in 0..policy.scene_count { + cycles.push(self.scene_cycle_at(scene), GFP_KERNEL)?; + } + self.graph.with_bytes_mut(|out| { + for (index, base) in bases.iter().take(blocks).enumerate() { + pb_put32(out, BLOCK_TABLE + index * 8, *base); + } + // [OPEN] M5-proven shared header; +8 is buildPageList's first block. + pb_put32(out, SHARED, policy.blocks); + pb_put32(out, SHARED + 4, policy.blocks); + pb_put32(out, SHARED + 8, 0); + pb_put32(out, SHARED + 0x60, 1); + pb_put32(out, COUNTER, 0); + for scene in 0..policy.scene_count as usize { + let record = SCENES + scene * SCENE_BYTES; + pb_put64(out, record, a.metrics_low + scene as u64 * METRICS_STRIDE); + pb_put64(out, record + 8, a.metrics_high + scene as u64 * METRICS_STRIDE); + pb_put64(out, record + 0x28, cycles[scene]); + pb_put64(out, record + 0x40, a.graph_high + SHARED as u64 + 0x40); + } + Ok(()) + })?; + self.state.with_bytes_mut(|out| { + let state: &mut [u8; G18_PB_STATE_BYTES] = + (&mut out[..G18_PB_STATE_BYTES]).try_into().map_err(|_| EINVAL)?; + encode_g18_pb_state(state, &inputs, true); + Ok(()) + })?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Map every low object again into `vm` at `low VA + delta` (delta 0: + /// another VM at the same addresses; COMPACT_BIAS: the biased window of + /// the dual low map). Verifies every page against `low_vm`, the VM the + /// objects were created in. Call before the job's roots are published. + pub(crate) fn alias_low_window(&mut self, low_vm: &mmu::Vm, vm: &mmu::Vm, delta: u64) -> Result { + let a = self.addresses; + let mut aliases = KVec::new(); + let mut ranges = KVec::new(); + let list_bytes = self.inputs.page_list_bytes as u64; + let at = |address: u64| address.checked_add(delta).ok_or(EOVERFLOW); + aliases.push(self.page_list.map_alias_at(vm, at(a.page_list_low)?, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + ranges.push((a.page_list_low, list_bytes), GFP_KERNEL)?; + aliases.push(self.metrics.map_alias_at(vm, at(a.metrics_low)?, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + ranges.push((a.metrics_low, METRICS_BYTES as u64), GFP_KERNEL)?; + aliases.push(self.scene_scratch.map_alias_at(vm, at(a.scene_scratch_low)?, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + ranges.push((a.scene_scratch_low, SCENE_SCRATCH_BYTES as u64), GFP_KERNEL)?; + aliases.push(self.discard.map_alias_at(vm, at(a.discard_low)?, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + ranges.push((a.discard_low, DISCARD_BYTES as u64), GFP_KERNEL)?; + for (index, block) in self.blocks.iter_mut().enumerate() { + let address = a.blocks_low + index as u64 * G18_PB_BLOCK_BYTES; + aliases.push(block.map_alias_at(vm, at(address)?, mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + ranges.push((address, G18_PB_BLOCK_BYTES), GFP_KERNEL)?; + } + for &(address, bytes) in ranges.iter() { + for offset in (0..bytes).step_by(mmu::UAT_PGSZ) { + if vm.translate_iova(at(address)? + offset)? + != low_vm.translate_iova(address + offset)? { + return Err(EFAULT); + } + } + } + fence(Ordering::SeqCst); + for alias in aliases { + self.window_aliases.push(alias, GFP_KERNEL)?; + } + Ok(()) + } + + /// The row halLoadPBDesc would write over `old` (ring zero after + /// buildPageList). + pub(crate) fn pb_row(&self, old: [u8; G18_PB_ROW_BYTES]) -> [u8; G18_PB_ROW_BYTES] { + g18_pb_row(self.inputs.page_list_gpu, g18_pb_page_count(&self.inputs.config), + G18PbRing::default(), old) + } + + /// The complete encoded PB state (firmware owns +0x08 after binding). + pub(crate) fn encode_pm_state(&self, out: &mut [u8; G18_PB_STATE_BYTES]) { + encode_g18_pb_state(out, &self.inputs, true); + } + + /// Native loadPBDesc order: state +0x0c (already encoded) then the row. + /// Refuses an occupied row or a second load; plain stores, then a fence, + /// before any tag15 names this PB. + pub(crate) fn load_row(&mut self, table: &mut MappedObject) -> Result<[u8; G18_PB_ROW_BYTES]> { + if self.row.is_some() { return Err(EBUSY); } + let offset = self.policy.slot as usize * G18_PB_ROW_BYTES; + let row = table.with_bytes_mut(|out| { + if out.len() != G18_PB_TABLE_ROWS as usize * G18_PB_ROW_BYTES { return Err(EINVAL); } + let old: [u8; G18_PB_ROW_BYTES] = out[offset..offset + G18_PB_ROW_BYTES] + .try_into().map_err(|_| EINVAL)?; + if old != [0; G18_PB_ROW_BYTES] { return Err(EBUSY); } + let row = self.pb_row(old); + out[offset..offset + G18_PB_ROW_BYTES].copy_from_slice(&row); + Ok(row) + })?; + fence(Ordering::SeqCst); + self.row = Some(row); + Ok(row) + } + + /// Pre-kick self-check: re-read the table row, the firmware-visible state + /// prefix (except the firmware-owned context word) and the page list + /// head, and compare with the encoders. Any difference refuses the kick. + pub(crate) fn verify_before_kick(&mut self, table: &mut MappedObject) -> Result { + let expected_row = self.row.ok_or(EINVAL)?; + let offset = self.policy.slot as usize * G18_PB_ROW_BYTES; + let row: [u8; G18_PB_ROW_BYTES] = table.with_bytes_mut(|out| { + out.get(offset..offset + G18_PB_ROW_BYTES).ok_or(EINVAL)?.try_into().map_err(|_| EINVAL) + })?; + if row != expected_row || row == [0; G18_PB_ROW_BYTES] { return Err(EIO); } + if g18_pb_row_decoded_address(&row) != Some(self.inputs.page_list_gpu) { return Err(EIO); } + let mut expected = [0u8; G18_PB_STATE_BYTES]; + self.encode_pm_state(&mut expected); + let state: [u8; G18_PB_STATE_FIRMWARE_BYTES] = self.state.with_bytes_mut(|out| { + out.get(..G18_PB_STATE_FIRMWARE_BYTES).ok_or(EINVAL)?.try_into().map_err(|_| EINVAL) + })?; + for (index, (actual, wanted)) in state.iter().zip(expected.iter()).enumerate() { + let firmware_owned = (8..12).contains(&index); + let unwritten = (0x1c..0x20).contains(&index); + if !firmware_owned && !unwritten && actual != wanted { return Err(EIO); } + } + let bases = self.page_bases()?; + let head = self.page_list.read_u32(self.addresses.page_list_high)?; + if head != bases[0] { return Err(EIO); } + for index in 0..self.policy.blocks as usize { + if self.graph.read_u32(self.addresses.graph_high + (index * 8) as u64)? != bases[index] { + return Err(EIO); + } + for page in 0..G18_PB_PAGES_PER_BLOCK as usize { + if self.page_list.read_u32(self.addresses.page_list_high + ((index * 16 + page) * 4) as u64)? + != bases[index] + page as u32 { return Err(EIO); } + } + } + let counter = self.graph.read_u32(self.addresses.graph_high + COUNTER as u64)?; + Ok(G18PbReadback { row, state, counter }) + } + + /// Native incrementPMSubmittedOperations (0x8bbcd50): one per 3D submit, + /// before the 3D command is published. + pub(crate) fn count_submitted_render(&mut self) -> Result { + let address = self.addresses.graph_high + COUNTER as u64; + let value = self.graph.read_u32(address)?.checked_add(1).ok_or(EOVERFLOW)?; + self.graph.write_u32(address, value)?; + fence(Ordering::SeqCst); + Ok(value) + } + + pub(crate) fn command_fields(&self) -> G18PbCommandFields { + let a = self.addresses; + G18PbCommandFields { + pb_slot: self.policy.slot, + pm_1d4: self.policy.pm_1d4, + pb_state: a.state_high, + pm_scene: a.graph_high + (SCENES + self.policy.scene as usize * SCENE_BYTES) as u64, + pm_b8: a.graph_high + (SCENES + self.policy.scene_count as usize * SCENE_BYTES) as u64, + } + } + + pub(crate) fn audit_job_mappings(&self, uat: &mmu::Uat, vm: &mmu::Vm, bias: u64) -> Result { + let a = self.addresses; + let mut pages = 0; + for (address, bytes) in [(a.page_list_low, self.inputs.page_list_bytes as usize), + (a.metrics_low, METRICS_BYTES), (a.scene_scratch_low, SCENE_SCRATCH_BYTES), + (a.discard_low, DISCARD_BYTES), (a.blocks_low, self.policy.blocks as usize * G18_PB_BLOCK_BYTES as usize)] { + for offset in (0..bytes).step_by(mmu::UAT_PGSZ) { + if vm.translate_iova(address + offset as u64)? != vm.translate_iova(bias + address + offset as u64)? { return Err(EFAULT); } + pages += 1; + } + } + for object in [&self.state, &self.graph, &self.page_list, &self.metrics] { + for off in (0..object.logical_size).step_by(mmu::UAT_PGSZ) { uat.kernel_vm().translate_iova(object.iova() + off as u64)?; } + } + Ok(pages) + } + + pub(crate) fn print_h1_digests(&mut self, dev: &AsahiDrmDevice, row: &[u8;16]) -> Result { + // Context +8 is firmware-owned and separately checked. Normalize only + // that field and the unwritten +1c word for host reproducibility. + self.state.with_bytes_mut(|out| { + let mut bytes = [0u8; G18_PB_STATE_BYTES]; + bytes.copy_from_slice(&out[..G18_PB_STATE_BYTES]); + bytes[8..12].fill(0); bytes[0x1c..0x20].fill(0); + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=pb-state fnv={:#018x}\n", crate::g18::fw::graphics_payload::fnv64(&bytes)); + Ok(()) + })?; + self.page_list.with_bytes_mut(|bytes| { + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=pb-page-list fnv={:#018x}\n", crate::g18::fw::graphics_payload::fnv64(bytes)); Ok(()) + })?; + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=pb-row fnv={:#018x}\n", crate::g18::fw::graphics_payload::fnv64(row)); + Ok(()) + } + + /// tag15 PB tuple for the TA queue; `manager` is the linked 3D queue index. + pub(crate) fn binding(&self, manager: u32) -> G18ConfigParameterBuffer { + G18ConfigParameterBuffer { + state: self.addresses.state_high, generation: self.policy.generation, + slot: self.policy.slot, manager, + } + } + + /// Register inputs owned by the PB (G17pRenderParameters.cycle and the + /// metrics record for register 0x1c910), for render-regs. + pub(crate) fn scene_cycle(&self) -> u64 { self.scene_cycle_at(self.policy.scene) } + + pub(crate) fn metrics_record(&self) -> u64 { + self.addresses.metrics_low + u64::from(self.policy.scene) * METRICS_STRIDE + } + + /// Firmware PB slot table entry (b000 0x71420 + slot * 0x50) expected + /// after the TA pool bind: state, generation, context, queue bit, manager bit. + pub(crate) fn expected_firmware_slot(&self, context: u32, ta_qid: u8, manager: u32) + -> [u64; 6] { + [self.addresses.state_high, self.policy.generation, u64::from(context), + 1u64 << ta_qid, 0, 1u64 << manager] + } +} diff --git a/drivers/gpu/drm/asahi/g18/queue/graphics_application.rs b/drivers/gpu/drm/asahi/g18/queue/graphics_application.rs new file mode 100644 index 0000000000000..623b98d506fee --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/queue/graphics_application.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retired-frame reuse. Ring slots wrap, but native event identities do not. +use super::*; +use crate::g18::uapi::application_runtime::Frame; + +impl G18GraphicsQueues { + pub(crate) fn prepare_application_pass(&mut self, dev: &AsahiDrmDevice, + pass: &G18QueuedPass, frame: Frame) -> Result { + let before = frame.inner_before(); + if self.counters()? != [[before;3];2] + || self.step != if frame.generation == 1 {3} else {5} + || self.completion_list()? != [0,self.job_list()] {return Err(EBUSY);} + let status=self.firmware_status()?; + if !Self::installed(&status,&self.addresses) || !self.scheduler_bound(&status) {return Err(EIO);} + let pb=pass.parameter_buffer.ok_or(EINVAL)?; + let (pool,slot)=self.pool_and_pb_status(pass.uma.slot,pb.slot)?; + if pool != [pass.uma.state,pass.uma.generation.ok_or(EINVAL)?,(2u64<<32)|3,0x600,0] + || slot != [pb.state,pb.generation,u64::from(pass.context_id),0x200,0,0x400] {return Err(EIO);} + for stage in [1,0] { + let state=G18KickQueueState {current_stamp:u64::from(frame.generation), + previous_stamp:if frame.generation==1 {None} else {Some(u64::from(frame.generation-1))}, + base:1,submitted_kicks:(frame.generation-1) as u16}; + let (entry,kick)=g18_stage_kick(stage,pass,state,self.queue_address(stage)).map_err(|_|EINVAL)?; + let config=g18_stage_config(stage,3,(self.addresses.entry_low[stage],self.addresses.entry_high[stage]),pass); + self.entries[stage].with_bytes_mut(|out| { + let slot=out.get_mut(entry.offset as usize..entry.offset as usize+entry.bytes.len()).ok_or(EINVAL)?; + slot.copy_from_slice(&entry.bytes); + if slot!=entry.bytes {return Err(EIO);} Ok(()) + })?; + let a=self.addresses.graph;let at=CONFIGS[stage]+0x300; + self.graph.with_bytes_mut(|out| { + out[at..at+G18_CHANNEL_CONFIG_BYTES].copy_from_slice(&config); + out[KICKS[stage]..KICKS[stage]+G18_KICK_BYTES].copy_from_slice(&kick); + for (i,address) in [pass.command_high[stage],a+at as u64,a+KICKS[stage] as u64].into_iter().enumerate() { + put64(out,ITEMS[stage]+((before as usize+i)%CAPACITY as usize)*8,address); + } + if out[at..at+G18_CHANNEL_CONFIG_BYTES]!=config + || out[KICKS[stage]..KICKS[stage]+G18_KICK_BYTES]!=kick {return Err(EIO);} Ok(()) + })?; + if frame.generation<=3 { + for (name,bytes) in [("sksm",entry.bytes.as_slice()),("kick",kick.as_slice()),("tag15",config.as_slice())] { + dev_info!(dev.as_ref(),"G18_APP_DIGEST frame={} stage={} name={} fnv={:#018x}\n", + frame.generation,stage,name,crate::g18::fw::graphics_payload::fnv64(bytes)); + } + } + } + fence(Ordering::SeqCst);self.step=4;Ok(()) + } + + /// The first firmware-owned precondition of the next frame's + /// [`Self::prepare_application_pass`] that does not hold after `frame` + /// retired, if any. A firmware recovery can reset queue state. + pub(crate) fn render_unready(&mut self, pass: &G18QueuedPass, frame: Frame) -> Result> { + let status=self.firmware_status()?; + let pb=pass.parameter_buffer.ok_or(EINVAL)?; + let (pool,slot)=self.pool_and_pb_status(pass.uma.slot,pb.slot)?; + Ok(if self.counters()? != [[frame.inner_after();3];2] {Some("counters")} + else if self.completion_list()? != [0,self.job_list()] {Some("completion-list")} + else if !Self::installed(&status,&self.addresses) {Some("installed")} + else if !self.scheduler_bound(&status) {Some("scheduler")} + else if pool != [pass.uma.state,pass.uma.generation.ok_or(EINVAL)?,(2u64<<32)|3,0x600,0] {Some("pool")} + else if slot != [pb.state,pb.generation,u64::from(pass.context_id),0x200,0,0x400] {Some("parameter-buffer")} + else {None}) + } + + /// Every firmware word [`Self::render_unready`] reads, for the kernel log. + pub(crate) fn render_state(&mut self, pass: &G18QueuedPass) -> Result<(G18GraphicsFirmwareStatus, [u64;5], [u64;6])> { + let pb=pass.parameter_buffer.ok_or(EINVAL)?; + let (pool,slot)=self.pool_and_pb_status(pass.uma.slot,pb.slot)?; + Ok((self.firmware_status()?,pool,slot)) + } + + pub(crate) fn publish_application_pass(&mut self, frame: Frame) -> Result<[[u8;24];2]> { + if self.step!=4 || self.counters()?!=[[frame.inner_before();3];2] {return Err(EBUSY);} + let mut records=[[0;24];2]; + fence(Ordering::SeqCst); + for stage in [1,0] { + self.graph.write_u32(self.addresses.graph+POINTERS[stage] as u64+0x40,frame.inner_after())?; + records[stage]=encode_g18_outer_record(self.queue_address(stage),G18_GRAPHICS_QUEUE_IDS[stage],frame.inner_after() as u16); + } + fence(Ordering::SeqCst);self.step=5;Ok([records[1],records[0]]) + } +} + +impl G18ParameterBuffer { + /// Keep the PB resident in its hardware slot across VM handoffs. Every + /// alias uses the addresses selected by the PB owner, including row VA. + pub(crate) fn application_aliases(&mut self, vm: &mmu::Vm) -> Result> { + let a=self.addresses;let bias=crate::g18::address_map::COMPACT_BIAS; + let mut aliases=KVec::new(); + for (object,low) in [(&mut self.page_list,a.page_list_low),(&mut self.metrics,a.metrics_low), + (&mut self.scene_scratch,a.scene_scratch_low),(&mut self.discard,a.discard_low)] { + for address in [low,low+bias] {aliases.push(object.map_alias_at(vm,address,mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?;} + } + if a.page_list_row!=a.page_list_low && a.page_list_row!=a.page_list_low+bias { + aliases.push(self.page_list.map_alias_at(vm,a.page_list_row,mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?; + } + for (index,block) in self.blocks.iter_mut().enumerate() { + let low=a.blocks_low+index as u64*G18_PB_BLOCK_BYTES; + for address in [low,low+bias] {aliases.push(block.map_alias_at(vm,address,mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?;} + } + Ok(aliases) + } + + /// The PB words a recovery can leave behind: the firmware-visible state + /// (ring cursors, free list, the +0x1c flag firmware sets when a recovery + /// flushes a render using this PB, b000 0xfd58-0xfd9c), the active scene + /// record, the submitted-render counter and the scene index. + pub(crate) fn recovery_words(&mut self) -> Result<([u32;G18_PB_STATE_FIRMWARE_BYTES/4],[u32;SCENE_BYTES/4],u32,u32)> { + let state: [u8;G18_PB_STATE_FIRMWARE_BYTES]=self.state.with_bytes_mut(|out|out[..G18_PB_STATE_FIRMWARE_BYTES].try_into().map_err(|_|EINVAL))?; + let start=SCENES+self.policy.scene as usize*SCENE_BYTES; + let scene: [u8;SCENE_BYTES]=self.graph.with_bytes_mut(|out|out[start..start+SCENE_BYTES].try_into().map_err(|_|EINVAL))?; + let word=|b:&[u8],i:usize|u32::from_le_bytes(b[i*4..i*4+4].try_into().unwrap_or([0;4])); + let counter=self.graph.read_u32(self.addresses.graph_high+COUNTER as u64)?; + Ok((core::array::from_fn(|i|word(&state,i)),core::array::from_fn(|i|word(&scene,i)),counter,self.policy.scene)) + } + + + pub(crate) fn application_scene(&mut self, template: Option<&[u8;SCENE_BYTES]>) -> Result<[u8;SCENE_BYTES]> { + let start=SCENES+self.policy.scene as usize*SCENE_BYTES; + self.graph.with_bytes_mut(|out| { + if let Some(bytes)=template {out[start..start+SCENE_BYTES].copy_from_slice(bytes);} + out[start..start+SCENE_BYTES].try_into().map_err(|_|EINVAL) + }) + } + + /// Hardware owns the ring cursors and free-list ordering after a render. + /// Do not re-encode them. Check the stable address/capacity and state owner. + pub(crate) fn verify_resident(&mut self, table: &mut MappedObject, context: u32, + generation: u32) -> Result { + if generation==1 {return self.verify_before_kick(table);} + let offset=self.policy.slot as usize*G18_PB_ROW_BYTES; + let row: [u8;G18_PB_ROW_BYTES]=table.with_bytes_mut(|out|out[offset..offset+G18_PB_ROW_BYTES].try_into().map_err(|_|EINVAL))?; + let word=|i:usize|u32::from_le_bytes(row[i..i+4].try_into().unwrap()); + if g18_pb_row_decoded_address(&row)!=Some(self.inputs.page_list_gpu) + || (word(4)&0x3f_ffff)!=(self.row.ok_or(EINVAL)?[4..8].try_into().map(u32::from_le_bytes).map_err(|_|EINVAL)?&0x3f_ffff) + || word(8)&0x3f_ffff>=self.policy.capacity*G18_PB_PAGES_PER_BLOCK + || word(12)&0x3f_ffff>=self.policy.capacity*G18_PB_PAGES_PER_BLOCK {return Err(EIO);} + let state: [u8;G18_PB_STATE_FIRMWARE_BYTES]=self.state.with_bytes_mut(|out|out[..G18_PB_STATE_FIRMWARE_BYTES].try_into().map_err(|_|EINVAL))?; + if u32::from_le_bytes(state[8..12].try_into().map_err(|_|EINVAL)?)!=context {return Err(EIO);} + let counter=self.graph.read_u32(self.addresses.graph_high+COUNTER as u64)?; + if counter!=generation {return Err(EIO);} + Ok(G18PbReadback {row,state,counter}) + } +} diff --git a/drivers/gpu/drm/asahi/g18/queue/mod.rs b/drivers/gpu/drm/asahi/g18/queue/mod.rs new file mode 100644 index 0000000000000..b63c47cc267c5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/queue/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 firmware queues: graphics and compute queue records, scheduler binding. + +#[allow(dead_code)] +pub(crate) mod graphics; +#[allow(dead_code)] +pub(crate) mod probe; +#[allow(dead_code)] +pub(crate) mod scheduler; diff --git a/drivers/gpu/drm/asahi/g18/queue/probe.rs b/drivers/gpu/drm/asahi/g18/queue/probe.rs new file mode 100644 index 0000000000000..ea739dab517b4 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/queue/probe.rs @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! J873/26A428 queue registration and one bounded compute submission. +//! +//! Native kernelcache: AGXChannel::resetChannelState 0xfffffe0008bdd178; +//! AGXCLChannelSKSM::submitBuffer 0xfffffe0008be3ddc..0x8be3fcc. +//! GFX b000: dispatcher 0x1267c, tag15 target 0x12824, table 0xb90b8. +//! M6 adds 12 bytes at config +0x6e; never use M5's 0x86-byte packet here. + +use crate::g18::buffer::object::MappedObject; +use crate::g18::buffer::pool::G18UmaPoolBinding; +use crate::g18::runtime::compute::G18PreparedComputeCommand; +use crate::{driver::AsahiDrmDevice, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{c_str, io, prelude::*}; + +const KERNEL_BASE: u64 = 0xffff_fc20_0000_0000; +const GRAPH_BASE: u64 = KERNEL_BASE + 0xc000_0000; +const ENTRY_HIGH: u64 = KERNEL_BASE + 0x39_0000; +const ENTRY_LOW: u64 = 0x70_005f_0000; +const ENTRY_SIZE: usize = 0x20000; // b000 0x1257c/0x125a0: 256 * 0x200 +pub(crate) const QUEUE_ID: u8 = 11; +const POINTERS: usize = 0x100; +const JOB_LIST: usize = 0x200; +const CONTROL: usize = 0x300; +const ITEMS: usize = 0x800; +const CAPACITY: u32 = 0x100; +const CONFIG: usize = 0x1800; +const DATA_SIZE: usize = 0xbc000; +const SLOT: usize = 0xb90b8 - 0x50000 + QUEUE_ID as usize * 0x28; +const SCHEDULER: usize = 0xb7468 - 0x50000 + QUEUE_ID as usize * 8; +const POOL_SLOT: usize = 0x5f660 - 0x50000; // native slot stride 0x100 +const QUEUE_POOLS: usize = 0x6f660 - 0x50000 + QUEUE_ID as usize * 0x38; + +/// Reserved GFX DATA, never written by the host. No native snapshot pointers +/// are used: the current DT reservation is the only physical address source. +struct FirmwareDataView { + map: io::mem::Mem, + cache_line: usize, +} +// SAFETY: The fixed reserved RAM mapping is observed only through bounded +// volatile reads; no host write or ownership transfer is possible. +unsafe impl Send for FirmwareDataView {} +// SAFETY: Concurrent read-only observations cannot mutate host-owned state. +unsafe impl Sync for FirmwareDataView {} + +impl FirmwareDataView { + fn new(dev: &AsahiDrmDevice) -> Result { + let resource = dev.as_ref().as_ref().of_node().ok_or(EINVAL)? + .reserved_mem_region_to_resource_byname(c_str!("gfx-data"))?; + if resource.size() != DATA_SIZE as u64 || resource.start() < 0x100_0000_0000 + || resource.start() & 0x3fff != 0 + || resource.start().checked_add(resource.size()).ok_or(EOVERFLOW)? > 0x104_0000_0000 + { + return Err(EINVAL); + } + let ctr: u64; + // SAFETY: Architectural read-only cache geometry register. + unsafe { core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)) }; + let cache_line = 4usize << ((ctr >> 16) & 0xf); + if cache_line > 0x4000 || (SLOT + 40 + cache_line - 1) & !(cache_line - 1) > DATA_SIZE { + return Err(EINVAL); + } + // SAFETY: WB matches reserved System RAM; the host never dirties it. + let map = unsafe { io::mem::Mem::try_new(resource, io::mem::MemFlag::WB.into()) }?; + Ok(Self { map, cache_line }) + } + + fn words(&self, offset: usize) -> [u64; N] { + // The only callers are the fixed slot and scheduler extents. Both + // are below the constructor-checked slot end inside reserved DATA. + let start = offset & !(self.cache_line - 1); + let end = (offset + N * 8 + self.cache_line - 1) & !(self.cache_line - 1); + for offset in (start..end).step_by(self.cache_line) { + // SAFETY: Constructor bounds the whole covering cache lines. + // Invalidate only: cleaning stale data would corrupt firmware. + unsafe { core::arch::asm!("dc ivac, {addr}", + addr = in(reg) self.map.ptr().add(offset), options(nostack, preserves_flags)) }; + } + // SAFETY: Complete cache invalidation before observing firmware stores. + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; + let mut out = [0; N]; + for (index, word) in out.iter_mut().enumerate() { + // SAFETY: Aligned reads wholly inside the constructor-checked slot. + *word = unsafe { core::ptr::read_volatile( + self.map.ptr().add(offset + index * 8).cast::()) }; + } + fence(Ordering::Acquire); + out + } +} + +pub(crate) struct G18QueueProbe { + _entry_alias: mmu::KernelMapping, + _entry: MappedObject, + graph: MappedObject, + firmware: FirmwareDataView, + compute_prepared: bool, + compute_generation: u32, + pool_slot: u32, +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +impl G18QueueProbe { + pub(crate) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, bind_scheduler: bool) -> Result { + if !uat.is_t8152_geometry() { return Err(ENODEV); } + let mut entry = MappedObject::new_wc_at(dev, uat.kernel_vm(), ENTRY_HIGH, + ENTRY_SIZE, mmu::PROT_FW_PRIV_RW)?; + let entry_alias = entry.map_alias_at(uat.kernel_lower_vm(), ENTRY_LOW, + mmu::PROT_GPU_SHARED_RW)?; + for offset in (0..ENTRY_SIZE).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(ENTRY_HIGH + offset as u64)? + != uat.kernel_lower_vm().translate_iova(ENTRY_LOW + offset as u64)? { + return Err(EFAULT); + } + } + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), GRAPH_BASE, + 0x4000, mmu::PROT_FW_PRIV_RW)?; + graph.with_bytes_mut(|out| { + // M6 channel state: resetChannelState zeros 0x80 bytes. The + // command processor only needs the pointers and ring capacity + // for this tag15-only experiment; retain native reset sentinels. + put64(out, 0, GRAPH_BASE + POINTERS as u64); + put64(out, 8, GRAPH_BASE + ITEMS as u64); + put64(out, 0x10, GRAPH_BASE + JOB_LIST as u64); + put32(out, 0x24, u32::MAX); + put32(out, 0x28, 4); + put32(out, 0x40, u32::MAX); + put32(out, 0x44, 0); + put64(out, 0x70, GRAPH_BASE + CONTROL as u64); + put32(out, POINTERS + 0x50, u32::MAX); + put32(out, POINTERS + 0x60, CAPACITY); + put64(out, JOB_LIST + 8, GRAPH_BASE + JOB_LIST as u64); + if bind_scheduler { + // 26A428 AGXCommandQueueReclaimableState::init at + // 0xfffffe0008bf9014. Both indices start unallocated. + out[CONTROL..CONTROL + 2].fill(0xff); + out[CONTROL + 5] = 1; + out[CONTROL + 0x33] = 0xff; + // b000 0xb234 sets2 on this fresh process-slot path. + out[CONTROL + 0x26] = 2; + } + + // b000 0x12824 requests 0x98 bytes. Unwritten trailing padding + // is zero. UMA/PB pool bindings and the new +0x6e/+0x72 grow + // pair are disabled. UMA +0x3e is a FList generation identity, + // not an event pointer; +0x46 selects its firmware pool slot. + let config = &mut out[CONFIG..CONFIG + 0x98]; + put32(config, 0, 15); + put64(config, 8, ENTRY_LOW); + put64(config, 0x10, ENTRY_HIGH); + config[0x18..0x1a].copy_from_slice(&u16::from(QUEUE_ID).to_le_bytes()); + for (offset, value) in [(0x1a, 1), (0x1e, 2), (0x22, 2), (0x32, 3), (0x5e, 2)] { + put32(config, offset, value); + } + put64(config, 0x3e, u64::MAX); + put32(config, 0x46, u32::MAX); + put64(config, 0x4a, GRAPH_BASE + CONTROL as u64); + config[0x82..0x92].fill(0xff); + put64(out, ITEMS, GRAPH_BASE + CONFIG as u64); + put32(out, POINTERS + 0x40, 1); + Ok(()) + })?; + fence(Ordering::SeqCst); + Ok(Self { _entry_alias: entry_alias, _entry: entry, graph, + firmware: FirmwareDataView::new(dev)?, compute_prepared: false, + compute_generation: 0, pool_slot: 0 }) + } + + pub(crate) fn queue_address(&self) -> u64 { GRAPH_BASE } + + /// The addition fixture owns context3. In DRM mode the retained graphics + /// owner already holds it, so compute must install the context it actually + /// acquired before its first descriptor is queued. The later context-update + /// command follows that descriptor and cannot repair the initial binding. + pub(crate) fn prepare_initial_context(&mut self, context: u32) -> Result { + if !(3..64).contains(&context) || self.counters()? != [0, 0, 1] + || self.slot() != [0; 5] { return Err(EBUSY); } + self.graph.with_bytes_mut(|out| { + put32(out, CONFIG + 0x32, context); + Ok(()) + })?; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn counters(&mut self) -> Result<[u32; 3]> { + Ok([self.graph.read_u32(GRAPH_BASE + POINTERS as u64)?, + self.graph.read_u32(GRAPH_BASE + POINTERS as u64 + 0x30)?, + self.graph.read_u32(GRAPH_BASE + POINTERS as u64 + 0x40)?]) + } + + pub(crate) fn slot(&self) -> [u64; 5] { self.firmware.words(SLOT) } + + pub(crate) fn scheduler_status(&mut self) -> Result<[u64; 4]> { + Ok([self.firmware.words::<1>(SCHEDULER)[0], + self.graph.read_u64(GRAPH_BASE + CONTROL as u64)?, + self.graph.read_u64(GRAPH_BASE + CONTROL as u64 + 0x12)?, + self.graph.read_u64(GRAPH_BASE + CONTROL as u64 + 0x1a)?]) + } + + pub(crate) fn prepare_scheduler_bind(&mut self) -> Result { + if self.counters()? != [1; 3] || !Self::installed(self.slot()) + || self.scheduler_status()? != [0, 0x1000000ffff, 0, 0] { + return Err(EBUSY); + } + self.graph.with_bytes_mut(|out| { + if out[CONFIG + 0x100..CONFIG + 0x198].iter().any(|x| *x != 0) { + return Err(EBUSY); + } + out.copy_within(CONFIG..CONFIG + 0x98, CONFIG + 0x100); + let config = &mut out[CONFIG + 0x100..CONFIG + 0x198]; + put32(config, 0x1a, 0); // already installed + put32(config, 0x52, 1); // bind scheduler + put64(out, ITEMS + 8, GRAPH_BASE + CONFIG as u64 + 0x100); + Ok(()) + })?; + fence(Ordering::SeqCst); + self.graph.write_u32(GRAPH_BASE + POINTERS as u64 + 0x40, 2)?; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn scheduler_bound([pointer, header, low, high]: [u64; 4]) -> bool { + pointer == GRAPH_BASE + CONTROL as u64 && header & 0xff != 0xff + && (header >> 8) & 0xff != 0xff + && (header >> 16) & 0xffff_ffff == 0x0104_0204 + && low == 1u64 << QUEUE_ID && high == 0 + } + + /// Observe both directions of firmware-owned pool membership, in addition + /// to the state/generation tuple. Retirement alone is not a binding proof. + pub(crate) fn pool_status(&self) -> [u64; 9] { + let slot = POOL_SLOT + self.pool_slot as usize * 0x100; + let header = self.firmware.words::<3>(slot); + let members = self.firmware.words::<2>(slot + 0x78); + let flags = self.firmware.words::<1>(slot + 0xe0); + let growth = self.firmware.words::<2>(slot + 0xf0); + let queues = self.firmware.words::<1>(QUEUE_POOLS); + [header[0], header[1], header[2], members[0], members[1], flags[0], + growth[0], growth[1], queues[0]] + } + + pub(crate) fn prepare_pool_bind(&mut self, binding: G18UmaPoolBinding) -> Result { + if binding.slot > 1 { return Err(EINVAL); } + self.pool_slot = binding.slot; + let before = self.pool_status(); + if self.counters()? != [2; 3] || !Self::installed(self.slot()) + || !Self::scheduler_bound(self.scheduler_status()?) + || binding.slot > 1 || binding.generation != 0 + || binding.state < KERNEL_BASE || binding.state & 0x3fff != 0 + || before[0] != 0 || before[3..5] != [0; 2] + || before[6..9] != [0; 3] { + return Err(EBUSY); + } + self.graph.with_bytes_mut(|out| { + if out[CONFIG + 0x200..CONFIG + 0x298].iter().any(|x| *x != 0) + || out[CONFIG + 0x136..CONFIG + 0x13e].iter().any(|x| *x != 0) + || out[CONFIG + 0x13e..CONFIG + 0x14a].iter().any(|x| *x != 0xff) { + return Err(EBUSY); + } + out.copy_within(CONFIG + 0x100..CONFIG + 0x198, CONFIG + 0x200); + let config = &mut out[CONFIG + 0x200..CONFIG + 0x298]; + put32(config, 0x52, 0); // scheduler is already bound + put64(config, 0x36, binding.state); + put64(config, 0x3e, binding.generation); + put32(config, 0x46, binding.slot); + put64(out, ITEMS + 16, GRAPH_BASE + CONFIG as u64 + 0x200); + Ok(()) + })?; + fence(Ordering::SeqCst); + self.graph.write_u32(GRAPH_BASE + POINTERS as u64 + 0x40, 3)?; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn pool_bound(status: [u64; 9], binding: G18UmaPoolBinding) -> bool { + Self::pool_bound_for_context(status, binding, 3) + } + + pub(crate) fn pool_bound_for_context(status: [u64; 9], binding: G18UmaPoolBinding, + context: u32) -> bool { + (3..64).contains(&context) + && status == [binding.state, binding.generation, (2u64 << 32) | u64::from(context), + 1u64 << QUEUE_ID, 0, 1, 0, 0, 1u64 << binding.slot] + } + + /// Store an unpublished SKSM entry and its three inner commands. The + /// original HAL400 encoder 8c4a290, dependency writer 8c4a5a4 and timestamp + /// converter 8c5620c agree byte-for-byte with the shared M5 serializer in + /// this bounded domain (audit-g18-sksm-native.py). The tag15 remains the + /// M6 0x98-byte form; never substitute the shorter M5 configuration. + pub(crate) fn prepare_compute_job(&mut self, command: G18PreparedComputeCommand, + pool: G18UmaPoolBinding) -> Result { + self.prepare_application_compute(command, pool, 1) + } + + pub(crate) fn prepare_application_compute(&mut self, command: G18PreparedComputeCommand, + pool: G18UmaPoolBinding, generation: u32) -> Result { + self.prepare_application_compute_rebind(command, pool, pool, command.context_id, generation) + } + + pub(crate) fn prepare_application_compute_rebind(&mut self, command: G18PreparedComputeCommand, + pool: G18UmaPoolBinding, previous: G18UmaPoolBinding, previous_context: u32, + generation: u32) -> Result { + use crate::g18::fw::sksm as sksm; + if pool.slot != previous.slot || pool.generation == u64::MAX + || !(pool.generation == previous.generation && pool.state == previous.state + || previous.generation.checked_add(1) == Some(pool.generation) + && pool.state != previous.state) + || pool.state < KERNEL_BASE || pool.state & 0x3fff != 0 { return Err(EINVAL); } + if generation == 0 || generation > 0x00ff_ffff + || generation != self.compute_generation + 1 { return Err(EINVAL); } + let before = (3 + (generation - 1) * 3) % 256; + if self.compute_prepared || self.counters()? != [before; 3] + || !Self::installed(self.slot()) + || !Self::scheduler_usable(self.scheduler_status()?) + || !Self::pool_bound_for_context(self.pool_status(), previous, previous_context) + || !(3..64).contains(&command.context_id) || command.expected_stamp != generation << 8 + || command.firmware_address < KERNEL_BASE + || command.registers.main_count > 127 || command.registers.secondary_count != 4 { + return Err(EINVAL); + } + let entry = sksm::prepare_m5_sksm_entry(sksm::M5SksmQueue { + id: QUEUE_ID, priority: 2, current_stamp: u64::from(generation), submitted: 0, + previous_stamp: if generation == 1 { None } else { Some(u64::from(generation - 1)) }, + }, sksm::M5SksmOperands { + barriers: &[], first_predecessor: 0, implicit_dependency_type: 1, + mcache: None, flag_bit1: false, flag_bit61: false, flag_bit63: false, + payload: [command.firmware_address, GRAPH_BASE], + qos_byte_32: 3, qos_class_40: 8, qos_word_48: 0xffff, + // Native CL event mask uses type 2 * 64 contexts + context ID. + event_mask: [0, 0, 1u64 << command.context_id, 0], rce_kind: 0, + rce: [sksm::M5SksmRce { address: command.registers.main_gpu, + tag: command.registers.main_count as u8 }, + sksm::M5SksmRce { address: command.registers.secondary_gpu, tag: 4 }, + sksm::M5SksmRce { address: 0, tag: 0 }, + sksm::M5SksmRce { address: 0, tag: 0 }], + auxiliary: 0x3f_ffff_ffff_ffff, + }).map_err(|_| EINVAL)?; + // Original 26A428 submitBuffer 8be481c..8be4848 writes this same + // packed 21-byte form: type14, qid, count, priority, stamp, DM2. + let kick = sksm::encode_m5_sksm_add_kicks(QUEUE_ID, 2, u64::from(generation), 1).map_err(|_| EINVAL)?; + self._entry.with_bytes_mut(|out| { + let bytes = &mut out[entry.offset as usize..entry.offset as usize + entry.bytes.len()]; + if generation < 256 && bytes.iter().any(|v| *v != 0) { return Err(EBUSY); } + bytes.copy_from_slice(&entry.bytes); + Ok(()) + })?; + self.graph.with_bytes_mut(|out| { + if generation == 1 && (out[CONFIG + 0x300..CONFIG + 0x500].iter().any(|v| *v != 0) + || out[ITEMS + 24..ITEMS + 48].iter().any(|v| *v != 0)) { + return Err(EBUSY); + } + out.copy_within(CONFIG + 0x200..CONFIG + 0x298, CONFIG + 0x300); + let config = &mut out[CONFIG + 0x300..CONFIG + 0x398]; + put32(config, 0x32, command.context_id); + put64(config, 0x36, pool.state); + put64(config, 0x3e, pool.generation); + put32(config, 0x46, pool.slot); + put32(config, 0x62, 1); // native descriptor+0x190: update context + put32(config, 0x66, 1); // retained process owner identity + out[CONFIG + 0x400..CONFIG + 0x400 + kick.len()].copy_from_slice(&kick); + for (index, address) in [command.firmware_address, + GRAPH_BASE + CONFIG as u64 + 0x300, + GRAPH_BASE + CONFIG as u64 + 0x400].into_iter().enumerate() { + put64(out, ITEMS + ((index + before as usize) % 256) * 8, address); + } + Ok(()) + })?; + fence(Ordering::SeqCst); + self.compute_prepared = true; + self.compute_generation = generation; + Ok(()) + } + + /// Publish only after process roots and submitted-counter are ready. + /// A later failure cannot authorize resetting the shared ring or entry. + pub(crate) fn publish_compute_job(&mut self) -> Result { + if !self.compute_prepared || self.compute_generation == 0 { return Err(EBUSY); } + let before = (3 + (self.compute_generation - 1) * 3) % 256; + if self.counters()? != [before; 3] { return Err(EBUSY); } + fence(Ordering::SeqCst); + self.graph.write_u32(GRAPH_BASE + POINTERS as u64 + 0x40, + (before + 3) % 256)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// The scheduler state a submission needs once bound: the published + /// control, allocated indices, state 4 and QID 11's membership. Unlike + /// [`Self::scheduler_bound`] (the bind gate, checked against the first + /// observed header) it does not pin header bytes 3..7: a recovery clears + /// byte 5, and render's equivalent check never required them. + pub(crate) fn scheduler_usable([pointer, header, low, high]: [u64; 4]) -> bool { + pointer == GRAPH_BASE + CONTROL as u64 && header & 0xff != 0xff + && (header >> 8) & 0xff != 0xff && (header >> 16) & 0xff == 4 + && low == 1u64 << QUEUE_ID && high == 0 + } + + pub(crate) fn retire_application_compute(&mut self, generation: u32) -> Result { + if !self.compute_prepared || self.compute_generation != generation + || self.counters()? != [(3 + generation * 3) % 256; 3] { return Err(EBUSY); } + self.compute_prepared = false; + Ok(()) + } + + /// The first firmware-owned precondition of + /// [`Self::prepare_application_compute`] that does not hold, if any. A + /// firmware recovery can reset queue state the next job requires. + pub(crate) fn compute_unready(&mut self, pool: G18UmaPoolBinding, context: u32, + generation: u32) -> Result> { + let before = (3 + generation.wrapping_sub(1) * 3) % 256; + Ok(if self.compute_prepared { Some("prepared") } + else if self.counters()? != [before; 3] { Some("counters") } + else if !Self::installed(self.slot()) { Some("installed") } + else if !Self::scheduler_usable(self.scheduler_status()?) { Some("scheduler") } + else if !Self::pool_bound_for_context(self.pool_status(), pool, context) { Some("pool") } + else { None }) + } + + /// Every word [`Self::compute_unready`] reads, for the kernel log. + pub(crate) fn compute_state(&mut self) -> Result<([u32; 3], [u64; 5], [u64; 4], [u64; 9])> { + Ok((self.counters()?, self.slot(), self.scheduler_status()?, self.pool_status())) + } + + pub(crate) fn installed(slot: [u64; 5]) -> bool { + slot[0] & 1 != 0 && slot[1] == ENTRY_HIGH && slot[2] == ENTRY_LOW + && slot[3] == 4u64 << 32 && slot[4] as u32 == 2 + } +} diff --git a/drivers/gpu/drm/asahi/g18/queue/scheduler.rs b/drivers/gpu/drm/asahi/g18/queue/scheduler.rs new file mode 100644 index 0000000000000..b5579ba9e534e --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/queue/scheduler.rs @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained command scheduler records and their independent submitted counters. +//! +//! Allocation and preparation do not publish a command. The containing live +//! firmware owner must retain these mappings until BOTH processors are stopped, +//! including after errors. A bounded bring-up session never recycles a record. + +#![allow(dead_code)] + +use crate::g18::buffer::object::MappedObject; +use crate::{driver::AsahiDrmDevice, g18::fw::scheduler::*, mmu}; +use core::ops::Range; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18SchedulerBinding { + pub(crate) record: u64, + pub(crate) submitted_counter: u64, + pub(crate) sequence: u32, +} + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18SchedulerSnapshot { + pub(crate) submitted: u32, + pub(crate) completed: u32, + pub(crate) linked: u32, + pub(crate) batches: [u32; 3], +} + +pub(crate) struct G18SchedulerMemory { + records: MappedObject, + counters: MappedObject, + armed: u64, + sequences: [u32; G18_SCHEDULER_RECORD_COUNT], + submitted: [u32; G18_SCHEDULER_RECORD_COUNT], +} + +impl G18SchedulerMemory { + pub(crate) fn print_render_digests(&mut self, dev: &AsahiDrmDevice) -> Result { + for (name, object) in [("scheduler", &mut self.records), ("submitted-pre-kick", &mut self.counters)] { + object.with_bytes_mut(|bytes| { + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name={} fnv={:#018x}\n",name,crate::g18::fw::graphics_payload::fnv64(bytes)); + Ok(()) + })?; + } + Ok(()) + } + /// Runtime's deterministic reservation, separate from the legacy compute + /// scheduler. Counters and records are independent uncached FW mappings. + pub(crate) fn new_render(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result { + use crate::g18::address_map::GRAPHICS_SCHEDULER_START as BASE; + if !uat.is_t8152_geometry() { return Err(ENODEV); } + let counters = MappedObject::new_wc_at(dev, uat.kernel_vm(), BASE, + G18_SCHEDULER_COUNTER_BYTES, mmu::PROT_FW_SHARED_RW)?; + let mut records = MappedObject::new_wc_at(dev, uat.kernel_vm(), BASE + 0x4000, + G18_SCHEDULER_RECORD_BYTES, mmu::PROT_FW_SHARED_RW)?; + records.with_bytes_mut(|out| initialize_g18_scheduler_records( + out.try_into().map_err(|_| EINVAL)?, counters.iova()).ok_or(EINVAL))?; + fence(Ordering::SeqCst); + Ok(Self { records, counters, armed: 0, + sequences: [0; G18_SCHEDULER_RECORD_COUNT], submitted: [0; G18_SCHEDULER_RECORD_COUNT] }) + } + + pub(crate) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, high_range: Range) -> Result { + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 { + return Err(ENODEV); + } + let counters = MappedObject::new_wc_in_range( + dev, + uat.kernel_vm(), + high_range.clone(), + G18_SCHEDULER_COUNTER_BYTES, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + )?; + let mut records = MappedObject::new_wc_in_range( + dev, + uat.kernel_vm(), + high_range, + G18_SCHEDULER_RECORD_BYTES, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + )?; + // Both native extents fit in one independently owned 16 KiB page. + // MappedObject zeroes its allocation before installing the mapping. + for object in [&counters, &records] { + uat.kernel_vm().translate_iova(object.iova())?; + } + records.with_bytes_mut(|out| { + let out = out.try_into().map_err(|_| EINVAL)?; + initialize_g18_scheduler_records(out, counters.iova()).ok_or(EINVAL) + })?; + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), + "G18 command scheduler: records={:#x} counters={:#x} count=36 stride=0x100 extent=0xc8 retained, unpublished\n", + records.iova(), counters.iova()); + Ok(Self { + records, + counters, + armed: 0, + sequences: [0; G18_SCHEDULER_RECORD_COUNT], + submitted: [0; G18_SCHEDULER_RECORD_COUNT], + }) + } + + /// Prepare one fresh record. A record can be armed only once during this + /// bounded session; completion alone does not authorize host reset/reuse. + pub(crate) fn prepare(&mut self, index: usize, sequence: u32) -> Result { + if index >= G18_SCHEDULER_RECORD_COUNT { + return Err(EINVAL); + } + if self.armed & (1u64 << index) != 0 { + return Err(EBUSY); + } + let start = index * G18_SCHEDULER_RECORD_STRIDE; + let submitted_counter = self.counters.iova() + index as u64 * 4; + self.counters.write_u32(submitted_counter, 0)?; + self.records.with_bytes_mut(|out| { + let record = out[start..start + G18_SCHEDULER_RECORD_STRIDE] + .as_mut() + .try_into() + .map_err(|_| EINVAL)?; + prepare_g18_scheduler_record( + record, + submitted_counter, + sequence, + G18_SCHEDULER_DEFAULT_THRESHOLD, + 0, + 0, + ) + .ok_or(EINVAL) + })?; + fence(Ordering::SeqCst); + self.sequences[index] = sequence; + self.armed |= 1u64 << index; + self.binding(index) + } + + pub(crate) fn binding(&self, index: usize) -> Result { + if index >= G18_SCHEDULER_RECORD_COUNT || self.armed & (1u64 << index) == 0 { + return Err(EINVAL); + } + Ok(G18SchedulerBinding { + record: self.records.iova() + (index * G18_SCHEDULER_RECORD_STRIDE) as u64, + submitted_counter: self.counters.iova() + index as u64 * 4, + sequence: self.sequences[index], + }) + } + + /// Final counter write before the caller publishes a fully prepared kick. + /// The caller must arrange all fallible preparation before this operation. + /// Once it succeeds, a failed publication requires retaining this session; + /// rolling the target back could race firmware. No firmware-owned record + /// fields are written here, even when submitting another job to the record. + pub(crate) fn publish_kick(&mut self, index: usize) -> Result { + let binding = self.binding(index)?; + let previous = self.submitted[index]; + if self.counters.read_u32(binding.submitted_counter)? != previous { + return Err(EIO); + } + let next = previous.checked_add(1).ok_or(EOVERFLOW)?; + self.counters.write_u32(binding.submitted_counter, next)?; + fence(Ordering::SeqCst); + self.submitted[index] = next; + Ok(next) + } + + /// Diagnostic snapshot only: these separate reads are not an atomic + /// completion fence and do not by themselves prove that a shader ran. + pub(crate) fn snapshot(&mut self, index: usize) -> Result { + let binding = self.binding(index)?; + fence(Ordering::SeqCst); + if self.records.read_u64(binding.record)? != binding.submitted_counter + || self.records.read_u32(binding.record + 8)? != binding.sequence + { + return Err(EIO); + } + Ok(G18SchedulerSnapshot { + submitted: self.counters.read_u32(binding.submitted_counter)?, + completed: self.records.read_u32(binding.record + 0xc)?, + linked: self.records.read_u32(binding.record + 0x94)?, + batches: [ + self.records.read_u32(binding.record + 0x14)?, + self.records.read_u32(binding.record + 0x18)?, + self.records.read_u32(binding.record + 0x1c)?, + ], + }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/regs.rs b/drivers/gpu/drm/asahi/g18/regs.rs new file mode 100644 index 0000000000000..c96ca475263fb --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/regs.rs @@ -0,0 +1,450 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! T8152 SGX register window and PSD power-state reads used before firmware start. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. +use kernel::{ + c_str, + device::Core, + io::{mem::DevresIoMem, Io, IoBase}, + platform, + prelude::*, + sync::aref::ARef, +}; +use crate::g18::{hw, identity}; + +const ID_VERSION: usize = 0xd04000; + +const ID_UNK08: usize = 0xd04008; + +const ID_COUNTS_1: usize = 0xd04010; + +const ID_COUNTS_2: usize = 0xd04014; + +const ID_UNK18: usize = 0xd04018; + +const ID_CLUSTERS: usize = 0xd0401c; + +// The "G14X" register family is shared by G14X (t602x) and every AGX3 part +// (G15/G16/G17). The ID block at 0xd04000 is unchanged, and the per-cluster +// core-enable masks are three u32 words at 0xe01500 (stride 4): AGXG17X +// `readChipInfo` builds the offset (`mov w23,#0x1500; movk w23,#0xe0,lsl#16` +// @ 0xfffffe0008b4b220) and reads [base], [base+4], [base+8]; the identical +// pattern appears in all eight AGX3 kexts of the pinned 26A5378n corpus +// (e.g. AGXG15G @ 0xfffffe0008af22f0). +const CORE_MASKS_G14X: usize = 0xe01500; + +/// Device resources for this GPU instance. +pub(crate) struct Resources { + dev: ARef, + sgx: DevresIoMem, + // The G17P SKSM FIFO lock and deferred SKSM writes are not carried: on + // T8152 the firmware owns KSMFE queue setup (hwdata+0x2724 = 0) and the + // host performs no SKSM MMIO. +} + +impl Resources { + + /// Map the required resources given our platform device. + pub(crate) fn new(pdev: &platform::Device>) -> Result { + // The pinned host maps accelerator device-memory index 0 and stores + // it at accelerator+0x348 before every SKSM scratch access. That range + // is the J700 GPU block exposed to Linux by the named `sgx` resource + // (the second platform `reg` tuple, after `asc`). + let sgx_req = pdev.io_request_by_name(c_str!("sgx")).ok_or(EINVAL)?; + let sgx_iomem = sgx_req.iomap()?.into_devres()?; + + Ok(Resources { + // SAFETY: This device does DMA via the UAT IOMMU. + dev: pdev.into(), + sgx: sgx_iomem, + }) + } + + /// Match the measured M5 native AP setup before starting firmware CPUs. + /// + /// The broad 25G76 AP trace writes 0x30001 at physical 0x580e90000; + /// Linux's prior stopped probe reads 0x30000. This diagnostic accepts + /// only those two observed states and changes only bit zero. The bit's + /// precise hardware role remains unqualified; do not generalize it. + pub(crate) fn m5_match_pmgr_boot_control(&self) -> Result { + const OFFSET: usize = 0xe90000; + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let res = (&*sgx).as_view().relaxed(); + let before = res.try_read32(OFFSET)?; + if before != 0x30000 && before != 0x30001 { + dev_err!(self.dev.as_ref(), "M5_INIT unexpected PMGR control {:#x}\n", before); + return Err(EIO); + } + res.try_write32(before | 1, OFFSET)?; + let after = res.try_read32(OFFSET)?; + dev_info!(self.dev.as_ref(), "M5_INIT native PMGR control: {:#x} -> {:#x}\n", + before, after); + if after != 0x30001 { + return Err(EIO); + } + Ok(()) + } + + /// Read and decode the T8152 (G18G) GPU identity. + /// + /// Requires the [`T8152SgxPowered`] proof: the SGX PMGR2 PSD service was + /// read and reported active immediately before, so these + /// reads cannot hit a gated block. Only the registers 26A428 + /// `AGX·PI_400·G·A0·Accelerator::readChipInfo` reads are touched: the ID + /// block at 0xd04000..0xd0401c and the single core-enable word at + /// 0xe01500. No register is written. + /// + /// The raw words are always logged. The decode never rejects: an unknown + /// or different family/variant byte is reported and the part is + /// identified by its chip id and DT compatible (`soc`), and implausible + /// counts fall back to the native registry topology (2 x 6 cores). + pub(crate) fn get_gpu_id_t8152( + &self, + powered: &T8152SgxPowered, + soc: &'static hw::agx3::SocConfig, + ) -> Result { + let registry = hw::agx3::T8152_REGISTRY; + let expected = + identity::decode_gpu_identity(soc.hw_family, soc.hw_variant, soc.num_dies as u8) + .ok_or(EINVAL)?; + if soc.chip_id != 0x8152 || expected.gpu_gen != hw::GpuGen::G18 { + return Err(EINVAL); + } + + let (id_version, id_unk08, id_counts_1, id_counts_2, id_unk18, id_clusters, core_mask) = { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + // Ordered (non-relaxed) reads, after the ordered PSD read. + ( + sgx.try_read32(ID_VERSION)?, + sgx.try_read32(ID_UNK08)?, + sgx.try_read32(ID_COUNTS_1)?, + sgx.try_read32(ID_COUNTS_2)?, + sgx.try_read32(ID_UNK18)?, + sgx.try_read32(ID_CLUSTERS)?, + sgx.try_read32(CORE_MASKS_G14X)?, + ) + }; + + dev_info!( + self.dev.as_ref(), + "T8152: PSD sgx state={:#x} ack={:#x}; raw GPU ID {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}; core mask {:#x}\n", + powered.sample().state, + powered.sample().ack, + id_version, + id_unk08, + id_counts_1, + id_counts_2, + id_unk18, + id_clusters, + core_mask + ); + + let family = ((id_version >> 24) & 0xff) as u8; + let variant = ((id_version >> 16) & 0xff) as u8; + let reg_dies = (id_counts_1 >> 16) & 0xf; + match identity::decode_gpu_identity(family, variant, reg_dies as u8) { + Some(decoded) if decoded == expected => dev_info!( + self.dev.as_ref(), + "T8152: ID family {:#x} variant {:#x} decodes to {:?}{:?} as expected\n", + family, + variant, + decoded.gpu_gen, + decoded.gpu_variant + ), + Some(decoded) => dev_warn!( + self.dev.as_ref(), + "T8152: ID family {:#x} variant {:#x} decodes to {:?}{:?}; using the chip-id/compatible identity {:?}{:?}\n", + family, + variant, + decoded.gpu_gen, + decoded.gpu_variant, + expected.gpu_gen, + expected.gpu_variant + ), + None => dev_warn!( + self.dev.as_ref(), + "T8152: unknown ID family {:#x} variant {:#x} dies {}; using the chip-id/compatible identity {:?}{:?}\n", + family, + variant, + reg_dies, + expected.gpu_gen, + expected.gpu_variant + ), + } + + // Same count layout as the PI_400 decode (@ 0xfffffe0008c3d53c..0x554): + // dies [19:16], clusters per die [15:8], cores per cluster [7:0]. + let reg_cores = id_counts_1 & 0xff; + let reg_clusters = ((id_counts_1 >> 8) & 0xff) * reg_dies; + let registry_cores = registry.num_cores / registry.num_mgpus; + let (num_dies, num_clusters, num_cores) = if (1..=32).contains(®_cores) + && reg_clusters != 0 + && reg_cores * reg_clusters <= 32 + { + (reg_dies, reg_clusters, reg_cores) + } else { + dev_warn!( + self.dev.as_ref(), + "T8152: implausible ID counts ({} dies, {} clusters, {} cores); using registry topology {} x {}\n", + reg_dies, + reg_clusters, + reg_cores, + registry.num_mgpus, + registry_cores + ); + (soc.num_dies, registry.num_mgpus, registry_cores) + }; + + let mut core_masks = KVec::new(); + let mut remaining = core_mask as u64; + let cluster_mask = (1u64 << num_cores) - 1; + let mut total_active_cores = 0; + for _ in 0..num_clusters { + let mask = (remaining & cluster_mask) as u32; + core_masks.push(mask, GFP_KERNEL)?; + total_active_cores += mask.count_ones(); + remaining >>= num_cores; + } + if remaining != 0 { + dev_warn!( + self.dev.as_ref(), + "T8152: core mask {:#x} has bits beyond {} x {} cores\n", + core_mask, + num_clusters, + num_cores + ); + } + if total_active_cores != registry.num_cores + || core_masks.as_slice() != registry.core_mask_list.as_slice() + { + dev_warn!( + self.dev.as_ref(), + "T8152: core masks {:#x?} ({} active) differ from the native registry {:#x?} ({})\n", + core_masks, + total_active_cores, + registry.core_mask_list, + registry.num_cores + ); + } + + let mut core_masks_packed = KVec::new(); + core_masks_packed.push(core_mask, GFP_KERNEL)?; + + let (gpu_rev, gpu_rev_id) = match (id_version >> 8) & 0xff { + 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), + 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), + 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), + 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), + 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), + 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), + a => { + dev_warn!( + self.dev.as_ref(), + "T8152: unknown GPU revision byte {:#x}; recorded as an A0 placeholder\n", + a + ); + (hw::GpuRevision::A0, hw::GpuRevisionID::A0) + } + }; + + dev_info!( + self.dev.as_ref(), + "T8152: {:?}{:?} rev {:?}, {} dies, {} clusters x {} cores, {} active, masks {:#x?}\n", + expected.gpu_gen, + expected.gpu_variant, + gpu_rev, + num_dies, + num_clusters, + num_cores, + total_active_cores, + core_masks + ); + + Ok(hw::GpuIdConfig { + gpu_gen: expected.gpu_gen, + gpu_variant: expected.gpu_variant, + usc_generation: expected.usc_generation, + gpu_hal_generation: expected.gpu_hal_generation, + gpu_rev, + gpu_rev_id, + num_dies, + num_clusters, + num_cores, + num_frags: num_cores, + num_gps: (id_counts_2 >> 16) & 0xff, + total_active_cores, + core_masks, + core_masks_packed, + gpc_perf_state_map: 0, + gpc_perf_state_map_low: 0, + gpc_perf_state_control: 0, + }) + } + +} + +const T8152_PSD_STATE: usize = 0x80; + +const T8152_PSD_ACK: usize = 0xbc; + +const T8152_PSD_STATE_MASK: u32 = 0x7; + +/// One of the three PSD services that power the T8152 GPU. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum T8152PsdService { + /// `sgx`, PSD service 3. + Sgx, + /// `gfx-asc` (GFX role ASC), PSD service 81. + GfxAsc, + /// `gfx1-asc` (GFX1 role ASC), PSD service 82. + Gfx1Asc, +} + +impl T8152PsdService { + /// Every GPU PSD service, in ADT order. + pub(crate) const ALL: [Self; 3] = [Self::Sgx, Self::GfxAsc, Self::Gfx1Asc]; + + /// Native 26A428 request/state, reproduced on J873 with PMP running. + pub(crate) const fn active_state(self) -> u32 { + match self { + Self::Sgx => 0, + Self::GfxAsc | Self::Gfx1Asc => 2, + } + } + + const fn resource_name(self) -> &'static CStr { + match self { + Self::Sgx => c_str!("psd-sgx"), + Self::GfxAsc => c_str!("psd-gfx-asc"), + Self::Gfx1Asc => c_str!("psd-gfx1-asc"), + } + } + + /// PMGR2 implicit service id. + pub(crate) const fn service_id(self) -> u32 { + match self { + Self::Sgx => 3, + Self::GfxAsc => 81, + Self::Gfx1Asc => 82, + } + } + + /// CPU-physical base of the service block. + pub(crate) const fn physical_base(self) -> u64 { + 0x3_0300_0000 + 0xa1_0000 + ((self.service_id() as u64) << 8) + } +} + +/// One read-only sample of a PSD service's state and ack words. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct T8152PsdSample { + pub(crate) service: T8152PsdService, + pub(crate) state: u32, + pub(crate) ack: u32, +} + +impl T8152PsdSample { + /// The PSD power state field, state[2:0]. + pub(crate) const fn power_state(&self) -> u32 { + self.state & T8152_PSD_STATE_MASK + } + + /// Point-in-time active-state proof, not a transition-completion test. + /// J873 hardware confirms STATE reads clear ACK. A request must poll + /// ACK first and verify STATE once; later access gates use STATE alone. + pub(crate) const fn powered(&self) -> bool { + self.power_state() == self.service.active_state() + } + + /// Short steady-state verdict for logs. + pub(crate) const fn verdict(&self) -> &'static str { + if self.powered() { + "powered" + } else { + "not powered" + } + } +} + +/// `-EHOSTDOWN`: the block's PSD service is not proven powered, so every +/// MMIO read of it was skipped. The mailbox provider returns the same errno. +pub(crate) fn t8152_not_powered() -> Error { + Error::from_errno(-(kernel::bindings::EHOSTDOWN as i32)) +} + +/// Proof that the SGX PSD service read as powered. +/// +/// Built only by [`t8152_prove_sgx_powered`] from a fresh powered sample. +/// [`Resources::get_gpu_id_t8152`] requires it, so the SGX ID block cannot +/// be read without a preceding PSD read. The proof is a point-in-time fact. +/// It does not keep the block powered, so use it right away. +pub(crate) struct T8152SgxPowered { + sample: T8152PsdSample, +} + +impl T8152SgxPowered { + pub(crate) const fn sample(&self) -> T8152PsdSample { + self.sample + } +} + +/// Read one GPU PSD service's state and ack words. Reads only, no writes. +/// +/// The block is mapped for this call alone and never claimed, so a later +/// power owner can map the same service block. +pub(crate) fn t8152_read_psd( + pdev: &platform::Device>, + service: T8152PsdService, +) -> Result { + let resource = pdev + .resource_by_name(service.resource_name()) + .ok_or(ENODEV)?; + if resource.start() != service.physical_base() || resource.size() != 0x100 { + dev_err!( + pdev.as_ref(), + "T8152: PSD service {} resource {:#x}+{:#x} does not match {:#x}+0x100\n", + service.service_id(), + resource.start(), + resource.size(), + service.physical_base() + ); + return Err(EINVAL); + } + let request = pdev + .io_request_by_name(service.resource_name()) + .ok_or(ENODEV)?; + let iomem = request.iomap()?.into_devres()?; + let io = iomem.try_access().ok_or(ENODEV)?; + // STATE consumes the transition ACK; preserve it in the snapshot first. + let ack = io.try_read32(T8152_PSD_ACK)?; + let state = io.try_read32(T8152_PSD_STATE)?; + Ok(T8152PsdSample { + service, + state, + ack, + }) +} + +/// Read the SGX PSD service and return a power proof only if it is active +/// (state 0). Otherwise log the sample and return [`t8152_not_powered`], so +/// the caller skips every SGX read. +pub(crate) fn t8152_prove_sgx_powered( + pdev: &platform::Device>, +) -> Result { + let sample = t8152_read_psd(pdev, T8152PsdService::Sgx)?; + if !sample.powered() { + dev_info!( + pdev.as_ref(), + "T8152: sgx PSD service 3 state={:#x} (power state {}) ack={:#x} is not active ({}); SGX reads skipped\n", + sample.state, + sample.power_state(), + sample.ack, + sample.verdict() + ); + return Err(t8152_not_powered()); + } + Ok(T8152SgxPowered { sample }) +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/compute.rs b/drivers/gpu/drm/asahi/g18/runtime/compute.rs new file mode 100644 index 0000000000000..867af831d8f7e --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/compute.rs @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained G18 channel working memory, one command, and completion storage. +//! +//! This owner never publishes a ring entry. All mappings remain retained by +//! G18FirmwareMemory until both firmware processors are stopped. Shader and +//! argument buffers, native device policy, and process context installation +//! remain separate obligations of the eventual submission path. + +#![allow(dead_code)] + +use crate::{g18::queue::scheduler::G18SchedulerBinding, g18::buffer::object::MappedObject}; +use crate::{driver::AsahiDrmDevice, g18::fw::compute::*, g18::fw::compute_memory::*, mmu}; +use core::ops::Range; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +// Host-selected range, after the existing UMA directory/block list, before +// the fixed graph. The VM allocator checks overlap with every existing owner. +const LOW_RANGE: Range = 0x70_0800_0000..0x70_1000_0000; +const SUPPORT_BYTES: usize = 0x100; +const STAMP_OFFSETS: [u64; 2] = [0, 0x40]; +const TIME_OFFSETS: [u64; 2] = [0x80, 0xc0]; + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18PreparedComputeCommand { + pub(crate) firmware_address: u64, + pub(crate) gpu_address: u64, + pub(crate) expected_stamp: u32, + pub(crate) context_id: u32, + pub(crate) scheduler_record: u64, + pub(crate) registers: G18ComputeRegisterSummary, +} + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18ComputeCompletionSnapshot { + pub(crate) stamps: [u32; 2], + pub(crate) timestamps: [u64; 2], +} + +pub(crate) struct G18ComputeMemory { + // Retract the process roots before dropping their aliases and backing. + process: Option, + // Alias owns the same GEM object and is released before the lower mapping. + command_high: mmu::KernelMapping, + command: MappedObject, + channel: MappedObject, + support: MappedObject, + layout: G18ComputeChannelLayout, + pointers: G18ComputeChannelPointers, + prepared: bool, + // Set before attempting publication, so even a partial failure cannot + // authorize the unpublished preparation rollback path. + context_published: bool, +} + +struct G18ComputeProcess { + context: mmu::G18ExecutionContext, + // Same GEM pages and GPU VAs as the firmware-owned command/channel. + aliases: [mmu::KernelMapping; 2], +} + +impl G18ComputeMemory { + /// The application owner calls this only after the complete queue, + /// scheduler and stamp gate, while it still retains the submitted VM. + pub(crate) fn retire_application(&mut self, stamp: u32) -> Result { + let completed = self.completion_snapshot()?; + if completed.stamps[0] != stamp || completed.timestamps[0] == 0 + || completed.timestamps[1] < completed.timestamps[0] { return Err(EBUSY); } + self.release_application() + } + + /// Release a dispatch that a firmware recovery ended after its queue + /// consumed it: the GPU was reset, so no stamp or timestamp will come. + pub(crate) fn retire_recovered(&mut self) -> Result { + if !self.prepared { return Err(EINVAL); } + self.release_application() + } + + fn release_application(&mut self) -> Result { + self.process = None; + self.prepared = false; + self.context_published = false; + self.command.with_bytes_mut(|bytes| { bytes.fill(0); Ok(()) })?; + self.support.with_bytes_mut(|bytes| { bytes.fill(0); Ok(()) })?; + Ok(()) + } + + /// Drop a process lease acquired by a failed preparation. No prepared + /// command or attempted root publication may cross this boundary. + pub(crate) fn abort_unpublished_process(&mut self) -> Result { + if self.prepared || self.context_published { return Err(EBUSY); } + self.command.with_bytes_mut(|bytes| { bytes.fill(0); Ok(()) })?; + self.process = None; + Ok(()) + } + + pub(crate) fn unpublished_idle(&self) -> bool { + !self.prepared && !self.context_published && self.process.is_none() + } + + pub(crate) fn new( + dev: &AsahiDrmDevice, + uat: &mmu::Uat, + high_range: Range, + clusters: u32, + width: u32, + ) -> Result { + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 { + return Err(ENODEV); + } + let layout = G18ComputeChannelLayout::new(clusters, width).ok_or(EINVAL)?; + let size: usize = layout.allocation_bytes.try_into()?; + let channel = MappedObject::new_wc_in_range( + dev, + uat.kernel_lower_vm(), + LOW_RANGE, + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + )?; + let pointers = layout.bind(channel.iova()).ok_or(ERANGE)?; + for offset in (0..size).step_by(mmu::UAT_PGSZ) { + uat.kernel_lower_vm() + .translate_iova(channel.iova() + offset as u64)?; + } + let mut command = MappedObject::new_wc_in_range( + dev, + uat.kernel_lower_vm(), + LOW_RANGE, + G18_COMPUTE_COMMAND_BYTES, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + )?; + let command_high = command.map_alias_in_range( + uat.kernel_vm(), + high_range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + )?; + for offset in (0..G18_COMPUTE_COMMAND_BYTES).step_by(mmu::UAT_PGSZ) { + if uat + .kernel_lower_vm() + .translate_iova(command.iova() + offset as u64)? + != uat + .kernel_vm() + .translate_iova(command_high.iova() + offset as u64)? + { + return Err(EFAULT); + } + } + let support = MappedObject::new_wc_in_range( + dev, + uat.kernel_vm(), + high_range, + SUPPORT_BYTES, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + )?; + uat.kernel_vm().translate_iova(support.iova())?; + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), + "G18 compute memory: channel={:#x}+{:#x} operands={:#x}/{:#x} control={:#x} tail={:#x} command={:#x}/{:#x} support={:#x}, retained unpublished\n", + channel.iova(), layout.allocation_bytes, pointers.operands[0], pointers.operands[1], + pointers.control, pointers.tail, command.iova(), command_high.iova(), support.iova()); + Ok(Self { + process: None, + command_high, + command, + channel, + support, + layout, + pointers, + prepared: false, + context_published: false, + }) + } + + /// Bind the payload VM while retaining both aliases and the context lease. + /// A failure leaves the original command/channel and firmware roots intact. + pub(crate) fn bind_process( + &mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, + ) -> Result { + if !self.unpublished_idle() { return Err(EBUSY); } + let context = uat.prepare_t8152_execution_context(vm)?; + let command = self.command.map_alias_at(vm, self.command.iova(), mmu::PROT_GPU_SHARED_RW)?; + let channel = self.channel.map_alias_at(vm, self.channel.iova(), mmu::PROT_GPU_SHARED_RW)?; + for (address, length) in [(self.command.iova(), G18_COMPUTE_COMMAND_BYTES), + (self.channel.iova(), self.layout.allocation_bytes as usize)] { + for offset in (0..length).step_by(mmu::UAT_PGSZ) { + let address = address + offset as u64; + if vm.translate_iova(address)? != uat.kernel_lower_vm().translate_iova(address)? { + return Err(EFAULT); + } + } + } + let id = u32::from(context.context_id()); + crate::g18_kick_info!(dev.as_ref(), + "G18 compute context: id={} roots={:x?}; command/channel aliases retained, unpublished\n", + id, context.roots()); + self.process = Some(G18ComputeProcess { context, aliases: [command, channel] }); + Ok(id) + } + + /// Encode one unpublished command. Caller must retain shader/argument + /// bindings in the selected process VM and supply native device policy. + pub(crate) fn prepare( + &mut self, + scheduler: G18SchedulerBinding, + inputs: &G18ComputeCommandInputs, + registers: &G18ComputeRegisterInputs, + ) -> Result { + if self.prepared { + return Err(EBUSY); + } + let process = self.process.as_ref().ok_or(EINVAL)?; + if registers.clusters != self.layout.clusters + || registers.cluster_width != self.layout.width + || registers.context_id != u32::from(process.context.context_id()) + || inputs.channel_ordinal >= 4 + || inputs.completion.stamp == 0 + || scheduler.record == 0 + { + return Err(EINVAL); + } + let inputs = G18ComputeCommandInputs { + d198: scheduler.record, + stamp_addresses: STAMP_OFFSETS.map(|offset| self.support.iova() + offset), + d1b0: self.support.iova() + TIME_OFFSETS[0], + d1b8: self.support.iova() + TIME_OFFSETS[1], + ..*inputs + }; + let registers = G18ComputeRegisterInputs { + command_gpu: self.command.iova(), + operands: self.pointers.operands, + support_control: self.pointers.control, + // 26A428 loadCommandBufferData 8be5eac selects this record; + // prepare 8be58b0 then takes its +8 word, independently of the + // UAT context ID. Optional SMAPP resource objects are disabled. + d630: scheduler.sequence, + ..*registers + }; + let summary = self.command.with_bytes_mut(|out| { + let out = out.try_into().map_err(|_| EINVAL)?; + encode_g18_compute_command(out, &inputs, ®isters).map_err(|_| EINVAL) + })?; + fence(Ordering::SeqCst); + self.prepared = true; + Ok(G18PreparedComputeCommand { + firmware_address: self.command_high.iova(), + gpu_address: self.command.iova(), + expected_stamp: inputs.completion.stamp, + context_id: registers.context_id, + scheduler_record: scheduler.record, + registers: summary, + }) + } + + /// Root publication is delayed until every command field is prepared. + /// It does not publish a queue entry or advance a scheduler counter. + pub(crate) fn publish_context(&mut self) -> Result<[u64; 2]> { + if !self.prepared { return Err(EINVAL); } + let process = self.process.as_mut().ok_or(EINVAL)?; + self.context_published = true; + process.context.publish()?; + if !process.context.is_current() { return Err(EIO); } + Ok(process.context.roots()) + } + + /// These independent WC reads are diagnostic, not an atomic completion + /// fence. Shader execution still requires verified output data and a clean + /// firmware/fault observation, even if both stamp words change. + pub(crate) fn completion_snapshot(&mut self) -> Result { + if !self.prepared { + return Err(EINVAL); + } + fence(Ordering::SeqCst); + let base = self.support.iova(); + Ok(G18ComputeCompletionSnapshot { + stamps: [ + self.support.read_u32(base + STAMP_OFFSETS[0])?, + self.support.read_u32(base + STAMP_OFFSETS[1])?, + ], + timestamps: [ + self.support.read_u64(base + TIME_OFFSETS[0])?, + self.support.read_u64(base + TIME_OFFSETS[1])?, + ], + }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/compute_payload.rs b/drivers/gpu/drm/asahi/g18/runtime/compute_payload.rs new file mode 100644 index 0000000000000..614a31167a434 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/compute_payload.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained private G18 compute fixture in its own, unpublished process VM. +//! Context-root publication, command aliases, device policy and the queue kick +//! are separate requirements. Loading this object never submits work. + +#![allow(dead_code)] + +use crate::g18::buffer::object::MappedObject; +use crate::{driver::AsahiDrmDevice, g18::fw::compute::*, g18::fixture::compute_payload as payload, + g18::fixture::compute_triangle::{TrianglePatch, TriangleResult, PATCH_COUNT}, + g18::fw::compute_state::{G18ComputeState, G18_COMPUTE_STATE_BYTES}, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{firmware::Firmware, prelude::*}; + +/// The verified CDM control stream (register 0x1a420) is a 40-byte direct +/// launch, this barrier and this terminate. Command +0xee0 (`d4e0`, the +/// G17/M5 `cdm_terminator`) is the terminate word's address. Block type is +/// bits 31:29, as Mesa agx3_cmdstream.h decodes this capture. +const CDM_LAUNCH_BYTES: usize = 40; +const CDM_BARRIER: u32 = 0x6000_0160; +const CDM_TERMINATE: u32 = 0x4000_0000; + +fn stream_words(bytes: &[u8]) -> [u32; 12] { + core::array::from_fn(|i| u32::from_le_bytes(bytes[4 * i..4 * i + 4].try_into().unwrap())) +} + +pub(crate) struct G18ComputePayloadMemory { + // Mappings and their GEM storage are released before the VM root owner. + pages: KVec, + vm: mmu::Vm, + state: [u8; G18_COMPUTE_STATE_BYTES], + output: usize, + /// `g18_compute_variant=1` replacement for command +0xee0. + cdm_terminator: Option, +} + +impl G18ComputePayloadMemory { + pub(crate) fn load(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result { + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != payload::PAGE_BYTES { + return Err(ENODEV); + } + let firmware = Firmware::request(c"asahi/g18-26a428-compute-add.bin", dev.as_ref().as_ref())?; + let reference = payload::ComputePayload::parse(firmware.data()).map_err(|_| EINVAL)?; + let vm = uat.new_t8152_compute_fixture_vm()?; + let mut pages = KVec::new(); + let mut output = None; + for index in 0..reference.page_count() { + let input = reference.page(index).ok_or(EINVAL)?; + let mut page = MappedObject::new_wc_at(dev, &vm, input.address, + payload::PAGE_BYTES, mmu::PROT_GPU_SHARED_RW)?; + page.with_bytes_mut(|bytes| { + bytes.copy_from_slice(input.bytes); + if input.address == payload::OUTPUT { + // Defense in depth: never inherit a capture's answer. + for word in bytes.chunks_exact_mut(4) { + word.copy_from_slice(&payload::POISON.to_le_bytes()); + } + } + Ok(()) + })?; + vm.translate_iova(input.address)?; + vm.translate_iova(input.address + payload::PAGE_BYTES as u64 - 1)?; + if input.address == payload::OUTPUT { output = Some(index); } + pages.push(page, GFP_KERNEL)?; + } + let state = reference.state_bytes().try_into().map_err(|_| EINVAL)?; + let mut owner = Self { pages, vm, state, output: output.ok_or(EINVAL)?, cdm_terminator: None }; + let initial = owner.output_snapshot()?; + if initial.checked != 4096 || initial.untouched != 4096 || initial.correct != 0 { + return Err(EIO); + } + match crate::g18::params::g18_compute_variant.value() { + 0 => {} + 1 => owner.empty_cdm_stream(dev)?, + 2 => owner.triangle_shader(dev, payload::fnv64(&firmware.data()[payload::HEADER_BYTES..]))?, + _ => return Err(EINVAL), + } + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), + "G18 compute payload: {} pages retained, all 4096 outputs poisoned; private VM roots unpublished\n", + owner.pages.len()); + Ok(owner) + } + + pub(crate) fn vm(&self) -> &mmu::Vm { &self.vm } + + pub(crate) fn scratch_plan(&self) -> Result { + crate::g18::fw::uma::g18_addition_scratch(&self.state).ok_or(EINVAL) + } + + pub(crate) fn apply_state(&self, c: &mut G18ComputeCommandInputs, r: &mut G18ComputeRegisterInputs) { + G18ComputeState::parse(&self.state).unwrap().apply(c, r); + } + + pub(crate) fn addition_inputs(&self) + -> Result<(G18ComputeCommandInputs, G18ComputeRegisterInputs)> { + let (mut command, registers) = payload::addition_inputs(&self.state).map_err(|_| EINVAL)?; + if let Some(terminator) = self.cdm_terminator { command.d4e0 = terminator; } + Ok((command, registers)) + } + + /// Stage8 bisection, `g18_compute_variant=1`: dispatch nothing. Only after + /// the unchanged verified load, rewrite the stream as the capture's own + /// barrier and terminate at +0/+4, zero +8..+0x2f (the old launch, + /// barrier and terminate) and point command +0xee0 at the new terminate. + /// Every other byte, page and register is unchanged; nothing is published. + fn empty_cdm_stream(&mut self, dev: &AsahiDrmDevice) -> Result { + let mut command = G18ComputeCommandInputs::default(); + let mut registers = G18ComputeRegisterInputs::default(); + self.apply_state(&mut command, &mut registers); + // g18_compute.rs emits program[1] as register 0x1a420. + let (stream, terminator) = (registers.program[1], command.d4e0); + let end = CDM_LAUNCH_BYTES + 8; + let page = self.pages.iter().position(|page| page.iova() == stream); + let patched = match page { + Some(index) if terminator == stream + end as u64 - 4 => + self.pages[index].with_bytes_mut(|bytes| { + let before = stream_words(bytes); + if before[0] >> 29 != 0 || before[10] != CDM_BARRIER + || before[11] != CDM_TERMINATE || bytes[end..].iter().any(|b| *b != 0) { + return Ok(None); + } + bytes[..end].fill(0); + bytes[..4].copy_from_slice(&CDM_BARRIER.to_le_bytes()); + bytes[4..8].copy_from_slice(&CDM_TERMINATE.to_le_bytes()); + Ok(Some((before, stream_words(bytes)))) + })?, + _ => None, + }; + let Some((before, after)) = patched else { + dev_err!(dev.as_ref(), + "G18_COMPUTE_VARIANT=1 refused: stream {:#x} terminator {:#x} is not the captured launch, barrier and terminate; nothing changed\n", + stream, terminator); + return Err(EINVAL); + }; + self.cdm_terminator = Some(stream + 4); + dev_info!(dev.as_ref(), + "G18_COMPUTE_VARIANT=1 empty stream {:#x}: words {:08x?} (were {:08x?}); command +0xee0 d4e0 {:#x} -> {:#x}; registers and all other bytes unchanged\n", + stream, after, before, terminator, stream + 4); + Ok(()) + } + + /// Validate every original site before mutating any page. All pages and + /// roots are still private; failure cannot publish a partial program. + fn triangle_shader(&mut self, dev: &AsahiDrmDevice, base_fnv: u64) -> Result { + let firmware = Firmware::request(c"asahi/g18-26a428-compute-triangle.bin", dev.as_ref().as_ref())?; + let patch = TrianglePatch::parse(firmware.data(), base_fnv).ok_or(EINVAL)?; + let mut indices = [0usize; PATCH_COUNT]; + for (i, index) in indices.iter_mut().enumerate() { + let site = patch.site(i).ok_or(EINVAL)?; + let base = site.address & !(payload::PAGE_BYTES as u64 - 1); + let offset = (site.address - base) as usize; + *index = self.pages.iter().position(|page| page.iova() == base).ok_or(EINVAL)?; + self.pages[*index].with_bytes_mut(|bytes| { + let before = bytes.get(offset..offset + site.bytes.len()).ok_or(EINVAL)?; + if payload::fnv64(before) != site.before_fnv { return Err(EINVAL); } + Ok(()) + })?; + } + for (i, index) in indices.into_iter().enumerate() { + let site = patch.site(i).ok_or(EINVAL)?; + let offset = (site.address & (payload::PAGE_BYTES as u64 - 1)) as usize; + self.pages[index].with_bytes_mut(|bytes| { + let target = bytes.get_mut(offset..offset + site.bytes.len()).ok_or(EINVAL)?; + target.copy_from_slice(site.bytes); + fence(Ordering::SeqCst); + for (j, expected) in site.bytes.iter().enumerate() { + // SAFETY: j is within this initialized, CPU-mapped slice. + if unsafe { core::ptr::read_volatile(target.as_ptr().add(j)) } != *expected { + return Err(EIO); + } + } + Ok(()) + })?; + dev_info!(dev.as_ref(), "G18_COMPUTE_VARIANT=2 patch {} address={:#x} bytes={} before_fnv={:#x} after_fnv={:#x}; VM unpublished\n", + i, site.address, site.bytes.len(), site.before_fnv, payload::fnv64(site.bytes)); + } + Ok(()) + } + + pub(crate) fn triangle_output_snapshot(&mut self) -> Result { + fence(Ordering::SeqCst); + self.pages[self.output].with_bytes_mut(|bytes| TriangleResult::inspect(bytes).ok_or(EINVAL)) + } + + /// Call only while the GPU is quiescent or after an independently verified + /// completion fence. This read does not establish completion itself. + pub(crate) fn output_snapshot(&mut self) -> Result { + fence(Ordering::SeqCst); + self.pages[self.output].with_bytes_mut(|bytes| + payload::ComputeResult::inspect(bytes).ok_or(EINVAL)) + } +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs new file mode 100644 index 0000000000000..01952223a0593 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned H1 graph, typed mapping checks and a single audited publication. +#[path = "graphics_job_application.rs"] +pub(crate) mod application; +use crate::{g18::buffer::object::MappedObject, g18::runtime::graphics_memory::G18GraphicsMemory, + g18::queue::scheduler::{G18SchedulerMemory, G18SchedulerSnapshot}, + g18::queue::graphics::{G18GraphicsQueues, G18GraphicsQueueAddresses, + G18ParameterBuffer, G18PbAddresses, G18PbPolicy, G18QueuedPass, G18PbReadback}}; +use crate::{driver::AsahiDrmDevice, g18::fw::render_params::G17pRenderParameters, + g18::address_map as va, g18::fw::render as regs, g18::fw::render_command as cmd, + g18::fw::render_assembly::{self as assembly, EncodedJob, RenderVariant}, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +pub(crate) struct G18GraphicsJob { + // Payload's context lease retracts before the additional job mappings drop. + program: G18GraphicsMemory, + aliases: KVec, + targets: KVec, + graph: MappedObject, + signals: MappedObject, + scheduler: G18SchedulerMemory, + queues: G18GraphicsQueues, + parameter_buffer: G18ParameterBuffer, + pub parameters: G17pRenderParameters, + pub controls: regs::G18RenderControls, + variant: RenderVariant, + encoded: Option, + audited: bool, +} + +impl G18GraphicsJob { + #[inline(never)] + pub(crate) fn from_payload(dev: &AsahiDrmDevice, uat: &mmu::Uat, + profile: u32, variant: RenderVariant) -> Result> { + if (variant == RenderVariant::GearsThreeFrames) != (profile == 3) { return Err(EINVAL); } + let mut program = G18GraphicsMemory::load_payload(dev, uat, profile)?; + let capture = match program.capture() { + crate::g18::fw::graphics_payload::Capture::ColorA => regs::G18Capture::ColorA, + crate::g18::fw::graphics_payload::Capture::ColorB => regs::G18Capture::ColorB, + crate::g18::fw::graphics_payload::Capture::GearsG1 => regs::G18Capture::GearsG1, + crate::g18::fw::graphics_payload::Capture::FragmentWitness => regs::G18Capture::FragmentWitness, + }; + let (mut p, mut controls) = regs::g18_capture_parameters(capture); + if crate::g18::params::g18_vdm_high_alias.value() == 1 { + let source = p.encoder; + p.encoder = program.alias_vdm_for_address_probe(source)?; + dev_info!(dev.as_ref(), "G18_VDM_ADDRESS_CONTROL source={:#x} alias={:#x} compact={:#x} bytes=16384 contents=unchanged\n", + source, va::GRAPHICS_HIGH_VDM, p.encoder); + dev_info!(dev.as_ref(), "G18_RENDER_OBJECT name=vdm-address-control va={:#x} bytes={:#x} context={}\n", + va::GRAPHICS_HIGH_VDM, va::PAGE, program.context_id()); + } + if crate::g18::params::g18_ppp_high_alias.value() == 1 { + program.alias_ppp_for_address_probe()?; + dev_info!(dev.as_ref(), "G18_PPP_ADDRESS_CONTROL records=14 pages=4 contents=unchanged vdm_changes=ppp-pointers-only\n"); + for source in va::GRAPHICS_HIGH_PPP_PAGES { + dev_info!(dev.as_ref(), "G18_RENDER_OBJECT name=ppp-address-control-{:x} va={:#x} bytes={:#x} context={}\n", + source, va::GRAPHICS_HIGH_PPP_BASE - 0x58000 + source, va::PAGE, program.context_id()); + } + } + if matches!(variant, RenderVariant::TaOnly | RenderVariant::Clear) { + program.terminate_vdm(p.encoder)?; + } + if variant == RenderVariant::Clear { p.process_empty_tiles = true; } + let plan = assembly::target_plan(&p).ok_or(EINVAL)?; + let mut aliases = KVec::new(); + let mut targets = KVec::new(); + for i in 0..plan.sizes.len() { + let address = va::GRAPHICS_TARGET_OFFSET + plan.offsets[i]; + let mut target = MappedObject::new_wc_at(dev, program.vm(), address, + plan.sizes[i], mmu::PROT_GPU_SHARED_RW)?; + if i == 2 { + target.with_bytes_mut(|bytes| + regs::initialize_g18_deflake(bytes, p.width, p.height).ok_or(EINVAL))?; + } + aliases.push(target.map_alias_at(program.vm(), va::COMPACT_BIAS + address, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + targets.push(target, GFP_KERNEL)?; + } + assembly::apply_target_plan(&mut p, &plan); + if crate::g18::params::g18_usc_statistics.value() != 0 { + if variant != RenderVariant::Color || program.context_id() >= 64 { + return Err(EINVAL); + } + // Original G18 setupSMAPPDesc: full GPU VA bits 6..42, context + // in bits 56..61, enable bit 0. These are the separately owned, + // zeroed target pages above, retained for the entire job lifetime. + let encode = |address: u64| -> Result { + if address & 63 != 0 || address & !0x7ff_ffff_ffc0 != 0 { + return Err(EINVAL); + } + Ok(address | (u64::from(program.context_id()) << 56) | 1) + }; + controls.ta.usc_statistics = encode(p.ta_status)?; + controls.fragment.usc_statistics = encode(p.fragment_status)?; + dev_info!(dev.as_ref(), "G18_USC_STATISTICS context={} ta={:#x} fragment={:#x} backing=16384-each\n", + program.context_id(), controls.ta.usc_statistics, controls.fragment.usc_statistics); + } + dev_info!(dev.as_ref(), "G18_RENDER_TARGET kind={} primary={:#x} tpc-allocation={:#x} native={:?}\n", + crate::g18::uapi::render_policy::target_kind(), plan.sizes[0], plan.sizes[1], plan.native); + let mut parameter_buffer = G18ParameterBuffer::new(dev, uat, program.vm(), G18PbAddresses { + state_high: va::GRAPHICS_PB_HIGH, graph_high: va::PB_GRAPH, + page_list_high: va::PB_PAGE_LIST_HIGH, page_list_low: va::PB_PAGE_LIST_LOW, + page_list_row: va::COMPACT_BIAS + va::PB_PAGE_LIST_LOW, + metrics_high: va::PB_METRICS_HIGH, metrics_low: va::PB_METRICS_LOW, + scene_scratch_low: va::PB_SCENE_SCRATCH, blocks_low: va::PB_BLOCKS, + discard_low: va::PB_DISCARD, user_view_delta: 0, + }, G18PbPolicy { blocks: 8, capacity: 1650, pm_154: 550, pm_15c: 1650, + pm_1d4: 0, scene_count: 80, scene: 1, slot: 0, generation: 1 })?; + parameter_buffer.alias_low_window(program.vm(), program.vm(), va::COMPACT_BIAS)?; + p.cycle = parameter_buffer.scene_cycle(); + p.record_index = parameter_buffer.metrics_record() | 1; + p.ta_hardware_buffer_id = 0; + p.usc_flist_hardware_buffer_id = 0; + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), va::GRAPHICS_JOB_HIGH, + va::GRAPHICS_JOB_BYTES, mmu::PROT_FW_PRIV_RW)?; + for vm in [uat.kernel_lower_vm(), program.vm()] { + aliases.push(graph.map_alias_at(vm, va::GRAPHICS_JOB_LOW, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + } + let signals = MappedObject::new_wc_at(dev, uat.kernel_vm(), va::GRAPHICS_SIGNALS, + va::GRAPHICS_SIGNALS_BYTES, mmu::PROT_FW_SHARED_RW)?; + let mut scheduler = G18SchedulerMemory::new_render(dev, uat)?; + scheduler.prepare(0, 0)?; + let queues = G18GraphicsQueues::new(dev, uat, G18GraphicsQueueAddresses { + entry_high: [va::GRAPHICS_ENTRY_HIGH, va::GRAPHICS_ENTRY_HIGH + 0x20000], + entry_low: [va::GRAPHICS_ENTRY_LOW, va::GRAPHICS_ENTRY_LOW + 0x20000], + graph: va::GRAPHICS_QUEUE_HIGH, + })?; + let mut job = KBox::init(try_init!(Self { program, aliases, targets, graph, signals, scheduler, queues, parameter_buffer, + parameters: p, controls, variant, encoded: None, audited: false }), GFP_KERNEL)?; + job.audit_mappings(uat)?; + if job.program.output()?.1 != 65536 { return Err(EIO); } + job.log_objects(dev); + Ok(job) + } + + pub(crate) fn vm(&self) -> &mmu::Vm { self.program.vm() } + pub(crate) fn layout_only_payload(&self) -> bool { self.program.layout_only_payload() } + pub(crate) fn context_id(&self) -> u8 { self.program.context_id() } + pub(crate) fn scheduler_record(&self) -> Result { Ok(self.scheduler.binding(0)?.record) } + pub(crate) fn queues(&mut self) -> &mut G18GraphicsQueues { &mut self.queues } + pub(crate) fn parameter_buffer(&mut self) -> &mut G18ParameterBuffer { &mut self.parameter_buffer } + #[inline(never)] + pub(crate) fn encode_h1(&mut self, dev: &AsahiDrmDevice) -> Result { + let inputs = self.h1_policy()?; + self.encode(dev, &inputs) + } + + #[inline(never)] + fn h1_policy(&mut self) -> Result> { + let context = u32::from(self.context_id()); + let inputs = crate::g18::uapi::render_policy::h1_inputs(self.program.command(), context, + &mut self.parameters, &mut self.controls); + Ok(KBox::new(inputs, GFP_KERNEL)?) + } + + pub(crate) fn pass(&self) -> Result { + Ok(G18QueuedPass { command_high: [va::GRAPHICS_JOB_HIGH, va::GRAPHICS_JOB_HIGH+0x4000], + context_id: u32::from(self.context_id()), scheduler_control: self.queues.scheduler_control(), + scheduler_index: 0, uma: cmd::G18ConfigUma { state: va::GRAPHICS_UMA_STATE, + generation: Some(0), slot: 0 }, parameter_buffer: Some(self.parameter_buffer.binding(10)), + rce: self.encoded.as_ref().ok_or(EINVAL)?.rce, stamps: [0x100;2], + event_generation: [0;2], ta_barrier_flag: crate::g18::uapi::render_policy::ta_barrier_flag() }) + } + + pub(crate) fn audit_h1(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, + pb: &G18PbReadback, uma: &[u64;16]) -> Result { + let pass = self.pass()?; + self.queues.audit_before_pass(uat, &pass)?; + if u32::from_le_bytes(pb.state[8..12].try_into().map_err(|_|EINVAL)?) != pass.context_id + || pb.counter != 1 || uma.iter().all(|&v|v==0) { return Err(EIO); } + let s = self.scheduler.snapshot(0)?; + if s.submitted!=0 || s.completed!=0 || s.linked!=0 || self.program.output()?.1!=65536 { return Err(EBUSY); } + for q in va::QUEUE_IDS { for bank in [0x4000,0x8000] { + if self.signals.read_u32(va::GRAPHICS_SIGNALS+bank+u64::from(q)*4)?!=0 {return Err(EBUSY);} + } } + for off in [0xc000,0xc008,0xc010,0xc018] { + if self.signals.read_u64(va::GRAPHICS_SIGNALS+off)?!=0 {return Err(EBUSY);} + } + let mut pages = self.audit_mappings(uat)?; + pages += self.parameter_buffer.audit_job_mappings(uat,self.vm(),va::COMPACT_BIAS)?; + // Command/RCE/target/PB/UMA graphs are owner-audited page by page. + // Fixed firmware-only objects not covered by those owners: + for (address,bytes) in [(va::GRAPHICS_SIGNALS,va::GRAPHICS_SIGNALS_BYTES), + (va::GRAPHICS_SCHEDULER_START,0x8000),(va::PB_TABLE,0x4000), + (va::UMA_TABLE,0x8000),(va::GRAPHICS_UMA_STATE,0x8000)] { + for off in (0..bytes).step_by(mmu::UAT_PGSZ) { uat.kernel_vm().translate_iova(address+off as u64)?;pages+=1; } + } + // Captured shader/VDM/PPP pages are immutable except terminate/poison. + // Validate all enabled direct register pointer inputs; bind/flag words + // and shader offsets are not mistaken for full addresses. + let p = &self.parameters; + // The register carries a compact VDM offset, not a full GPU VA. + // The low fixture alias used to hide this distinction in the audit. + let vdm_address = va::COMPACT_BIAS.checked_add(p.encoder).ok_or(EINVAL)?; + self.vm().translate_iova(vdm_address)?; + pages += 1; + for address in [p.tilemap,p.heapmeta,p.layermeta,p.tpc,p.deflake_1,p.deflake_2,p.deflake_3, + p.ta_status,p.fragment_status,p.scissor_array,p.depth_bias_array,p.aux_fb, + p.occlusion_query_base,p.depth_buffer,p.stencil_buffer,p.depth_aux_buffer,p.stencil_aux_buffer, + p.sampler_array,p.cycle,p.record_index & !1,p.load_pipeline,p.store_pipeline, + p.partial_load_pipeline,p.partial_store_pipeline] { + if address!=0 { self.vm().translate_iova(address)?; pages+=1; } + } + let inputs=self.h1_policy()?; + let f=&inputs.fragment; + // Native descriptor copies of attachment/resource and shader pointers. + // Other d0518 pairs are disabled in the admitted colour captures; the + // last pair is a packed format word rather than a GPU pointer. + for address in [f.d04e0,f.d04e8,f.d04f0,f.d0518[18][1],f.d0518[21][1],f.d0790[1],f.d07b0[1]] { + if address!=0 {self.vm().translate_iova(address)?;pages+=1;} + } + for (index,pair) in f.d0518.iter().enumerate() { + if ![18,21,22].contains(&index) && *pair!=[0;2] {return Err(ENOTSUPP);} + } + self.parameter_buffer.print_h1_digests(dev,&pb.row)?; + self.scheduler.print_render_digests(dev)?; + for (target,name) in self.targets.iter_mut().zip(assembly::TARGET_NAMES) { + target.with_bytes_mut(|bytes| { + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name={} fnv={:#018x}\n",name,crate::g18::fw::graphics_payload::fnv64(bytes));Ok(()) + })?; + } + if self.variant==RenderVariant::Clear { + let digest=self.program.terminated_digest(self.parameters.encoder)?; + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=terminated-vdm-page fnv={:#018x}\n",digest); + } + let expected=self.encoded.as_ref().ok_or(EINVAL)?.digests; + self.graph.with_bytes_mut(|bytes| { + let ta=&bytes[..cmd::G18_TA_COMMAND_BYTES]; + let fr=&bytes[assembly::FRAGMENT_OFFSET..assembly::FRAGMENT_OFFSET+cmd::G18_3D_COMMAND_BYTES]; + let f=crate::g18::fw::graphics_payload::fnv64; + let actual=[f(ta),f(fr),f(&ta[0x60..0x60+91*12]),f(&fr[0xa0..0xa0+92*12]), + f(&fr[0x7c0..0x7c0+17*12]),f(&fr[0xee0..0xee0+24*12]),f(&fr[0x1600..0x1600+12*12])]; + if actual!=expected {return Err(EIO);} + for (name,digest) in assembly::DIGEST_NAMES.iter().zip(actual) { + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name={} fnv={:#018x}\n",name,digest); + } + Ok(()) + })?; + let mut row=[0u8;128]; for (out,word) in row.chunks_exact_mut(8).zip(uma) {out.copy_from_slice(&word.to_le_bytes());} + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=uma-row fnv={:#018x}\n", crate::g18::fw::graphics_payload::fnv64(&row)); + dev_info!(dev.as_ref(), "G18_RENDER_AUDIT pb=match uma=match slots=0,1 scheduler=bound stamps=zero timestamps=zero poison=65536 translations={} policy=captured-M5-H1\n",pages); + self.audited = true; + Ok(()) + } + + pub(crate) fn publish_h1(&mut self) -> Result<[u64;2]> { + if !self.audited {return Err(EINVAL);} + self.program.publish()?; + self.scheduler.publish_kick(0)?; + self.scheduler.publish_kick(0)?; + self.audited=false; + Ok(self.program.roots()) + } + + pub(crate) fn stamps(&mut self) -> Result<[u32;2]> { + Ok([self.signals.read_u32(va::GRAPHICS_SIGNALS+0x4000+9*4)?, + self.signals.read_u32(va::GRAPHICS_SIGNALS+0x4000+10*4)?]) + } + /// A firmware recovery ended this frame after its queues consumed it, so + /// no stage will write its stamps. Publish them from the host; a stage + /// that already wrote its own must have written exactly `stamp`. + pub(crate) fn publish_recovered_stamps(&mut self, previous: u32, stamp: u32) -> Result { + for slot in [9u64,10] { + let address=va::GRAPHICS_SIGNALS+0x4000+slot*4; + match self.signals.read_u32(address)? { + value if value==stamp => {} + value if value==previous => self.signals.write_u32(address,stamp)?, + _ => return Err(EIO), + } + } + fence(Ordering::SeqCst); + Ok(()) + } + pub(crate) fn pixels(&mut self) -> Result<([u32;8],u32,u32)> { + self.program.verify_pixels(self.variant==RenderVariant::Clear) + } + pub(crate) fn capture_fields(&self) -> (cmd::G18RenderCommandInputs, usize) { + let (mut inputs, open) = crate::g18::fw::render_passthrough::capture_fields(self.program.command()); + let pb = self.parameter_buffer.command_fields(); + inputs.ta.pb_slot = pb.pb_slot; inputs.ta.pm_1d4 = pb.pm_1d4; + inputs.ta.pb_state = pb.pb_state; inputs.fragment.pm_state = pb.pb_state; + inputs.ta.pm_scene = pb.pm_scene; inputs.fragment.pm_scene = pb.pm_scene; + inputs.ta.pm_b8 = pb.pm_b8; inputs.fragment.pm_b8 = pb.pm_b8; + (inputs, open) + } + + /// Caller has allocated/retained PB and pool owners and filled all native + /// fields. In particular a default resource word is not an admitted policy. + #[inline(never)] + pub(crate) fn encode(&mut self, _dev: &AsahiDrmDevice, inputs: &cmd::G18RenderCommandInputs) -> Result { + if self.encoded.is_some() || self.controls.ta.resource_word == 0 + || self.controls.fragment.resource_word == 0 + || inputs.context_id != u32::from(self.context_id()) + || inputs.scheduler_record != self.scheduler_record()? + || inputs.ta.pb_state == 0 || inputs.ta.pb_state != inputs.fragment.pm_state + || inputs.ta.uma_state == 0 || inputs.ta.uma_state != inputs.fragment.uma_state { + return Err(EINVAL); + } + let mut workspace = KBox::::new_uninit(GFP_KERNEL)?; + // SAFETY: Workspace contains only integer scalars and integer arrays; + // zero is valid for every field. Initialize the allocation in place. + unsafe { core::ptr::write_bytes(workspace.as_mut_ptr(), 0, 1); } + let mut workspace = unsafe { workspace.assume_init() }; + let encoded = self.graph.with_bytes_mut(|bytes| { + let (ta_page, rest) = bytes.split_at_mut(assembly::FRAGMENT_OFFSET); + assembly::encode_into((&mut ta_page[..cmd::G18_TA_COMMAND_BYTES]).try_into().map_err(|_| EINVAL)?, + (&mut rest[..cmd::G18_3D_COMMAND_BYTES]).try_into().map_err(|_| EINVAL)?, + &self.parameters, &self.controls, inputs, &mut workspace).map_err(|_| EINVAL) + })?; + self.encoded = Some(encoded); + fence(Ordering::SeqCst); + Ok(()) + } + + fn audit_mappings(&self, uat: &mmu::Uat) -> Result { + let mut count = self.program.audit()?; + for target in self.targets.iter() { + for offset in (0..target.logical_size).step_by(mmu::UAT_PGSZ) { + let low = target.iova() + offset as u64; + if self.vm().translate_iova(low)? != self.vm().translate_iova(va::COMPACT_BIAS + low)? { + return Err(EFAULT); + } + count += 1; + } + } + for offset in (0..va::GRAPHICS_JOB_BYTES).step_by(mmu::UAT_PGSZ) { + let pa = uat.kernel_vm().translate_iova(va::GRAPHICS_JOB_HIGH + offset as u64)?; + for vm in [uat.kernel_lower_vm(), self.vm()] { + if vm.translate_iova(va::GRAPHICS_JOB_LOW + offset as u64)? != pa { return Err(EFAULT); } + } + count += 1; + } + Ok(count) + } + + pub(crate) fn scheduler_snapshot(&mut self) -> Result { self.scheduler.snapshot(0) } + pub(crate) fn output(&mut self) -> Result<(u64, u32)> { self.program.output() } + + fn log_objects(&self, dev: &AsahiDrmDevice) { + for (i, target) in self.targets.iter().enumerate() { + dev_info!(dev.as_ref(), "G18_RENDER_OBJECT name={} va={:#x} bytes={:#x} context={}\n", + assembly::TARGET_NAMES[i], target.iova(), target.logical_size, self.context_id()); + } + for (name, address, bytes, context) in [ + ("queue-state", va::GRAPHICS_QUEUE_HIGH, 0x8000, 0), + ("ta-entry", va::GRAPHICS_ENTRY_HIGH, 0x20000, 0), + ("3d-entry", va::GRAPHICS_ENTRY_HIGH + 0x20000, 0x20000, 0), + ("ta-command", va::GRAPHICS_JOB_HIGH, cmd::G18_TA_COMMAND_BYTES, 0), + ("3d-command", va::GRAPHICS_JOB_HIGH + 0x4000, cmd::G18_3D_COMMAND_BYTES, 0), + ("signals", va::GRAPHICS_SIGNALS, va::GRAPHICS_SIGNALS_BYTES, 0), + ("scheduler", va::GRAPHICS_SCHEDULER_START + 0x4000, 0x2400, 0), + ("submitted", va::GRAPHICS_SCHEDULER_START, 0x90, 0), + ("pb-table", va::PB_TABLE, 0x4000, 0), ("uma-table", va::UMA_TABLE, 0x8000, 0), + ("pb-state", va::GRAPHICS_PB_HIGH, 0x4000, 0), + ("pb-graph", va::PB_GRAPH, 0x8000, 0), + ("pb-page-list", va::PB_PAGE_LIST_LOW, va::PB_PAGE_LIST_BYTES, self.context_id()), + ("pb-metrics", va::PB_METRICS_LOW, 0x4000, self.context_id()), + ("pb-scene-scratch", va::PB_SCENE_SCRATCH, 0x4000, self.context_id()), + ("pb-discard", va::PB_DISCARD, 0x8000, self.context_id()), + ("pb-blocks", va::PB_BLOCKS, 8*0x80000, self.context_id()), + ("uma-state", va::GRAPHICS_UMA_STATE, 0xa0, 0), + ("uma-counter", va::GRAPHICS_UMA_COUNTER, 8, 0), + ("uma-list", va::GRAPHICS_UMA_LIST, 0x2000, 0), + ("pixels", crate::g18::fw::graphics_payload::OUTPUT, 0x40000, self.context_id()), + ] { + dev_info!(dev.as_ref(), "G18_RENDER_OBJECT name={} va={:#x} bytes={:#x} context={}\n", + name, address, bytes, context); + } + for index in 0..crate::g18::uapi::render_policy::uma_blocks() { + dev_info!(dev.as_ref(), "G18_RENDER_OBJECT name=uma-block-{} va={:#x} bytes=0x100000 context={}\n", + index,crate::g18::uapi::render_policy::block_base()+u64::from(index)*va::POOL_BLOCK_STRIDE,self.context_id()); + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs new file mode 100644 index 0000000000000..3eb58cce499b9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Application lifetime around the H1 allocation graph. The fixture VM stays +//! unpublished storage; commands are rebuilt entirely from Mesa parameters. +use super::*; +use crate::{g18::uapi::application::G18ApplicationRender,g18::uapi::application_runtime::{self as g18_application_runtime,Frame}}; + +pub(crate) struct ApplicationJob { + pub(crate) job: KBox, + storage: mmu::Vm, + user_aliases: KVec, + pub(crate) pool_aliases: KVec, + guard: Option, + scene: [u8;0x80], + generation: u32, + retired: bool, + /// Frames of the current client VM, counting this one. + client_frames: u32, + /// The current render's adapter-derived private-memory selection. + scratch: Option, +} + +impl ApplicationJob { + pub(crate) fn new(mut job: KBox) -> Result> { + let storage=job.vm().clone(); + let scene=job.parameter_buffer.application_scene(None)?; + // Remove the fixture-only aliases before target growth. The original + // payload and PB backing stay retained but no capture bytes execute. + job.aliases.clear(); + Ok(KBox::new(Self {job,storage,user_aliases:KVec::new(),pool_aliases:KVec::new(), + guard:None,scene,generation:0,retired:true,client_frames:0,scratch:None},GFP_KERNEL)?) + } + pub(crate) fn frame(&self) -> Frame {Frame::new(self.generation).expect("admitted frame")} + /// The last frame kicked to both graphics queues (0 before the first). + pub(crate) fn submitted_generation(&self) -> u32 {self.generation} + pub(crate) fn quarantine(&mut self) { + if let Some(guard)=&self.guard {guard.quarantine();} + if let Some(status)=self.job.vm().status(){status.record(EIO);} + } + pub(crate) fn retired(&mut self) {self.retired=true;self.guard=None;} + /// One of the first frames of its client: logged even when job logging + /// is sparse, so every isolated client leaves a kick/retirement pair. + pub(crate) fn traced(&self) -> bool {self.client_frames<=3} + + #[inline(never)] + pub(crate) fn prepare(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, + vm: &mmu::Vm, render: &G18ApplicationRender) -> Result { + if !self.retired {return Err(EBUSY);} + let frame=Frame::new(self.generation.checked_add(1).ok_or(EOVERFLOW)?).ok_or(EOVERFLOW)?; + let plan=assembly::target_plan(&render.parameters).ok_or(EINVAL)?; + // Growth retains each object's original IOVA. Reject a changed target + // policy before touching mappings rather than encode unowned addresses. + if self.job.targets.len()!=plan.offsets.len() + || self.job.targets.iter().zip(plan.offsets) + .any(|(target,offset)|target.iova()!=va::GRAPHICS_TARGET_OFFSET+offset) { + return Err(EINVAL); + } + let guard=vm.retain_t8140_job()?; + let new_client=!self.job.program.matches_vm(vm); + let remap=self.generation==0 || new_client + || self.job.targets.iter().zip(plan.sizes).any(|(target,size)|target.logical_size Result { + let inputs=self.application_policy(frame)?; + self.job.encode(dev,&inputs) + } + + // Return the heap owner before entering the register builders. Otherwise + // LLVM keeps the ~3.6 KiB command-input construction frame beneath them. + #[inline(never)] + fn application_policy(&mut self, frame: Frame) -> Result> { + let context=u32::from(self.job.context_id()); + let scratch=self.scratch; + let job=&mut *self.job; + let mut inputs=KBox::new(g18_application_runtime::inputs(&mut job.parameters,&mut job.controls,context,frame),GFP_KERNEL)?; + // A render whose programs carry a thread-private frame takes both + // launch pairs from its own plan; the TA/3D pool stays the owned one. + // The adapter admitted it only at one sample, so no 4x override exists. + if let Some(selection)=scratch { + if job.controls.fragment_sample_control!=0 {return Err(EINVAL);} + job.controls.ta.resource_word=selection.stages[0].resource_word(); + job.controls.fragment.resource_word=selection.stages[1].resource_word(); + } + let pb=self.job.parameter_buffer.command_fields(); + inputs.ta.pb_slot=pb.pb_slot;inputs.ta.pm_1d4=pb.pm_1d4; + inputs.ta.pb_state=pb.pb_state;inputs.fragment.pm_state=pb.pb_state; + inputs.ta.pm_scene=pb.pm_scene;inputs.fragment.pm_scene=pb.pm_scene; + inputs.ta.pm_b8=pb.pm_b8;inputs.fragment.pm_b8=pb.pm_b8; + inputs.scheduler_record=self.job.scheduler_record()?; + Ok(inputs) + } + + pub(crate) fn pass(&self) -> Result { + let mut pass=self.job.pass()?;pass.stamps=[self.frame().stamp();2];Ok(pass) + } + + pub(crate) fn audit(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, + pb: &G18PbReadback, uma: &[u64;16], mut pages: usize) -> Result { + let frame=self.frame();let pass=self.pass()?; + if frame.generation==1 {self.job.queues.audit_before_pass(uat,&pass)?;} + if pb.counter!=frame.generation || uma.iter().all(|v|*v==0) + || u32::from_le_bytes(pb.state[8..12].try_into().map_err(|_|EINVAL)?)!=pass.context_id {return Err(EIO);} + let scheduler=self.job.scheduler.snapshot(0)?; + if scheduler.submitted!=frame.submitted()-2 || scheduler.completed!=frame.submitted()-2 + || scheduler.linked!=0 || self.job.stamps()?!=[frame.previous_stamp();2] {return Err(EBUSY);} + for target in &self.job.targets { + for off in (0..target.logical_size).step_by(mmu::UAT_PGSZ) { + let low=target.iova()+off as u64;let pa=self.storage.translate_iova(low)?; + if self.job.vm().translate_iova(low)?!=pa || self.job.vm().translate_iova(low+va::COMPACT_BIAS)?!=pa {return Err(EFAULT);}pages+=1; + } + } + pages+=self.job.parameter_buffer.audit_job_mappings(uat,self.job.vm(),va::COMPACT_BIAS)?; + for off in (0..va::GRAPHICS_JOB_BYTES).step_by(mmu::UAT_PGSZ) { + let pa=uat.kernel_vm().translate_iova(va::GRAPHICS_JOB_HIGH+off as u64)?; + for vm in [uat.kernel_lower_vm(),self.job.vm()] { + if vm.translate_iova(va::GRAPHICS_JOB_LOW+off as u64)?!=pa {return Err(EFAULT);} + }pages+=1; + } + for (address,bytes) in [(va::GRAPHICS_SIGNALS,va::GRAPHICS_SIGNALS_BYTES), + (va::GRAPHICS_SCHEDULER_START,0x8000),(va::PB_TABLE,0x4000),(va::UMA_TABLE,0x8000)] { + for off in (0..bytes).step_by(mmu::UAT_PGSZ) {uat.kernel_vm().translate_iova(address+off as u64)?;pages+=1;} + } + let expected=self.job.encoded.as_ref().ok_or(EINVAL)?.digests; + self.job.graph.with_bytes_mut(|out| { + if crate::g18::fw::graphics_payload::fnv64(&out[..cmd::G18_TA_COMMAND_BYTES])!=expected[0] + || crate::g18::fw::graphics_payload::fnv64(&out[0x4000..0x4000+cmd::G18_3D_COMMAND_BYTES])!=expected[1] {return Err(EIO);}Ok(()) + })?; + if frame.generation<=3 { + for (name,digest) in assembly::DIGEST_NAMES.iter().zip(expected) { + dev_info!(dev.as_ref(),"G18_APP_DIGEST frame={} name={} fnv={:#018x}\n",frame.generation,name,digest); + } + dev_info!(dev.as_ref(),"G18_APP_AUDIT frame={} pb=resident uma=match translations={} previous_stamp={:#x}\n",frame.generation,pages,frame.previous_stamp()); + } + if frame.generation==1 { + self.job.parameter_buffer.print_h1_digests(dev,&pb.row)?; + self.job.scheduler.print_render_digests(dev)?; + } + self.job.audited=true;Ok(()) + } + + pub(crate) fn publish(&mut self) -> Result<[u64;2]> {self.job.publish_h1()} +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs new file mode 100644 index 0000000000000..3175072b950e8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained G18GFX01 v2 pages. Construction never publishes roots or work. +use crate::g18::buffer::object::MappedObject; +use crate::{driver::AsahiDrmDevice, g18::address_map as va, g18::fw::graphics_payload as payload, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{firmware::Firmware, prelude::*}; + +pub(crate) struct G18GraphicsMemory { + // Root lease retracts before aliases/backing can drop. The live session + // retains the whole job even after faults; this order also protects errors + // during construction before publication. + context: mmu::G18ExecutionContext, + aliases: KVec, + vdm_alias: Option<(u64, u64)>, + ppp_aliased: bool, + pages: KVec, + capture: payload::Capture, + body_digest: u64, + command_digest: u64, + command: KVec, +} + +impl G18GraphicsMemory { + pub(crate) fn load_payload(dev: &AsahiDrmDevice, uat: &mmu::Uat, profile: u32) -> Result { + let name = match profile { + 1 => c"asahi/g18-26a428-color-a-v2.bin", + 2 => c"asahi/g18-26a428-color-b-v2.bin", + 3 => c"asahi/g18-26a428-gears-g1-v2.bin", + 4 => c"asahi/g18-26a428-fragment-witness-v2.bin", + _ => return Err(EINVAL), + }; + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != payload::PAGE_BYTES { return Err(ENODEV); } + let firmware = match Firmware::request(name, dev.as_ref().as_ref()) { + Ok(firmware) => firmware, + // Stage 10 executes no capture bytes: every application command + // is rebuilt from Mesa's parameters. It needs only the colour-a + // page layout, which is mapped here with zeroed contents. + Err(e) if e == ENOENT && profile == 1 + && crate::g18::params::g18_initdata_test.value() == 10 => + return Self::layout_only(dev, uat), + Err(e) => return Err(e), + }; + let input = payload::GraphicsPayload::parse(firmware.data()).map_err(|_| EINVAL)?; + if input.capture() as u32 != profile { return Err(EINVAL); } + let (context, aliases, pages) = Self::map_pages(dev, uat, + (0..input.page_count()).map(|index| input.page(index).map(|page| (page.address, Some(page.bytes)))))?; + let mut command = KVec::new(); + command.extend_from_slice(input.command(), GFP_KERNEL)?; + let owner = Self { context, aliases, vdm_alias: None, ppp_aliased: false, pages, capture: input.capture(), command, + body_digest: input.body_digest(), command_digest: payload::fnv64(input.command()) }; + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), "G18_RENDER_PAYLOAD profile={} pages={} low_aliases={} body_fnv={:#018x} command_fnv={:#018x} context={} roots={:x?}; unpublished\n", + profile, owner.pages.len(), owner.aliases.len(), owner.body_digest, owner.command_digest, + owner.context_id(), owner.roots()); + Ok(owner) + } + + /// The colour-a VM layout without its capture: zeroed pages, no command. + fn layout_only(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result { + let (context, aliases, pages) = Self::map_pages(dev, uat, + payload::color_a_pages().map(|(address, _)| Some((address, None))))?; + let owner = Self { context, aliases, vdm_alias: None, ppp_aliased: false, pages, + capture: payload::Capture::ColorA, command: KVec::new(), body_digest: 0, command_digest: 0 }; + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), "G18_RENDER_PAYLOAD layout-only colour-a pages={} low_aliases={} context={} roots={:x?}; no capture, stage 10 encodes from Mesa only\n", + owner.pages.len(), owner.aliases.len(), owner.context_id(), owner.roots()); + Ok(owner) + } + + /// True when no capture was loaded: there is no H1 command to encode. + pub(crate) fn layout_only_payload(&self) -> bool { self.command.is_empty() } + + fn map_pages<'a>(dev: &AsahiDrmDevice, uat: &mmu::Uat, + sources: impl Iterator)>>) + -> Result<(mmu::G18ExecutionContext, KVec, KVec)> { + let vm = uat.new_t8152_compute_fixture_vm()?; + let context = uat.prepare_t8152_execution_context(&vm)?; + let mut pages = KVec::new(); + let mut aliases = KVec::new(); + for source in sources { + let (address, bytes) = source.ok_or(EINVAL)?; + // Captured usage flags are not PTE permissions. These private + // fixture mappings have an explicit diagnostic RW policy. + let mut page = MappedObject::new_wc_at(dev, &vm, address, + payload::PAGE_BYTES, mmu::PROT_GPU_SHARED_RW)?; + page.with_bytes_mut(|out| { + match bytes { Some(bytes) => out.copy_from_slice(bytes), None => out.fill(0) } + if (payload::OUTPUT..payload::OUTPUT + payload::OUTPUT_BYTES).contains(&address) { + out.fill(0xa5); + } + Ok(()) + })?; + if address < 0x78000 { + aliases.push(page.map_alias_at(&vm, va::COMPACT_BIAS + address, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for delta in [0, va::PAGE - 1] { + if vm.translate_iova(address + delta)? + != vm.translate_iova(va::COMPACT_BIAS + address + delta)? { + return Err(EFAULT); + } + } + } + vm.translate_iova(address)?; + vm.translate_iova(address + va::PAGE - 1)?; + pages.push(page, GFP_KERNEL)?; + } + Ok((context, aliases, pages)) + } + + pub(crate) fn vm(&self) -> &mmu::Vm { self.context.vm() } + pub(crate) fn context_id(&self) -> u8 { self.context.context_id() } + pub(crate) fn roots(&self) -> [u64; 2] { self.context.roots() } + pub(crate) fn capture(&self) -> payload::Capture { self.capture } + pub(crate) fn command(&self) -> &[u8] { &self.command } + pub(crate) fn publish(&mut self) -> Result { self.context.publish() } + /// Called only after both graphics queues have independently retired. + pub(crate) fn rebind_retired(&mut self, vm: &mmu::Vm) -> Result { self.context.rebind(vm) } + pub(crate) fn matches_vm(&self, vm: &mmu::Vm) -> bool { self.context.matches(vm) } + pub(crate) fn terminated_digest(&mut self, address: u64) -> Result { + let base=address & !(va::PAGE-1); + let page=self.pages.iter_mut().find(|p|p.iova()==base).ok_or(EFAULT)?; + page.with_bytes_mut(|bytes|Ok(payload::fnv64(bytes))) + } + + pub(crate) fn audit(&self) -> Result { + let mut checked = 0; + for page in self.pages.iter() { + for offset in [0, va::PAGE - 1] { + let address = page.iova() + offset; + self.vm().translate_iova(address)?; + if page.iova() < 0x78000 && self.vm().translate_iova(address)? + != self.vm().translate_iova(address + va::COMPACT_BIAS)? { return Err(EFAULT); } + checked += 1; + } + } + if let Some((source, target)) = self.vdm_alias { + for offset in [0, va::PAGE - 1] { + if self.vm().translate_iova(source + offset)? != + self.vm().translate_iova(target + offset)? { return Err(EFAULT); } + checked += 1; + } + } + if self.ppp_aliased { + for source in va::GRAPHICS_HIGH_PPP_PAGES { + let target = va::GRAPHICS_HIGH_PPP_BASE - 0x58000 + source; + for offset in [0, va::PAGE - 1] { + if self.vm().translate_iova(source + offset)? != + self.vm().translate_iova(target + offset)? { return Err(EFAULT); } + checked += 1; + } + } + } + Ok(checked) + } + + /// Isolate PPP address encoding using the pinned native witness. Only the + /// pointer bits in its 14 PPP records change; the pages themselves alias + /// their original backing. No shader or PPP payload is rewritten. + pub(crate) fn alias_ppp_for_address_probe(&mut self) -> Result { + if self.context.is_current() || self.ppp_aliased { return Err(EBUSY); } + if self.capture != payload::Capture::FragmentWitness || + self.vdm_alias != Some((0x18000, va::GRAPHICS_HIGH_VDM)) { return Err(EINVAL); } + const RECORDS: [(usize, u32, u32); 14] = [ + (0x00, 0x58000, 6), (0x08, 0x58018, 33), (0x10, 0x5809c, 33), + (0x34, 0x40, 5), (0x3c, 0x58120, 7), (0x44, 0x5813c, 6), + (0x4c, 0x58154, 5), (0x54, 0x58168, 7), (0x5c, 0x58184, 5), + (0x64, 0x6ac00, 10), (0x6c, 0x58198, 3), (0x74, 0x581a4, 2), + (0x7c, 0x48000, 5), (0x84, 0x581ac, 3), + ]; + self.pages.iter_mut().find(|p| p.iova() == 0x18000).ok_or(EFAULT)? + .with_bytes_mut(|bytes| { + if payload::fnv64(bytes) != 0x6f21_d211_cfaf_7af1 { return Err(EINVAL); } + for (offset, pointer, words) in RECORDS { + if bytes[offset..offset + 4] != (words << 8).to_le_bytes() || + bytes[offset + 4..offset + 8] != pointer.to_le_bytes() { return Err(EINVAL); } + } + Ok(()) + })?; + let vm = self.context.vm(); + for source in va::GRAPHICS_HIGH_PPP_PAGES { + let target = va::GRAPHICS_HIGH_PPP_BASE - 0x58000 + source; + let page = self.pages.iter_mut().find(|p| p.iova() == source).ok_or(EFAULT)?; + self.aliases.push(page.map_alias_at(vm, target, mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + } + self.ppp_aliased = true; + self.audit()?; + self.pages.iter_mut().find(|p| p.iova() == 0x18000).ok_or(EFAULT)? + .with_bytes_mut(|bytes| { + for (offset, pointer, words) in RECORDS { + let compact = va::GRAPHICS_HIGH_PPP_BASE - 0x58000 - va::COMPACT_BIAS + u64::from(pointer); + bytes[offset..offset + 4].copy_from_slice(&((words << 8) | (compact >> 32) as u32).to_le_bytes()); + bytes[offset + 4..offset + 8].copy_from_slice(&(compact as u32).to_le_bytes()); + } + Ok(()) + })?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Retain a second mapping of the unchanged native VDM page. This isolates + /// the address used for fetching commands from their contents and backing. + pub(crate) fn alias_vdm_for_address_probe(&mut self, address: u64) -> Result { + if self.context.is_current() || self.vdm_alias.is_some() { + return Err(EBUSY); + } + if self.capture != payload::Capture::FragmentWitness || address != 0x18000 || + va::GRAPHICS_HIGH_VDM & (va::PAGE - 1) != 0 { return Err(EINVAL); } + let vm = self.context.vm(); + let page = self.pages.iter_mut().find(|page| page.iova() == address).ok_or(EFAULT)?; + let alias = page.map_alias_at(vm, va::GRAPHICS_HIGH_VDM, + mmu::PROT_GPU_SHARED_RW)?; + self.aliases.push(alias, GFP_KERNEL)?; + self.vdm_alias = Some((address, va::GRAPHICS_HIGH_VDM)); + self.audit()?; + Ok(va::GRAPHICS_HIGH_VDM - va::COMPACT_BIAS) + } + + /// The caller supplies the register builder's decoded VDM stream address. + /// A standalone termination word affects no shaders or other capture data. + pub(crate) fn terminate_vdm(&mut self, address: u64) -> Result { + if self.context.is_current() || address & 3 != 0 { return Err(EBUSY); } + let address = if (va::COMPACT_BIAS..va::COMPACT_BIAS + 0x78000).contains(&address) { + address - va::COMPACT_BIAS + } else { address }; + let base = address & !(va::PAGE - 1); + let page = self.pages.iter_mut().find(|page| page.iova() == base).ok_or(EFAULT)?; + page.write_u32(address, 6 << 29)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Output digest is observation only, never a completion predicate. + pub(crate) fn verify_pixels(&mut self, clear: bool) -> Result<([u32;8],u32,u32)> { + let mut sha = crate::g18::fixture::render_pixels::Pixels::new(); + for index in 0..16 { + let address = payload::OUTPUT + index * va::PAGE; + let page = self.pages.iter_mut().find(|p|p.iova()==address).ok_or(EFAULT)?; + page.with_bytes_mut(|bytes| { + if !sha.page(bytes) {return Err(EINVAL);} Ok(()) + })?; + } + let poison=sha.poison; let background=sha.background; + let digest=sha.finish().ok_or(EINVAL)?; + if poison!=0 || (clear && background!=65536) + || (!clear && (background!=47104 || digest!=crate::g18::fixture::render_pixels::COLOR)) {return Err(EIO);} + Ok((digest,background,poison)) + } + + /// Output digest is observation only, never a completion predicate. + pub(crate) fn output(&mut self) -> Result<(u64, u32)> { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + let mut poison = 0; + let mut count = 0; + for page in self.pages.iter_mut().filter(|p| + (payload::OUTPUT..payload::OUTPUT + payload::OUTPUT_BYTES).contains(&p.iova())) { + page.with_bytes_mut(|bytes| { + for word in bytes.chunks_exact(4) { + // SAFETY: aligned four-byte word inside retained WC RAM. + let value = unsafe { core::ptr::read_volatile(word.as_ptr().cast::()) }; + if value == 0xa5a5_a5a5 { poison += 1; } + for b in value.to_le_bytes() { hash = (hash ^ u64::from(b)).wrapping_mul(0x100_0000_01b3); } + count += 1; + } + Ok(()) + })?; + } + if count != 65536 { return Err(EFAULT); } + fence(Ordering::Acquire); + Ok((hash, poison)) + } +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/mod.rs b/drivers/gpu/drm/asahi/g18/runtime/mod.rs new file mode 100644 index 0000000000000..d54b26bb187df --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 job storage: graphics and compute job memory and encoders. + +#[allow(dead_code)] +pub(crate) mod compute; +#[allow(dead_code)] +pub(crate) mod compute_payload; +#[allow(dead_code)] +pub(crate) mod graphics_job; +#[allow(dead_code)] +pub(crate) mod graphics_memory; diff --git a/drivers/gpu/drm/asahi/g18/uapi/application.rs b/drivers/gpu/drm/asahi/g18/uapi/application.rs new file mode 100644 index 0000000000000..f3de19433a52a --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/application.rs @@ -0,0 +1,599 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 application path: one decoded Mesa render command -> G18 job inputs. +//! +//! This is the DRM counterpart of `g18_render::g18_capture_parameters`. +//! Where the capture helper inverts Apple's native gears/colour descriptors, +//! this module starts from the UAPI `drm_asahi_cmd_render` that Mesa's G18G +//! (M5-path) Gallium driver submits and produces: +//! +//! - `G17pRenderParameters` with every user field translated by the shared +//! `g17_uapi::translate_render_command` (the M5-proven parser and range +//! checks) and the G18 adaptations below; +//! - `G18RenderControls` from `g18_render::g18_default_controls`; +//! - every job-owned field left at zero (see [`job_owned_fields_clear`]) for +//! the job (`G18RenderJobSink`) to allocate and fill. +//! +//! G18 adaptations, each against a named source: +//! - VDM stream: the UAPI carries the full GART-alias VA; `g18_render` takes +//! the native compact view (the render-register notes, "Kernel-owned inputs"), +//! exactly Mesa's `agx_m5_vdm_compact` (`agx_m5_va.h:34-41`). +//! - aux_fb_flags 0x30c000: indexed multi-draw, the gears-g1 value +//! (`g18_render.rs:490-494`), where M5 used 0x10c000 (`m5_drm.rs:251`). +//! - Main-pass tile flags 0xc00000 (native +8d2/+8d3): present in all three +//! G18 captures (`g18_render.rs:489`, tile_config 0xc10280), dropped by the +//! partial helper (`g18_render.rs:396`). The UAPI cannot express them. +//! - Single-layer ZLS array strides are zeroed, as on M5 (`m5_drm.rs:238-241`). +//! - Uncompressed Z16 uses the shared range-checked ZLS format and preserves +//! ISP bit 18. AGXMetalG18G selects that bit for Depth16Unorm at +//! 0x2196383e8..0x219638400 (the indirect DBIAS/PBE notes, section 2). +//! - msaa163, only with `g18_admit_msaa4=1`: the native 4x render set +//! (`is_msaa4_command`) is admitted with the native sample control +//! 0x40000; every other multisampled command keeps its refusal. +//! - Native682 layered Z32/S8 uses cacheline or page array strides and +//! independently ranged metadata. Single-layer compressed stencil retains +//! its previous inert behavior. +//! `isp_bgobjvals` 0x300 is re-asserted by the register builder; its low +//! byte carries the application's stencil clear value. +//! +//! Pure: no allocation, MMIO, firmware or VM access. The job boundary at the +//! end of the file is the only kernel-typed item (a host parity +//! check compiles it against a stub `kernel` crate). + +use crate::{g18::fw::render_params::G17pRenderParameters, g18::uapi::command as g17_uapi, g18::fw::render as g18_render, g18::fw::vm_layout as m5_vm}; + +/// Largest framebuffer side the target layouts and merge coefficients encode. +/// Whether a target fits is the job's own target plan +/// (`g18_render_assembly::target_plan`), which admission also requires: its +/// TPC slot is 0x20000 bytes, which the conformance FBO surface (64 x +/// GL_MAX_RENDERBUFFER_SIZE = 64x16384, 4x128 macrotiles) fills exactly. +pub(crate) const G18_MAX_RENDER_DIMENSION: u32 = 16384; +/// Indexed multi-draw auxiliary framebuffer flags (gears-g1 capture). +pub(crate) const G18_AUX_FB_FLAGS: u64 = 0x30c000; +/// Native main-pass tile flags (+8d2/+8d3), witnessed in colour-a/b and gears-g1. +pub(crate) const G18_MAIN_TILE_FLAGS: u64 = 0xc0_0000; +/// One-sample PPP multisample control; the only value `g18_render` admits. +pub(crate) const G18_MULTISAMPLE_CONTROL: u64 = 0x88; + +// drm_asahi_render_flags (include/uapi/drm/asahi_drm.h:859-895). +const RENDER_PROCESS_EMPTY_TILES: u32 = 1 << 1; +const RENDER_RSRC_SPEC_HI: u32 = 1 << 4; +const RENDER_DBIAS_IS_INT: u32 = 1 << 18; +const G18_RENDER_FLAGS: u32 = + RENDER_PROCESS_EMPTY_TILES | RENDER_RSRC_SPEC_HI | RENDER_DBIAS_IS_INT; +// ZLS control: stencil load (14) / store (18), stencil compression (4, 8), +// depth load (15) / store (19), depth format (25-26: 0 = Z32F, 2 = Z16). +const ZLS_STENCIL_TRANSFER: u64 = (1 << 14) | (1 << 18); +const ZLS_STENCIL_COMPRESSION: u64 = (1 << 4) | (1 << 8); +const ZLS_DEPTH_TRANSFER: u64 = (1 << 15) | (1 << 19); +const ZLS_DEPTH_FORMAT: u64 = 3 << 25; +/// Upper ISP_BGOBJVALS bits every Mesa backend seeds (`agx_pipe.c:1333`). +const ISP_BGOBJVALS_BASE: u32 = 0x300; +/// Width of the compact VDM view (`agx_m5_va.h:37`, `g17_render.rs:88`). +const VDM_COMPACT_WINDOW: u64 = 1 << 39; + +/// Why a render is outside what the G18 job can do yet. `Invalid` is a +/// malformed or unmapped command (EINVAL); every other variant is a valid +/// command outside the admitted envelope (ENOTSUPP). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G18ApplicationRefusal { + Invalid(g17_uapi::UapiTranslateError), + Flags(u32), + /// Vertex or fragment scratch helper program (spilling shaders). + HelperProgram, + Dimensions { width: u16, height: u16 }, + Layers(u16), + Samples(u8), + Utile(u8, u8), + MultisampleControl(u64), + Samplers { heap: u64, count: u16 }, + StencilTransfer(u64), + StencilClear(u32), + Depth(g17_uapi::UapiTranslateError), + DepthFormat(u64), + VdmCompact(u64), +} + +impl G18ApplicationRefusal { + /// True when the command itself is malformed rather than out of envelope. + pub(crate) fn is_invalid(&self) -> bool { + matches!(self, Self::Invalid(_)) + } +} + +/// Render-target geometry the job must plan and allocate for this command. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18TargetRequest { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) layers: u16, + pub(crate) utile: (u8, u8), + pub(crate) samples: u8, +} + +/// Everything the job receives for one admitted render. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18ApplicationRender { + /// User fields translated; every job-owned field is zero. + pub(crate) parameters: G17pRenderParameters, + /// `g18_default_controls(¶meters)`; job-owned control fields are zero. + pub(crate) controls: g18_render::G18RenderControls, + pub(crate) target: G18TargetRequest, + /// VDM stream as submitted (full VA in the submitting VM). The compact + /// form is `parameters.encoder`; the job needs both views (risk #1). + pub(crate) vdm_base: u64, + /// Attachment ranges Mesa declared (render targets and PBE images). + pub(crate) vertex_attachments: g17_uapi::UapiAttachmentList, + pub(crate) fragment_attachments: g17_uapi::UapiAttachmentList, + /// An inert compressed stencil plane was removed (see module doc). + pub(crate) stencil_stripped: bool, + /// UAPI render flags: only PROCESS_EMPTY_TILES and RSRC_SPEC_HI remain. + pub(crate) flags: u32, + /// TA/3D private-memory selection derived from the render's + /// `drm_asahi_g18_render_resources` suffix by the adapter; `None` keeps + /// the fixed first-use policy (`g18_render_policy::resource_pairs`). + pub(crate) scratch: Option, +} + +/// User pointers the G18 3D command carries as native descriptor copies +/// (render-cmd `G183dCommandFields`), because the G18 fragment program drops +/// their registers 0x15101/0x15109 (`m5_render.rs:306`). The slots are the +/// ones G17P fills from the same parameters (`g17_render.rs:1381-1383`); the +/// gears-g1 native command carries its depth-bias and scissor arrays there. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18DescriptorPointers { + /// d04e0 -> 3D +0x1d20: ISP depth-bias array. + pub(crate) d04e0: u64, + /// d04e8 -> 3D +0x1d30: ISP scissor array. + pub(crate) d04e8: u64, + /// d04f0 -> 3D +0x1d40 / register 0x15311: aligned occlusion query heap. + pub(crate) d04f0: u64, +} + +impl G18ApplicationRender { + /// The job must copy these into its 3D command fields; the register + /// programs built from `parameters` do not contain them. + pub(crate) fn descriptor_pointers(&self) -> G18DescriptorPointers { + G18DescriptorPointers { + d04e0: self.parameters.depth_bias_array, + d04e8: self.parameters.scissor_array, + d04f0: self.parameters.occlusion_query_base, + } + } +} + +/// Whether every job-owned field is still zero. The translation leaves them +/// zero; the job must assign each before building registers: +/// - parameters: context_base, tilemap, layermeta, heapmeta, tpc, +/// deflake_1/2/3, ta_status, fragment_status, aux_fb, cycle, record_index, +/// lifecycle, fragment_lifecycle, ta_hardware_buffer_id, +/// usc_flist_hardware_buffer_id, parameter_buffer and the four timestamps +/// (which `g18_render` requires to stay zero); +/// - controls: resource_word, work_key and usc_statistics of both stages, +/// and the optional TA cluster fields (zero means absent). +/// +/// Sources: the render-register notes, "Kernel-owned inputs", the H1 job +/// (`g18_graphics_job.rs:61-74`, `g18_render_assembly.rs:34-47`) and its +/// policy (`g18_render_policy.rs:25-28`). +pub(crate) fn job_owned_fields_clear( + p: &G17pRenderParameters, + c: &g18_render::G18RenderControls, +) -> bool { + let parameters = [p.context_base, p.tilemap, p.layermeta, p.heapmeta, p.tpc, + p.deflake_1, p.deflake_2, p.deflake_3, p.ta_status, p.fragment_status, + p.aux_fb, p.cycle, p.record_index, p.lifecycle, p.fragment_lifecycle, + p.parameter_buffer, p.ta_timestamp_start, p.ta_timestamp_end, + p.fragment_timestamp_start, p.fragment_timestamp_end]; + let controls = [c.ta.resource_word, c.fragment.resource_word, c.ta.work_key, + c.fragment.work_key, c.ta.usc_statistics, c.fragment.usc_statistics, + c.ta_cluster_tilemap, c.ta_cluster_layermeta]; + parameters.iter().chain(controls.iter()).all(|&v| v == 0) + && p.ta_hardware_buffer_id == 0 && p.usc_flist_hardware_buffer_id == 0 + && c.ta_cluster_meta == [0; 4] && c.ta_cluster_control == [0; 3] +} + +/// Internal state handed to `translate_render_command`: the M5 reference +/// profile's role (`m5_drm.rs:234`) with every job-owned field zeroed and +/// G18 policy applied. `translate_render_command` overwrites all user fields. +pub(crate) fn g18_application_base() -> G17pRenderParameters { + let mut p = G17pRenderParameters::source_rsrc8(); + p.context_base = 0; + p.tilemap = 0; + p.layermeta = 0; + p.heapmeta = 0; + p.tpc = 0; + p.deflake_1 = 0; + p.deflake_2 = 0; + p.deflake_3 = 0; + p.ta_status = 0; + p.fragment_status = 0; + p.aux_fb = 0; + p.cycle = 0; + p.record_index = 0; + p.lifecycle = 0; + p.fragment_lifecycle = 0; + p.ta_hardware_buffer_id = 0; + p.usc_flist_hardware_buffer_id = 0; + p.parameter_buffer = 0; + p.gpc_perf_state_map = 0; + p.gpc_perf_state_map_low = 0; + p.gpc_perf_state_control = 0; + p.ta_timestamp_start = 0; + p.ta_timestamp_end = 0; + p.fragment_timestamp_start = 0; + p.fragment_timestamp_end = 0; + // G18 policy. aux_fb_page_count keeps the value every M5/G18 capture + // carries (0x100000, `g17_render.rs:421`). + p.aux_fb_flags = G18_AUX_FB_FLAGS; + p.samples = 1; + p.absolute_pointers = false; + p.bg_resource_prefix = 0; + p.native_pm_bytes = false; + p.native_ta_registers = 0; + p +} + +/// Mesa's compact VDM view (`agx_m5_vdm_compact`). +pub(crate) fn g18_vdm_compact(vdm_base: u64) -> Option { + let offset = vdm_base.checked_sub(m5_vm::DRIVER_BASE)?; + if vdm_base & 3 != 0 || offset >= VDM_COMPACT_WINDOW { + return None; + } + Some(offset) +} + +/// msaa163: whether `command` is exactly the native 4x render set (see +/// `g18_render::G18_MSAA4_UTILE`): four samples, 32x16 utiles, sample +/// positions 0xeaa26e26. Native331 adds sixteen-byte samples +/// to the eight-byte profile, retaining its geometry. Native315 adds +/// D32/S8 attachments to that render geometry. Gallium currently allocates +/// them uncompressed; compressed multisample metadata remains unadmitted. +fn is_multisample_command(command: &g17_uapi::UapiRenderCommand) -> bool { + let (utile, positions) = match command.samples { + 2 => (g18_render::G18_MSAA2_UTILE, g18_render::G18_MSAA2_MULTISAMPLE_CONTROL), + 4 => (g18_render::G18_MSAA4_UTILE, g18_render::G18_MSAA4_MULTISAMPLE_CONTROL), + _ => return false, + }; + (command.utile_width, command.utile_height) == utile + && command.multisample_control == positions + && matches!(command.sample_size, + g18_render::G18_MSAA4_SAMPLE_SIZE | g18_render::G18_MSAA_WIDE_SAMPLE_SIZE + | g18_render::G18_MSAA_MRT_SAMPLE_SIZE) + && (command.layers == 1 || + ((2..=2048).contains(&command.layers) + && matches!(command.sample_size, + g18_render::G18_MSAA4_SAMPLE_SIZE | g18_render::G18_MSAA_WIDE_SAMPLE_SIZE) + && command.depth.base == 0 && command.stencil.base == 0)) + && command.depth.compression_base == 0 + && command.stencil.compression_base == 0 +} + +/// Complete single-level D16/D32/S8 storage for the admitted one/four-sample +/// render. AIL's tile contains 16 KiB: multiplying bytes per pixel by samples +/// shrinks its logical-pixel dimensions, rather than multiplying the rounded +/// one-sample allocation (which is wrong across a tile boundary). +fn uncompressed_plane_bytes(width: u64, height: u64, bytes_per_sample: u64, + samples: u8) -> u64 { + let block_bytes = bytes_per_sample * u64::from(samples); + let (tile_width, tile_height) = match block_bytes { + 1 => (128, 128), + 2 => (128, 64), + 4 => (64, 64), + 8 => (64, 32), + 16 => (32, 32), + _ => return 0, // Only admitted D16/D32/S8 at 1x/4x reaches this helper. + }; + if width >= tile_width && height >= tile_height { + width.div_ceil(tile_width) * height.div_ceil(tile_height) * 0x4000 + } else { + (width.next_power_of_two() * height.next_power_of_two() * block_bytes) + .div_ceil(128) * 128 + } +} + +/// Native682: bit0 selects 16-KiB units; zero selects 128-byte units. +/// Zero therefore represents a real 128-byte layer, not an absent stride. +/// Validate the complete final layer, not just its first mapped byte. +fn validate_layered_zls_plane( + plane: g17_uapi::UapiZlsBuffer, width: u64, height: u64, + bytes_per_sample: u64, layers: u16, address_space: &A, +) -> Result<(), g17_uapi::UapiTranslateError> { + use g17_uapi::{GpuAccess, UapiTranslateError as E}; + if plane.base == 0 { + return if plane.compression_base == 0 && plane.stride == 0 + && plane.compression_stride == 0 { Ok(()) } else { Err(E::ZlsFields) }; + } + if !matches!(plane.stride & 0x3fff, 0 | 1) || plane.base & 0x7f != 0 { + return Err(E::ZlsFields); + } + let unit = if plane.stride & 1 != 0 { 0x4000 } else { 0x80 }; + let stride = ((u64::from(plane.stride) >> 14) + 1) * unit; + let data_bytes = uncompressed_plane_bytes(width, height, bytes_per_sample, 1); + if data_bytes == 0 || stride < data_bytes { + return Err(E::ZlsFields); + } + let extent = stride.checked_mul(u64::from(layers - 1)) + .and_then(|n| n.checked_add(data_bytes)).ok_or(E::AddressRange)?; + if plane.base.checked_add(extent).is_none() + || !address_space.covers(plane.base, extent, GpuAccess::ReadWrite) { + return Err(E::AddressRange); + } + if plane.compression_base != 0 { + if plane.compression_base & 0x7f != 0 || plane.compression_stride & 0x3fff != 0 { + return Err(E::ZlsFields); + } + let stride = ((u64::from(plane.compression_stride) >> 14) + 1) * 0x80; + let metadata_bytes = (width.next_power_of_two().div_ceil(16) + * height.next_power_of_two().div_ceil(16) * 8).div_ceil(128) * 128; + if stride < metadata_bytes { + return Err(E::ZlsFields); + } + let extent = stride.checked_mul(u64::from(layers - 1)) + .and_then(|n| n.checked_add(metadata_bytes)).ok_or(E::AddressRange)?; + if plane.compression_base.checked_add(extent).is_none() + || !address_space.covers(plane.compression_base, extent, GpuAccess::ReadWrite) { + return Err(E::AddressRange); + } + } else if plane.compression_stride != 0 { + return Err(E::ZlsFields); + } + Ok(()) +} + +/// The envelope checks that need only the raw command. Order matters for +/// diagnostics only; every check is independent. `msaa4` admits exactly the +/// native 4x set (module parameter `g18_admit_msaa4`); refusal classes are +/// unchanged for everything else. +fn admit_envelope(command: &g17_uapi::UapiRenderCommand, msaa4: bool) + -> Result<(), G18ApplicationRefusal> { + use G18ApplicationRefusal as R; + if command.flags & !G18_RENDER_FLAGS != 0 { + return Err(R::Flags(command.flags)); + } + let no_helper = g17_uapi::UapiHelperProgram { binary: 0, config: 0, data: 0 }; + if command.vertex_helper != no_helper || command.fragment_helper != no_helper { + return Err(R::HelperProgram); + } + if !(1..=G18_MAX_RENDER_DIMENSION).contains(&u32::from(command.width)) + || !(1..=G18_MAX_RENDER_DIMENSION).contains(&u32::from(command.height)) + { + return Err(R::Dimensions { width: command.width, height: command.height }); + } + // Native598 proves layered colour target allocation and all five + // register lists. Native618/639 add layered four-sample colour through + // sixteen bytes per sample. Native682 adds one-sample layered Z32/S8; + // each plane and its metadata receive complete array range checks below. + // Native1060 also verifies uncompressed D16 arrays, including48 layers. + // Shared format validation below continues to reject compressed D16. + if !(1..=2048).contains(&command.layers) + || (command.layers > 1 && ((command.samples != 1 + && !(msaa4 && is_multisample_command(command))) + || ((command.depth.base != 0 || command.stencil.base != 0) + && (command.samples != 1 + || !matches!(command.zls_control & ZLS_DEPTH_FORMAT, 0 | 0x0400_0000))))) { + return Err(R::Layers(command.layers)); + } + if !(msaa4 && is_multisample_command(command)) { + if command.samples != 1 { + return Err(R::Samples(command.samples)); + } + if (command.utile_width, command.utile_height) != (32, 32) { + return Err(R::Utile(command.utile_width, command.utile_height)); + } + if command.multisample_control != G18_MULTISAMPLE_CONTROL { + return Err(R::MultisampleControl(command.multisample_control)); + } + } + if command.sampler_heap != 0 || command.sampler_count != 0 { + return Err(R::Samplers { heap: command.sampler_heap, count: command.sampler_count }); + } + if command.occlusion_query_base & 0xf != 0 { + return Err(R::Invalid(g17_uapi::UapiTranslateError::AddressAlignment)); + } + if command.zls_control & ZLS_STENCIL_TRANSFER != 0 + && (command.stencil.base == 0 + || (command.stencil.compression_base != 0 && command.layers == 1)) { + return Err(R::StencilTransfer(command.zls_control)); + } + if command.stencil_clear & !(ISP_BGOBJVALS_BASE | 0xff) != 0 { + return Err(R::StencilClear(command.stencil_clear)); + } + Ok(()) +} + +/// Translate one render command for the G18 job. `address_space` must answer +/// for the submitting VM; `window` is the queue's validated USC window. +/// Multisampled commands are refused (`g18_admit_msaa4=0` behaviour). The +/// adapter calls [`translate_g18_render_admitting`]; this form is kept for +/// the host parity tools (a host parity check). +#[allow(dead_code)] +#[inline(never)] +pub(crate) fn translate_g18_render( + command: &g17_uapi::UapiRenderCommand, + vertex_attachments: g17_uapi::UapiAttachmentList, + fragment_attachments: g17_uapi::UapiAttachmentList, + window: g17_uapi::QueueUscWindow, + address_space: &A, +) -> Result { + translate_g18_render_admitting(command, vertex_attachments, fragment_attachments, + window, address_space, false) +} + +/// [`translate_g18_render`], additionally admitting the native 4x render set +/// when `msaa4` is true (module parameter `g18_admit_msaa4=1`, msaa163). +#[inline(never)] +pub(crate) fn translate_g18_render_admitting( + command: &g17_uapi::UapiRenderCommand, + vertex_attachments: g17_uapi::UapiAttachmentList, + fragment_attachments: g17_uapi::UapiAttachmentList, + window: g17_uapi::QueueUscWindow, + address_space: &A, + msaa4: bool, +) -> Result { + use G18ApplicationRefusal as R; + admit_envelope(command, msaa4)?; + // Native G18 masks the low four base bits. The PPP slot is 15 bits + // and addresses an 8-byte result, matching Gallium's 32768-entry heap. + // Validate the complete read/modify/write range, not only slot zero: + // a mapped first result does not make later indices safe. + if command.occlusion_query_base != 0 { + const QUERY_HEAP_BYTES: u64 = 32768 * 8; + if command.occlusion_query_base.checked_add(QUERY_HEAP_BYTES).is_none() + || !address_space.covers(command.occlusion_query_base, QUERY_HEAP_BYTES, + g17_uapi::GpuAccess::ReadWrite) { + return Err(R::Invalid(g17_uapi::UapiTranslateError::AddressRange)); + } + } + // Validate Z32F and uncompressed Z16, including the complete format-sized + // backing. The shared validator refuses compressed Z16 metadata. + // Retain the existing depth validation while admitting stencil's own + // load/store bits. Its full uncompressed S8 range is checked separately. + // Native321 confirms D16's UNORM16 integer clear at 1x and 4x. + let depth16 = command.zls_control & ZLS_DEPTH_FORMAT == 0x0400_0000; + if command.depth.base != 0 && depth16 && command.depth_clear > 0xffff { + return Err(R::Depth(g17_uapi::UapiTranslateError::ZlsFields)); + } + let mut depth_command = *command; + depth_command.zls_control &= !ZLS_STENCIL_TRANSFER; + // Retain the shared single-plane format/control validation. The G18 + // checks below independently validate every layer's data and metadata. + depth_command.layers = 1; + g17_uapi::validate_m5_reference_depth(&depth_command, address_space).map_err(R::Depth)?; + if command.layers > 1 { + // D16 uses two bytes per sample; Z32 uses four. Applying the Z32 + // extent to D16 rejects its native128-byte8x8 layer stride. + let depth_bytes = if depth16 { 2 } else { 4 }; + for (plane, bytes_per_sample) in [(command.depth, depth_bytes), (command.stencil, 1)] { + validate_layered_zls_plane(plane, u64::from(command.width), + u64::from(command.height), bytes_per_sample, command.layers, + address_space).map_err(R::Depth)?; + } + } + if command.samples > 1 && command.depth.base != 0 { + let bytes = uncompressed_plane_bytes(u64::from(command.width), + u64::from(command.height), if depth16 { 2 } else { 4 }, command.samples); + if bytes == 0 || command.depth.base.checked_add(bytes).is_none() + || !address_space.covers(command.depth.base, bytes, g17_uapi::GpuAccess::ReadWrite) { + return Err(R::Invalid(g17_uapi::UapiTranslateError::AddressRange)); + } + } + if command.stencil.base != 0 && command.stencil.compression_base == 0 { + let bytes = uncompressed_plane_bytes(u64::from(command.width), + u64::from(command.height), 1, command.samples); + if bytes == 0 || command.stencil.base & 0x7f != 0 + || command.stencil.base.checked_add(bytes).is_none() + || !address_space.covers(command.stencil.base, bytes, g17_uapi::GpuAccess::ReadWrite) { + return Err(R::Invalid(g17_uapi::UapiTranslateError::AddressRange)); + } + } + if command.depth.base != 0 + && !matches!(command.zls_control & ZLS_DEPTH_FORMAT, 0 | 0x0400_0000) { + return Err(R::DepthFormat(command.zls_control)); + } + let translated = g17_uapi::translate_g18_render_command(*command, vertex_attachments, + fragment_attachments, window, address_space, + g17_uapi::RenderInternalState { parameters: g18_application_base() }) + .map_err(R::Invalid)?; + let mut p = translated.parameters; + + p.encoder = g18_vdm_compact(command.vdm_base).ok_or(R::VdmCompact(command.vdm_base))?; + // Native single-layer ZLS does not encode an array stride. Layered + // Native682 retains all four already-validated data/metadata strides. + if command.layers == 1 { + p.depth_stride = 0; + p.depth_aux_stride = 0; + p.stencil_stride = 0; + p.stencil_aux_stride = 0; + } + let stencil_stripped = p.stencil_aux_buffer != 0 && command.layers == 1; + if stencil_stripped { + p.stencil_buffer = 0; + p.stencil_aux_buffer = 0; + p.stencil_stride = 0; + p.stencil_aux_stride = 0; + p.depth_flags &= !ZLS_STENCIL_COMPRESSION; + // Same derivation as translate_render_command, now without stencil. + p.partial_depth_flags = p.depth_flags + | if p.depth_buffer != 0 { ZLS_DEPTH_TRANSFER } else { 0 }; + if p.depth_buffer == 0 { + // Neither plane remains: the M5 "no ZLS" envelope. + p.depth_dimensions = 0; + } + } + // ISP_BGOBJVALS: retain the S8 clear value and normalize the fixed bits. + p.stencil_clear_value = command.stencil_clear & 0xff; + // Native G18 passes the integer-depth selector to ISP register 0x15021. + // Do not erase it while replacing the shared translator's base flags. + p.aux_fb_flags = G18_AUX_FB_FLAGS | u64::from(command.flags & RENDER_DBIAS_IS_INT); + p.tile_config |= G18_MAIN_TILE_FLAGS; + + // The job plans every target inside its fixed target window; refuse here + // what it could not plan rather than fail the job. + if crate::g18::fw::render_assembly::target_plan(&p).is_none() { + return Err(R::Dimensions { width: command.width, height: command.height }); + } + + let mut controls = g18_render::g18_default_controls(&p); + // The older UAPI translator's tib_blocks is storage size, whereas G18 + // +770 / 10051 is sample control: native run112 has zero at 1x and + // 0x40000 at 4x. Keep storage accounting in p.tib_blocks for the + // independent +4d0 metadata. msaa163: the admitted 4x render (utile + // 32x16, utile_config 0x6002 and positions from the UAPI; p.samples + // stays 1, see g18_render::G18_MSAA4_UTILE) takes the native 0x40000. + let multisample_render = + msaa4 && is_multisample_command(command) && g18_render::g18_multisample(&p); + // Native464's 48-byte samples occupy two 32-byte partitions; native331's + // 16-byte samples and the original eight-byte profile occupy one. + let partitions = if command.sample_size == g18_render::G18_MSAA_MRT_SAMPLE_SIZE { + 2 + } else { + 1 + }; + controls.fragment_sample_control = + if multisample_render { (u64::from(command.samples) * partitions) << 16 } else { 0 }; + Ok(G18ApplicationRender { + parameters: p, + controls, + target: G18TargetRequest { + width: p.width, + height: p.height, + layers: p.layers, + utile: (p.utile_width, p.utile_height), + samples: if multisample_render { command.samples } else { p.samples }, + }, + vdm_base: command.vdm_base, + vertex_attachments: translated.vertex_attachments, + fragment_attachments: translated.fragment_attachments, + stencil_stripped, + flags: command.flags, + scratch: None, + }) +} + +/// Stage timestamps a retired job can report, in ticks of the architectural +/// counter (`CNTPCT_EL0`, the `DRM_IOCTL_ASAHI_GET_TIME` source): +/// `[vertex start, vertex end, fragment start, fragment end]`. +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct G18RenderRetirement { + pub(crate) stage_ticks: Option<[u64; 4]>, +} + +/// The job boundary. The conductor implements this over the G18 render +/// runtime (`g18_graphics_job.rs` and its queues/PB/UMA owners). +/// +/// Contract for `submit_render`: +/// - calls are serialized by the adapter (one at a time per device); +/// - the job fills every field [`job_owned_fields_clear`] lists, allocates targets +/// for `render.target`, maps them into `vm` (both compact views while risk +/// #1 is open), builds registers from `render.parameters/controls`, copies +/// `render.descriptor_pointers()` into its 3D command fields, encodes, +/// publishes and waits; +/// - `Ok` only after both TA and 3D retired with no fault or recovery; +/// - any error after publication must record it in `vm.status()`, retain +/// every owner and make `G18DrmRuntime::is_crashed` true (M5 contract, +/// `m5_live_boot.rs:639-656`). +#[cfg(not(test))] +pub(crate) trait G18RenderJobSink: Send + Sync { + fn submit_render(&self, vm: &crate::mmu::Vm, render: &G18ApplicationRender) + -> kernel::error::Result; +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/application_runtime.rs b/drivers/gpu/drm/asahi/g18/uapi/application_runtime.rs new file mode 100644 index 0000000000000..d8dc2fff769ec --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/application_runtime.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Pure application frame policy shared with the three-frame host replay. +//! Ring storage wraps; event stamps remain unique until their native u24 limit. +use crate::{g18::fw::render_params::G17pRenderParameters, g18::fw::render::{self as g18_render, G18RenderControls}, + g18::fw::render_command::{G18RenderCommandInputs, G18StageCompletion}, g18::uapi::render_policy as g18_render_policy}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Frame { pub generation: u32 } +impl Frame { + pub(crate) fn new(generation: u32) -> Option { + if generation==0 || generation>0xff_ffff {None} else {Some(Self {generation})} + } + pub(crate) fn stamp(self) -> u32 {self.generation<<8} + pub(crate) fn previous_stamp(self) -> u32 {(self.generation-1)<<8} + pub(crate) fn inner_before(self) -> u32 {(3+(self.generation-1)*3)%256} + pub(crate) fn inner_after(self) -> u32 {(3+self.generation*3)%256} + pub(crate) fn outer_before(self) -> u32 {(6+(self.generation-1)*2)%256} + pub(crate) fn outer_after(self) -> u32 {(6+self.generation*2)%256} + pub(crate) fn submitted(self) -> u32 {self.generation*2} +} + +/// The native G18 tail is populated from application parameters, never a +/// capture template. Optional unsupported features are explicit zero policy. +pub(crate) fn inputs(p: &mut G17pRenderParameters, c: &mut G18RenderControls, + context: u32, frame: Frame) -> G18RenderCommandInputs { + let mut i=g18_render_policy::owned_inputs(G18RenderCommandInputs::default(),context,p,c); + let g=u64::from(frame.generation); + p.lifecycle=g<<32;p.fragment_lifecycle=p.lifecycle; + i.ta.sequence=u64::from(frame.submitted()-1);i.fragment.sequence=u64::from(frame.submitted()-2); + i.fragment.pass=frame.generation as u8; + for (stage,completion) in [&mut i.ta.completion,&mut i.fragment.completion].into_iter().enumerate() { + *completion=G18StageCompletion {stamp:frame.stamp(),completion:(1<<40)|g, + key:u64::from(frame.submitted()-stage as u32),..*completion}; + } + i.ta.linked_3d.stamp=frame.stamp();i.ta.linked_3d.completion=i.fragment.completion.completion; + i.ta.d08a4=frame.generation;i.fragment.d08a4=frame.generation; + i.fragment.d04e0=p.depth_bias_array;i.fragment.d04e8=p.scissor_array;i.fragment.d04f0=p.occlusion_query_base; + i.fragment.d0518[18]=[p.load_pipeline_bind,p.load_pipeline]; + i.fragment.d0518[21]=[p.partial_load_pipeline_bind,p.partial_load_pipeline]; + i.fragment.d0518[22]=[p.depth_flags,0x0404_0404]; + i.fragment.d0690=p.depth_dimensions; + i.fragment.d0698=p.depth_buffer; + i.fragment.d06a0=[p.depth_buffer,p.depth_buffer]; + i.fragment.d0728=p.depth_aux_buffer; + // Captured gears/colour and M5 shared-runtime flag policy. These are + // independent of the application-owned depth/PPP/sample-size fields. + // msaa163: +0x212c (flag 2) is 1 in native 4x (M5: 1 at 2x and 4x). + let msaa4=g18_render::g18_msaa4(p); + i.fragment.d08c5_flags=[1,1,u8::from(msaa4),0]; + i.fragment.d0770=[c.fragment_sample_control,p.aux_fb_flags]; + i.fragment.d0780=[u64::from(p.width)|(u64::from(p.height)<<32),p.aux_fb_page_count]; + i.fragment.d0790=[p.store_pipeline_bind<<32,p.store_pipeline]; + i.fragment.d07b0=[p.partial_store_pipeline_bind<<32,p.partial_store_pipeline]; + i.fragment.d07d8=u64::from(p.depth_clear_value_bits)|(0x300<<32); + // +0x1fb0 is bytes per sample, not storage blocks or sample control. + // Both native2x/4x geometries contain2048 raster samples;1x contains1024. + // All admitted strides are multiples of8, so the2-KiB block count is exact. + let raster_samples=u64::from(p.utile_width)*u64::from(p.utile_height) + *u64::from(g18_render::g18_sample_count(p)); + i.fragment.d04d0=(p.tib_blocks*2048/raster_samples) as u32; + // msaa163: +0x50 samples (native 4) and +0x54 macro size (0x40008). + i.fragment.d0900=g18_render::g18_sample_count(p); + let tiles_x=(p.width+31)/32;let tiles_y=(p.height+31)/32; + i.fragment.d0908=g18_render::g18_macro_size(p); + i.fragment.d0910=[0,u64::from(p.merge_upper_x_bits)|(u64::from(p.merge_upper_y_bits)<<32)]; + i.fragment.d0930=tiles_x*tiles_y; + i +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/command.rs b/drivers/gpu/drm/asahi/g18/uapi/command.rs new file mode 100644 index 0000000000000..a0810a50644d1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/command.rs @@ -0,0 +1,2258 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! Pure modern-Asahi UAPI parsing and G17P command translation. +//! +//! The byte layouts match `drm_asahi_cmd_header`, `drm_asahi_cmd_render`, +//! `drm_asahi_cmd_compute`, and `drm_asahi_attachment`. Translation validates +//! the queue's 4 GiB USC window and every userspace range before constructing +//! G17P render parameters or compute-entry operands. This module allocates no +//! object, publishes no queue state, and performs no firmware or MMIO action. + +#[cfg(not(test))] +use crate::{g18::fw::compute_params as g17_compute, g18::fw::render_params as g17_render, g18::fw::submission as g17_submission}; +#[cfg(test)] +#[path = "../fw/compute_params.rs"] +mod g17_compute; +#[cfg(test)] +#[path = "../fw/render_params.rs"] +mod g17_render; +#[cfg(test)] +#[allow(dead_code)] +#[path = "../fw/submission.rs"] +mod g17_submission; + +use g17_render::{ + G17pRenderParameters, RenderBuildError, RenderDescriptorMetadata, RenderDescriptorObjects, +}; +use g17_submission::{ + G17PClBarrierDependency, G17PClKickEntryOperands, G17PClMcacheAperture, G17PClRceBinding, +}; + +pub(crate) const UAPI_COMMAND_HEADER_SIZE: usize = 8; +/// `sizeof(struct drm_asahi_cmd_render)`. Shorter payloads are zero-extended. +pub(crate) const UAPI_RENDER_SIZE: usize = 448; +/// `drm_asahi_cmd_render::ts_frag_end_copies`, which G18 does not implement: +/// the words must be zero. +const RENDER_TIMESTAMP_COPIES: core::ops::Range = 0x100..0x140; +/// `drm_asahi_cmd_render::prior_vdm` and `prior_cdm`. +const RENDER_PRIOR_VDM: usize = 0x140; +const RENDER_PRIOR_CDM: usize = 0x148; +/// `drm_asahi_cmd_render::ray_query`. G18 has no ray queries: the words +/// must be zero. +const RENDER_RAY_QUERY: core::ops::Range = 0x150..UAPI_RENDER_SIZE; +/// `sizeof(struct drm_asahi_cmd_render_g18)`: the complete render payload +/// followed by the 80-byte `drm_asahi_g18_render_resources`. +pub(crate) const UAPI_G18_RENDER_SIZE: usize = UAPI_RENDER_SIZE + 80; +/// `sizeof(struct drm_asahi_cmd_compute)`. Shorter payloads are zero-extended. +pub(crate) const UAPI_COMPUTE_SIZE: usize = 200; +/// `drm_asahi_cmd_compute::ts_end_copies`, which G18 does not implement. +const COMPUTE_TIMESTAMP_COPIES: core::ops::Range = 0x40..0x80; +/// `drm_asahi_cmd_compute::scs_layout`. +const COMPUTE_SCS_LAYOUT: usize = 0x80; +/// `drm_asahi_cmd_compute::prior_vdm` and `prior_cdm`. +const COMPUTE_PRIOR_VDM: usize = 0x88; +const COMPUTE_PRIOR_CDM: usize = 0x90; +/// `cdm_auxiliary_allocation` through `intersection_default_control`, which +/// G18 does not implement: the words must be zero. +const COMPUTE_UNSUPPORTED: core::ops::Range = 0x98..UAPI_COMPUTE_SIZE; +/// `sizeof(struct drm_asahi_cmd_compute_g18)`: the complete compute payload +/// followed by the 64-byte `drm_asahi_g18_compute_resources`. +pub(crate) const UAPI_G18_COMPUTE_SIZE: usize = UAPI_COMPUTE_SIZE + 64; +/// `DRM_ASAHI_COMPUTE_G18_RESOURCES`. +pub(crate) const COMPUTE_G18_RESOURCES: u32 = 1 << 19; +/// `DRM_ASAHI_RENDER_G18_RESOURCES`. Only the G18 adapter may consume and +/// strip it; the shared translator's RENDER_FLAGS keeps refusing it. +pub(crate) const RENDER_G18_RESOURCES: u32 = 1 << 25; +pub(crate) const UAPI_ATTACHMENT_SIZE: usize = 24; +pub(crate) const UAPI_MAX_ATTACHMENTS: usize = 16; +pub(crate) const UAPI_MAX_HARDWARE_COMMANDS: u32 = 64; +// Match the image/framebuffer limit advertised by Honeykrisp. Storage is +// sized from each submission; this is a format limit, not a demo window cap. +pub(crate) const M5_MAX_RENDER_DIMENSION: u32 = 16384; +pub(crate) const UAPI_BARRIER_NONE: u16 = 0xffff; +pub(crate) const USC_WINDOW_SIZE: u64 = 1 << 32; +#[cfg(not(test))] +pub(crate) const COMPUTE_G17P_ADD3_PROOF: u32 = + kernel::uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF as u32; +#[cfg(test)] +pub(crate) const COMPUTE_G17P_ADD3_PROOF: u32 = 1 << 0; +const G17P_ADD3_BUFFER_BYTES: u64 = 64 * 4; + +const COMMAND_RENDER: u16 = 0; +const COMMAND_COMPUTE: u16 = 1; +const SET_VERTEX_ATTACHMENTS: u16 = 2; +const SET_FRAGMENT_ATTACHMENTS: u16 = 3; +const SET_COMPUTE_ATTACHMENTS: u16 = 4; + +const RENDER_VERTEX_SCRATCH: u32 = 1 << 0; +const RENDER_PROCESS_EMPTY_TILES: u32 = 1 << 1; +const RENDER_NO_VERTEX_CLUSTERING: u32 = 1 << 2; +const RENDER_RSRC_SPEC_HI: u32 = 1 << 4; +const RENDER_DBIAS_IS_INT: u32 = 1 << 18; +const RENDER_FLAGS: u32 = RENDER_VERTEX_SCRATCH + | RENDER_PROCESS_EMPTY_TILES + | RENDER_NO_VERTEX_CLUSTERING + | RENDER_RSRC_SPEC_HI + | RENDER_DBIAS_IS_INT + // Private, opt-in matched-workload diagnostic. Ordinary builds reject it. + | if cfg!(g17p_aux_fb_small_probe) { 1 << 31 } else { 0 } + | if cfg!(g17p_mcache_aperture_probe) { 1 << 30 } else { 0 }; + +const SAMPLER_SIZE: u64 = 8; +const MAX_SAMPLERS: u32 = 1024; +const PAGE_SIZE: u64 = 0x4000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum UapiTranslateError { + HeaderTruncated, + PayloadTruncated, + PayloadTail, + CommandType, + AttachmentPayloadSize, + AttachmentCount, + AttachmentFields, + AttachmentBarrier, + HardwareCommandCount, + BarrierFuture, + QueueUscAlignment, + QueueUscRange, + AddressRange, + AddressAlignment, + FlagsUnknownBits, + FlagsUnsupported, + Dimensions, + Layers, + UtileDimensions, + Samples, + TilebufferSize, + SamplerCount, + SamplerPair, + HelperProgram, + Add3ProofAttachments, + ProgramOffset, + ZlsFields, + StencilClearBits, + RenderBuild(RenderBuildError), + ComputeBuild(g17_compute::ComputeBuildError), +} + +impl From for UapiTranslateError { + fn from(value: RenderBuildError) -> Self { + Self::RenderBuild(value) + } +} + +impl From for UapiTranslateError { + fn from(value: g17_compute::ComputeBuildError) -> Self { + Self::ComputeBuild(value) + } +} + +impl UapiTranslateError { + /// Stable short name for one rejection, so a failing submit reports which + /// check refused it instead of a bare EINVAL. Kept as `&'static str` (no + /// `Debug` formatting) to stay cheap on the ioctl path. + pub(crate) fn name(&self) -> &'static str { + match self { + Self::HeaderTruncated => "header-truncated", + Self::PayloadTruncated => "payload-truncated", + Self::PayloadTail => "payload-tail", + Self::CommandType => "command-type", + Self::AttachmentPayloadSize => "attachment-payload-size", + Self::AttachmentCount => "attachment-count", + Self::AttachmentFields => "attachment-fields", + Self::AttachmentBarrier => "attachment-barrier", + Self::HardwareCommandCount => "hardware-command-count", + Self::BarrierFuture => "barrier-future", + Self::QueueUscAlignment => "queue-usc-alignment", + Self::QueueUscRange => "queue-usc-range", + Self::AddressRange => "address-range", + Self::AddressAlignment => "address-alignment", + Self::FlagsUnknownBits => "flags-unknown-bits", + Self::FlagsUnsupported => "flags-unsupported", + Self::Dimensions => "dimensions", + Self::Layers => "layers", + Self::UtileDimensions => "utile-dimensions", + Self::Samples => "samples", + Self::TilebufferSize => "tilebuffer-size", + Self::SamplerCount => "sampler-count", + Self::SamplerPair => "sampler-pair", + Self::HelperProgram => "helper-program", + Self::Add3ProofAttachments => "add3-proof-attachments", + Self::ProgramOffset => "program-offset", + Self::ZlsFields => "zls-fields", + Self::StencilClearBits => "stencil-clear-bits", + Self::RenderBuild(_) => "render-build", + Self::ComputeBuild(_) => "compute-build", + } + } +} + +fn get_u16(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) +} + +fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) +} + +fn padded_payload(raw: &[u8]) -> Result<[u8; N], UapiTranslateError> { + if raw.len() > N && raw[N..].iter().any(|byte| *byte != 0) { + return Err(UapiTranslateError::PayloadTail); + } + let mut out = [0u8; N]; + let count = core::cmp::min(raw.len(), N); + out[..count].copy_from_slice(&raw[..count]); + Ok(out) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiCommandHeader { + pub(crate) command_type: u16, + pub(crate) size: u16, + pub(crate) render_barrier: u16, + pub(crate) compute_barrier: u16, +} + +impl UapiCommandHeader { + fn parse(raw: &[u8]) -> Result { + if raw.len() < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + Ok(Self { + command_type: get_u16(raw, 0), + size: get_u16(raw, 2), + render_barrier: get_u16(raw, 4), + compute_barrier: get_u16(raw, 6), + }) + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct UapiAttachment { + pub(crate) address: u64, + pub(crate) size: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiAttachmentList { + pub(crate) entries: [UapiAttachment; UAPI_MAX_ATTACHMENTS], + pub(crate) count: u8, +} + +impl UapiAttachmentList { + pub(crate) const EMPTY: Self = Self { + entries: [UapiAttachment { + address: 0, + size: 0, + }; UAPI_MAX_ATTACHMENTS], + count: 0, + }; + + pub(crate) fn as_slice(&self) -> &[UapiAttachment] { + &self.entries[..self.count as usize] + } + + fn parse(raw: &[u8]) -> Result { + if raw.len() % UAPI_ATTACHMENT_SIZE != 0 { + return Err(UapiTranslateError::AttachmentPayloadSize); + } + let count = raw.len() / UAPI_ATTACHMENT_SIZE; + if count > UAPI_MAX_ATTACHMENTS { + return Err(UapiTranslateError::AttachmentCount); + } + let mut out = Self::EMPTY; + for index in 0..count { + let offset = index * UAPI_ATTACHMENT_SIZE; + if get_u32(raw, offset + 0x10) != 0 || get_u32(raw, offset + 0x14) != 0 { + return Err(UapiTranslateError::AttachmentFields); + } + out.entries[index] = UapiAttachment { + address: get_u64(raw, offset), + size: get_u64(raw, offset + 8), + }; + } + out.count = count as u8; + Ok(out) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiHelperProgram { + pub(crate) binary: u32, + pub(crate) config: u32, + pub(crate) data: u64, +} + +fn helper(raw: &[u8], offset: usize) -> UapiHelperProgram { + UapiHelperProgram { + binary: get_u32(raw, offset), + config: get_u32(raw, offset + 4), + data: get_u64(raw, offset + 8), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiProgram { + pub(crate) usc: u32, + pub(crate) resource_spec: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamp { + pub(crate) handle: u32, + pub(crate) offset: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamps { + pub(crate) start: UapiTimestamp, + pub(crate) end: UapiTimestamp, +} + +fn timestamps(raw: &[u8], offset: usize) -> UapiTimestamps { + UapiTimestamps { + start: UapiTimestamp { + handle: get_u32(raw, offset), + offset: get_u32(raw, offset + 4), + }, + end: UapiTimestamp { + handle: get_u32(raw, offset + 8), + offset: get_u32(raw, offset + 12), + }, + } +} + +fn program(raw: &[u8], offset: usize, high_offset: usize) -> UapiProgram { + UapiProgram { + usc: get_u32(raw, offset), + resource_spec: get_u32(raw, offset + 4) as u64 + | (get_u32(raw, high_offset) as u64) << 32, + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiZlsBuffer { + pub(crate) base: u64, + pub(crate) compression_base: u64, + pub(crate) stride: u32, + pub(crate) compression_stride: u32, +} + +fn zls(raw: &[u8], offset: usize) -> UapiZlsBuffer { + UapiZlsBuffer { + base: get_u64(raw, offset), + compression_base: get_u64(raw, offset + 8), + stride: get_u32(raw, offset + 0x10), + compression_stride: get_u32(raw, offset + 0x14), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiRenderCommand { + pub(crate) flags: u32, + pub(crate) depth_dimensions: u32, + pub(crate) vdm_base: u64, + pub(crate) vertex_helper: UapiHelperProgram, + pub(crate) fragment_helper: UapiHelperProgram, + pub(crate) scissor_base: u64, + pub(crate) depth_bias_base: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth: UapiZlsBuffer, + pub(crate) stencil: UapiZlsBuffer, + pub(crate) zls_control: u64, + pub(crate) multisample_control: u64, + pub(crate) sampler_heap: u64, + pub(crate) ppp_control: u32, + pub(crate) width: u16, + pub(crate) height: u16, + pub(crate) layers: u16, + pub(crate) sampler_count: u16, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) samples: u8, + pub(crate) sample_size: u8, + pub(crate) merge_upper_x: u32, + pub(crate) merge_upper_y: u32, + pub(crate) background: UapiProgram, + pub(crate) end_of_tile: UapiProgram, + pub(crate) partial_background: UapiProgram, + pub(crate) partial_end_of_tile: UapiProgram, + pub(crate) depth_clear: u32, + pub(crate) stencil_clear: u32, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, + pub(crate) prior_vdm: u64, + pub(crate) prior_cdm: u64, + /// `drm_asahi_g18_render_resources` words; zero unless flagged. + pub(crate) g18_resources: [u32; 20], +} + +impl UapiRenderCommand { + fn parse(raw: &[u8]) -> Result { + // Shorter payloads are zero-extended. A flagged G18 suffix must be + // complete; padding could hide a truncated field. Words G18 does not + // implement (timestamp copies, ray queries) must be zero, as must + // anything after the structure or, when flagged, its suffix. + let has_resources = raw.len() >= 4 && get_u32(raw, 0) & RENDER_G18_RESOURCES != 0; + if has_resources && raw.len() < UAPI_G18_RENDER_SIZE { + return Err(UapiTranslateError::PayloadTruncated); + } + let raw = padded_payload::(raw)?; + let unused_tail = if has_resources { UAPI_G18_RENDER_SIZE } else { UAPI_RENDER_SIZE }; + if raw[RENDER_TIMESTAMP_COPIES].iter().chain(&raw[RENDER_RAY_QUERY]) + .chain(&raw[unused_tail..]).any(|byte| *byte != 0) + { + return Err(UapiTranslateError::PayloadTail); + } + let mut g18_resources = [0; 20]; + if has_resources { + for (i, word) in g18_resources.iter_mut().enumerate() { + *word = get_u32(&raw, UAPI_RENDER_SIZE + i * 4); + } + } + Ok(Self { + flags: get_u32(&raw, 0x00), + depth_dimensions: get_u32(&raw, 0x04), + vdm_base: get_u64(&raw, 0x08), + vertex_helper: helper(&raw, 0x10), + fragment_helper: helper(&raw, 0x20), + scissor_base: get_u64(&raw, 0x30), + depth_bias_base: get_u64(&raw, 0x38), + occlusion_query_base: get_u64(&raw, 0x40), + depth: zls(&raw, 0x48), + stencil: zls(&raw, 0x60), + zls_control: get_u64(&raw, 0x78), + multisample_control: get_u64(&raw, 0x80), + sampler_heap: get_u64(&raw, 0x88), + ppp_control: get_u32(&raw, 0x90), + width: get_u16(&raw, 0x94), + height: get_u16(&raw, 0x96), + layers: get_u16(&raw, 0x98), + sampler_count: get_u16(&raw, 0x9a), + utile_width: raw[0x9c], + utile_height: raw[0x9d], + samples: raw[0x9e], + sample_size: raw[0x9f], + merge_upper_x: get_u32(&raw, 0xa0), + merge_upper_y: get_u32(&raw, 0xa4), + background: program(&raw, 0xa8, 0xf8), + end_of_tile: program(&raw, 0xb0, 0xf0), + partial_background: program(&raw, 0xb8, 0xfc), + partial_end_of_tile: program(&raw, 0xc0, 0xf4), + depth_clear: get_u32(&raw, 0xc8), + stencil_clear: get_u32(&raw, 0xcc), + vertex_timestamps: timestamps(&raw, 0xd0), + fragment_timestamps: timestamps(&raw, 0xe0), + prior_vdm: get_u64(&raw, RENDER_PRIOR_VDM), + prior_cdm: get_u64(&raw, RENDER_PRIOR_CDM), + g18_resources, + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiComputeCommand { + pub(crate) flags: u32, + pub(crate) sampler_count: u32, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) sampler_heap: u64, + pub(crate) helper: UapiHelperProgram, + pub(crate) timestamps: UapiTimestamps, + pub(crate) scs_layout: u64, + pub(crate) g18_resources: [u32; 16], + pub(crate) prior_vdm: u64, + pub(crate) prior_cdm: u64, +} + +impl UapiComputeCommand { + fn parse(raw: &[u8]) -> Result { + // Shorter compute payloads are zero-extended. A flagged metadata + // payload must be complete; padding it could hide truncation of a + // valid zero-valued field or reserved word. Words G18 does not + // implement (timestamp copies, the auxiliary allocation, intersection + // tables) must be zero, as must anything after the structure or, + // when flagged, its suffix. + let has_resources = raw.len() >= 4 && get_u32(raw, 0) & COMPUTE_G18_RESOURCES != 0; + if has_resources && raw.len() < UAPI_G18_COMPUTE_SIZE { + return Err(UapiTranslateError::PayloadTruncated); + } + let raw = padded_payload::(raw)?; + let unused_tail = if has_resources { UAPI_G18_COMPUTE_SIZE } else { UAPI_COMPUTE_SIZE }; + if raw[COMPUTE_TIMESTAMP_COPIES].iter().chain(&raw[COMPUTE_UNSUPPORTED]) + .chain(&raw[unused_tail..]).any(|byte| *byte != 0) + { + return Err(UapiTranslateError::PayloadTail); + } + let mut g18_resources = [0; 16]; + if has_resources { + for (i, word) in g18_resources.iter_mut().enumerate() { + *word = get_u32(&raw, UAPI_COMPUTE_SIZE + i * 4); + } + } + Ok(Self { + flags: get_u32(&raw, 0), + sampler_count: get_u32(&raw, 4), + control_stream_base: get_u64(&raw, 8), + control_stream_end: get_u64(&raw, 0x10), + sampler_heap: get_u64(&raw, 0x18), + helper: helper(&raw, 0x20), + timestamps: timestamps(&raw, 0x30), + scs_layout: get_u64(&raw, COMPUTE_SCS_LAYOUT), + g18_resources, + prior_vdm: get_u64(&raw, COMPUTE_PRIOR_VDM), + prior_cdm: get_u64(&raw, COMPUTE_PRIOR_CDM), + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ParsedHardwareCommand { + Render { + index: u32, + header: UapiCommandHeader, + payload: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + }, + Compute { + index: u32, + header: UapiCommandHeader, + payload: UapiComputeCommand, + attachments: UapiAttachmentList, + }, +} + +pub(crate) struct UapiCommandParser<'a> { + raw: &'a [u8], + offset: usize, + command_index: u32, + hardware_count: u32, + render_count: u16, + compute_count: u16, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + compute_attachments: UapiAttachmentList, +} + +impl<'a> UapiCommandParser<'a> { + pub(crate) const fn new(raw: &'a [u8]) -> Self { + Self { + raw, + offset: 0, + command_index: 0, + hardware_count: 0, + render_count: 0, + compute_count: 0, + vertex_attachments: UapiAttachmentList::EMPTY, + fragment_attachments: UapiAttachmentList::EMPTY, + compute_attachments: UapiAttachmentList::EMPTY, + } + } + + pub(crate) fn finish(&self) -> Result { + if self.offset != self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + if self.hardware_count == 0 || self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + Ok(self.hardware_count) + } + + pub(crate) fn next_hardware( + &mut self, + ) -> Result, UapiTranslateError> { + loop { + if self.offset == self.raw.len() { + return Ok(None); + } + if self.raw.len() - self.offset < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + let header = UapiCommandHeader::parse(&self.raw[self.offset..])?; + self.offset += UAPI_COMMAND_HEADER_SIZE; + let end = self + .offset + .checked_add(header.size as usize) + .ok_or(UapiTranslateError::PayloadTruncated)?; + if end > self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + let payload = &self.raw[self.offset..end]; + self.offset = end; + self.command_index += 1; + + match header.command_type { + COMMAND_RENDER | COMMAND_COMPUTE => { + if header.render_barrier != UAPI_BARRIER_NONE + && header.render_barrier > self.render_count + { + return Err(UapiTranslateError::BarrierFuture); + } + if header.compute_barrier != UAPI_BARRIER_NONE + && header.compute_barrier > self.compute_count + { + return Err(UapiTranslateError::BarrierFuture); + } + self.hardware_count += 1; + if self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + if header.command_type == COMMAND_RENDER { + self.render_count += 1; + return Ok(Some(ParsedHardwareCommand::Render { + index: self.command_index, + header, + payload: UapiRenderCommand::parse(payload)?, + vertex_attachments: self.vertex_attachments, + fragment_attachments: self.fragment_attachments, + })); + } + self.compute_count += 1; + return Ok(Some(ParsedHardwareCommand::Compute { + index: self.command_index, + header, + payload: UapiComputeCommand::parse(payload)?, + attachments: self.compute_attachments, + })); + } + SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS | SET_COMPUTE_ATTACHMENTS => { + if header.render_barrier != UAPI_BARRIER_NONE + || header.compute_barrier != UAPI_BARRIER_NONE + { + return Err(UapiTranslateError::AttachmentBarrier); + } + let attachments = UapiAttachmentList::parse(payload)?; + match header.command_type { + SET_VERTEX_ATTACHMENTS => self.vertex_attachments = attachments, + SET_FRAGMENT_ATTACHMENTS => self.fragment_attachments = attachments, + SET_COMPUTE_ATTACHMENTS => self.compute_attachments = attachments, + _ => unreachable!(), + } + } + _ => return Err(UapiTranslateError::CommandType), + } + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum GpuAccess { + Read, + Write, + ReadWrite, +} + +pub(crate) trait GpuAddressSpace { + fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool; +} + +fn require_range( + address_space: &A, + address: u64, + size: u64, + access: GpuAccess, +) -> Result<(), UapiTranslateError> { + let end = address + .checked_add(size) + .ok_or(UapiTranslateError::AddressRange)?; + if size == 0 || end <= address || !address_space.covers(address, size, access) { + return Err(UapiTranslateError::AddressRange); + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueueUscWindow { + pub(crate) base: u64, + pub(crate) user_start: u64, + pub(crate) user_end: u64, +} + +impl QueueUscWindow { + pub(crate) fn validate(self) -> Result<(), UapiTranslateError> { + if self.base & (USC_WINDOW_SIZE - 1) != 0 { + return Err(UapiTranslateError::QueueUscAlignment); + } + let end = self + .base + .checked_add(USC_WINDOW_SIZE) + .ok_or(UapiTranslateError::QueueUscRange)?; + if self.base < self.user_start || end > self.user_end || end <= self.base { + return Err(UapiTranslateError::QueueUscRange); + } + Ok(()) + } + + fn full_program( + self, + address_space: &A, + tagged_offset: u32, + ) -> Result { + self.validate()?; + if tagged_offset == 0 { + return Err(UapiTranslateError::ProgramOffset); + } + let allocation = self + .base + .checked_add((tagged_offset & !7) as u64) + .ok_or(UapiTranslateError::ProgramOffset)?; + require_range(address_space, allocation, 4, GpuAccess::Read)?; + // Honeykrisp carries resource metadata in the low three UAPI bits, + // but G17P's native 3D command strips those bits before programming + // every BG/EOT/partial USC address. Preserving bit 2 launched four + // bytes into the instruction stream and contradicted all complete + // Metal register-list captures. + Ok(allocation) + } +} + +fn validate_sampler( + address_space: &A, + heap: u64, + count: u32, +) -> Result<(), UapiTranslateError> { + if count > MAX_SAMPLERS { + return Err(UapiTranslateError::SamplerCount); + } + if (heap == 0) != (count == 0) { + return Err(UapiTranslateError::SamplerPair); + } + if count != 0 { + if heap & (SAMPLER_SIZE - 1) != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + heap, + count as u64 * SAMPLER_SIZE, + GpuAccess::Read, + )?; + } + Ok(()) +} + +fn validate_attachments( + address_space: &A, + attachments: &UapiAttachmentList, +) -> Result<(), UapiTranslateError> { + for attachment in attachments.as_slice() { + require_range( + address_space, + attachment.address, + attachment.size, + GpuAccess::Write, + )?; + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderInternalState { + pub(crate) parameters: G17pRenderParameters, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedRenderCommand { + /// Exact logical render prefix assigned by the DRM ioctl dispatcher. + pub(crate) render_order: g17_submission::G17PRenderOrder, + /// Original UAPI render index, retained through admission. + pub(crate) render_barrier: u16, + /// Original UAPI compute index, retained through admission. + pub(crate) compute_barrier: u16, + /// Exact compute prefix resolved at ioctl admission. Later independent + /// compute commands must not become parents of this render. + pub(crate) compute_wait_through: Option, + pub(crate) flags: u32, + pub(crate) vdm_base: u64, + pub(crate) parameters: G17pRenderParameters, + pub(crate) vertex_attachments: UapiAttachmentList, + pub(crate) fragment_attachments: UapiAttachmentList, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, +} + +fn validate_zls( + address_space: &A, + value: UapiZlsBuffer, + layers: u16, + g18_subpage: bool, +) -> Result<(), UapiTranslateError> { + if value.base == 0 + && (value.compression_base != 0 || value.stride != 0 || value.compression_stride != 0) + { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_base == 0 && value.compression_stride != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if !g18_subpage && layers > 1 && value.base != 0 && value.stride == 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.stride != 0 && value.stride & 0x3fff != 1 + && !(g18_subpage && value.stride & 0x3fff == 0) { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_stride & 0x3fff != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.base != 0 { + let layer_stride = if g18_subpage && layers > 1 { + let unit = if value.stride & 1 != 0 { PAGE_SIZE } else { 0x80 }; + ((value.stride as u64 >> 14) + 1) * unit + } else if value.stride == 0 { + 0 + } else { + ((value.stride as u64 >> 14) + 1) * PAGE_SIZE + }; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range(address_space, value.base, size, GpuAccess::ReadWrite)?; + } + if value.compression_base != 0 { + let layer_stride = ((value.compression_stride as u64 >> 14) + 1) * 0x80; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range( + address_space, + value.compression_base, + size, + GpuAccess::ReadWrite, + )?; + } + Ok(()) +} + +/// Bounded M5 D32 clear/store envelope, alongside the original color-only +/// profile. Native 25G76 supplies ff00ff dimensions and 80044 with compression; +/// bits2/6 select compression, while bit19 stores the depth plane. Geometry, +/// stencil absence, and application programs are checked by the M5 caller. +pub(crate) fn validate_m5_reference_depth( + command: &UapiRenderCommand, + address_space: &A, +) -> Result<(), UapiTranslateError> { + let depth = command.depth; + if depth.base == 0 { + // No depth plane. The surface may still carry an inert stencil one, + // which contributes nothing but its compression pair and the shared + // pixel dimensions; a render that transfers it is refused with the + // rest of the stencil profile. + let stencil_only = command.stencil.base != 0 + && command.zls_control + == if command.stencil.compression_base != 0 { 0x110 } else { 0 } + && command.depth_dimensions + == ((u32::from(command.width) - 1) + | ((u32::from(command.height) - 1) << 16)); + return if depth.compression_base == 0 && depth.stride == 0 + && depth.compression_stride == 0 + && (stencil_only + || (command.zls_control == 0 && command.depth_dimensions == 0)) { + Ok(()) + } else { + Err(UapiTranslateError::ZlsFields) + }; + } + let compressed = depth.compression_base != 0; + // The control word's load and store bits are the only part a render + // chooses: bit15 loads the depth plane and bit19 stores it, and a render + // that neither tests nor writes depth sets neither, which is how a + // conformance surface with an unused depth buffer arrives. Everything + // else -- the compression pair in bits 2 and 6, the GPU tiling, the 32F + // format -- is the witnessed profile and stays exact. depth_flags reaches + // register 0x15319 unchanged either way, so the admitted envelope is the + // set of load/store combinations over that fixed remainder. + let compression_bits = if compressed { 0x44 } else { 0 }; + // Bits 25-26 select the depth format: 0 is 32-bit float, 2 is 16-bit + // unorm. The narrow plane was captured on 25G76 alongside the wide one + // and moves nothing else in the register program except the clear value + // and one ISP flag, both of which the caller derives. Its compression + // metadata has no witness, so a compressed 16-bit plane stays out. + let z16 = command.zls_control & (3 << 25) == (2 << 25); + if z16 && compressed { + return Err(UapiTranslateError::ZlsFields); + } + let format_bits = if z16 { 2 << 25 } else { 0 }; + // A stencil plane may be attached without being transferred: a surface + // with a depth-stencil buffer carries one for every render, whether or + // not the render uses it. Its compression pair in bits 4 and 8 describes + // the attached layout and is the only stencil state admitted here -- + // loading or storing the plane (bits 14 and 18) needs its own witnesses, + // as does the stencil unit in the graphics record. + let stencil_compression_bits = + if command.stencil.compression_base != 0 { 0x110 } else { 0 }; + let control_transfers = command.zls_control & 0x88000; + let expected_control = + compression_bits | stencil_compression_bits | format_bits | control_transfers; + let width = u32::from(command.width); + let height = u32::from(command.height); + if !(1..=M5_MAX_RENDER_DIMENSION).contains(&width) + || !(1..=M5_MAX_RENDER_DIMENSION).contains(&height) + || command.layers != 1 || (compressed && (width < 16 || height < 16)) { + return Err(UapiTranslateError::ZlsFields); + } + // AIL tiles are 16 KiB whatever the format: 64x64 at four bytes a pixel, + // 128x64 at two. While both axes fit one tile the level is a whole + // number of them; if either axis is smaller, the level uses a + // power-of-two miptail aligned to a 128-byte cacheline. Single-layer M5 + // surfaces are not page-padded. + let (tile_width, bytes_per_pixel) = if z16 { (128u32, 2u64) } else { (64u32, 4u64) }; + let data_bytes = if width >= tile_width && height >= 64 { + u64::from(width.div_ceil(tile_width)) * u64::from(height.div_ceil(64)) * 0x4000 + } else { + (u64::from(width.next_power_of_two()) + * u64::from(height.next_power_of_two()) * bytes_per_pixel) + .div_ceil(128) * 128 + }; + let metadata_bytes = (u64::from(width.next_power_of_two().div_ceil(16)) + * u64::from(height.next_power_of_two().div_ceil(16)) * 8).div_ceil(128) * 128; + let dimensions = (width - 1) | ((height - 1) << 16); + if command.depth_dimensions != dimensions + || command.zls_control != expected_control + // The clear value is whatever the application asked for; it reaches + // register 0x15301 verbatim and is only consumed when the render + // actually clears the plane. + || (command.depth_clear & 0x7fffffff) > 0x7f800000 + // Image suballocations and small miptails use cacheline alignment; + // the backing VM pages remain independently validated below. + || depth.base & 0x7f != 0 || depth.compression_base & 0x7f != 0 + // Both strides are array strides, and the M5 caller zeroes them + // because native single-layer ZLS does not encode one. Every render + // admitted here is single-layer -- layers != 1 is refused above -- so + // the value that arrives is whatever the userspace packer computed + // and the hardware never sees it: for a surface smaller than a page + // Mesa's (pages - 1) << 14 | 1 underflows to 0xffffc001. Comparing it + // would refuse a valid render over a field that is discarded. The + // backing store is still validated in full below, from the + // dimensions rather than from either stride. + || (!compressed && depth.compression_stride != 0) { + return Err(UapiTranslateError::ZlsFields); + } + // Validate the entire resized surface and metadata before the + // M5 caller removes the unused single-layer array stride fields. + require_range(address_space, depth.base, data_bytes, GpuAccess::ReadWrite)?; + if compressed { + require_range(address_space, depth.compression_base, metadata_bytes, + GpuAccess::ReadWrite)?; + } + Ok(()) +} + +pub(crate) fn translate_render_command( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, +) -> Result { + translate_render_command_with_capacity(command, vertex_attachments, + fragment_attachments, queue, address_space, internal, 32_768, false) +} + +/// G18 has a separately admitted native 96-KiB resident multisample profile. +/// Its caller validates sample count, utile geometry, stride and layer count. +pub(crate) fn translate_g18_render_command( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, +) -> Result { + // Native1044's 1x RGBA32 MRT footprint reaches128 bytes per sample at 32x32 + // (128KiB); native1121 adds 40/56/72. Layered, geometry and MSAA stay separate. + let wide_single = command.samples == 1 && command.layers == 1 + && (command.utile_width, command.utile_height) == (32, 32) + && (((48..=128).contains(&command.sample_size) && command.sample_size & 15 == 0) + || matches!(command.sample_size, 40 | 56 | 72)); + let capacity = if wide_single { 131_072 } + else if matches!(command.samples, 2 | 4) { 98_304 } + else { 32_768 }; + translate_render_command_with_capacity(command, vertex_attachments, + fragment_attachments, queue, address_space, internal, capacity, true) +} + +fn translate_render_command_with_capacity( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, + tile_capacity: u64, + g18_subpage: bool, +) -> Result { + if command.flags & !RENDER_FLAGS != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + // Genuinely unimplemented on G17P rather than merely unvalidated: + // `G17pRenderParameters` carries no helper-program fields at all, so + // VERTEX_SCRATCH has nowhere to put the scratch binding, and there is no + // cluster-count control for NO_VERTEX_CLUSTERING. A clean EINVAL beats + // rendering with the request silently dropped. + if command.flags & (RENDER_VERTEX_SCRATCH | RENDER_NO_VERTEX_CLUSTERING) != 0 { + return Err(UapiTranslateError::FlagsUnsupported); + } + // RSRC_SPEC_HI is *declarative*, and honouring it must not mean rejecting. + // The UAPI defines it as "the appended resource-specifier high dwords are + // present", and states its purpose: "This flag lets old kernels report an + // unsupported render payload instead of silently truncating 64-bit G17 + // values." It exists for kernels that do NOT understand the dwords. + // + // This kernel does understand them -- `program()` ORs the high dword in + // unconditionally, with no reference to the flag -- so the flag cannot + // change what we do with a payload that sets it. The old check therefore + // refused payloads it would otherwise have handled identically, protecting + // nothing; it is the same bring-up leftover as the stencil-clear mask. + // + // Honour the documented meaning instead: when the flag is clear the high + // dwords are "not present", so mask them off rather than refuse the + // submission. A userspace built against the 240-byte upstream struct (whose + // payload `padded_payload` zero-fills) and one that clears the flag now + // behave identically, which is what "not present" has to mean. + let resource_spec_mask = if command.flags & RENDER_RSRC_SPEC_HI != 0 { + u64::MAX + } else { + u32::MAX as u64 + }; + if !(1..=16_384).contains(&command.width) || !(1..=16_384).contains(&command.height) { + return Err(UapiTranslateError::Dimensions); + } + if !(1..=2048).contains(&command.layers) { + return Err(UapiTranslateError::Layers); + } + if !matches!( + (command.utile_width, command.utile_height), + (32, 32) | (32, 16) | (16, 16) + ) { + return Err(UapiTranslateError::UtileDimensions); + } + let samples_log2 = match command.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err(UapiTranslateError::Samples), + }; + let utile_bytes = command.sample_size as u64 + * command.utile_width as u64 + * command.utile_height as u64 + * command.samples as u64; + if utile_bytes > tile_capacity { + return Err(UapiTranslateError::TilebufferSize); + } + let blocks_per_utile = utile_bytes.div_ceil(2048); + + if command.vdm_base == 0 || command.vdm_base & 3 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.vdm_base, 4, GpuAccess::Read)?; + + if command.scissor_base == 0 || command.scissor_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.scissor_base, 8, GpuAccess::Read)?; + if command.depth_bias_base != 0 { + if command.depth_bias_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.depth_bias_base, 8, GpuAccess::Read)?; + } + if command.occlusion_query_base != 0 { + if command.occlusion_query_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.occlusion_query_base, + 8, + GpuAccess::Write, + )?; + } + validate_zls(address_space, command.depth, command.layers, g18_subpage)?; + validate_zls(address_space, command.stencil, command.layers, g18_subpage)?; + validate_sampler( + address_space, + command.sampler_heap, + command.sampler_count as u32, + )?; + if command.vertex_helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + || command.fragment_helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + // `isp_bgobjvals` is the *whole* ISP_BGOBJVALS register value, not just + // the stencil clear: the UAPI documents "the bottom 8-bits contain the + // stencil buffer clear value", and every Mesa backend unconditionally + // seeds the upper bits with 0x300 before OR-ing the clear in + // (hk_cmd_draw.c `render->cr.isp_bgobjvals = 0x300;`, + // agx_pipe.c `c->isp_bgobjvals = 0x300;`, d12_queue.c `0x300 | stencil`). + // The legacy queue path passes the field straight through to the firmware + // (queue/render.rs `let load_bgobjvals = cmdbuf.isp_bgobjvals as u64`), so + // rejecting anything above 0xff refused every real render pass while the + // in-tree render selftest -- which leaves the field zero -- was accepted. + // Accept the register value; `build_*_descriptor` re-asserts the 0x300 + // bits with `stencil_clear_value | 0x300`, which is idempotent for the + // value userspace sends and still supplies 0x300 when it sends zero. + + let background = queue.full_program(address_space, command.background.usc)?; + let end_of_tile = queue.full_program(address_space, command.end_of_tile.usc)?; + let partial_background = queue.full_program(address_space, command.partial_background.usc)?; + let partial_end_of_tile = queue.full_program(address_space, command.partial_end_of_tile.usc)?; + validate_attachments(address_space, &vertex_attachments)?; + validate_attachments(address_space, &fragment_attachments)?; + + let utile_config = ((command.utile_width as u64 / 16) << 12) + | ((command.utile_height as u64 / 16) << 14) + | samples_log2; + let mut tile_config = 0x280; + if command.layers > 1 { + tile_config |= 1; + } + if command.flags & RENDER_PROCESS_EMPTY_TILES != 0 { + tile_config |= 0x1_0000; + } + + let mut parameters = internal.parameters; + parameters.width = command.width as u32; + parameters.height = command.height as u32; + parameters.encoder = command.vdm_base; + parameters.layers = command.layers; + parameters.utile_width = command.utile_width; + parameters.utile_height = command.utile_height; + parameters.utile_config = utile_config; + parameters.multisample_control = command.multisample_control; + parameters.ppp_control = command.ppp_control as u64; + parameters.tib_blocks = blocks_per_utile; + parameters.tile_config = tile_config; + parameters.depth_dimensions = command.depth_dimensions as u64; + parameters.depth_buffer = command.depth.base; + parameters.depth_aux_buffer = command.depth.compression_base; + parameters.depth_stride = command.depth.stride as u64; + parameters.depth_aux_stride = command.depth.compression_stride as u64; + parameters.stencil_buffer = command.stencil.base; + parameters.stencil_aux_buffer = command.stencil.compression_base; + parameters.stencil_stride = command.stencil.stride as u64; + parameters.stencil_aux_stride = command.stencil.compression_stride as u64; + parameters.depth_flags = command.zls_control; + // The firmware's internal partial-render phases preserve attached Z/S + // surfaces even when the render-pass boundary is store-only. Mesa already + // computes this distinction for its partial BG/EOT programs, but the + // current 256-byte render UAPI carries only the boundary ZLS word. Derive + // the exact uncompressed enable bits here until the append-only partial + // ZLS UAPI field is plumbed; tiling/compression/format bits remain those + // packed into the original word. + parameters.partial_depth_flags = command.zls_control + | if command.depth.base != 0 { (1 << 15) | (1 << 19) } else { 0 } + | if command.stencil.base != 0 { (1 << 14) | (1 << 18) } else { 0 }; + parameters.occlusion_query_base = command.occlusion_query_base; + parameters.scissor_array = command.scissor_base; + parameters.depth_bias_array = command.depth_bias_base; + parameters.sampler_array = command.sampler_heap; + parameters.sampler_count = command.sampler_count as u32; + parameters.process_empty_tiles = command.flags & RENDER_PROCESS_EMPTY_TILES != 0; + parameters.merge_upper_x_bits = command.merge_upper_x; + parameters.merge_upper_y_bits = command.merge_upper_y; + parameters.depth_clear_value_bits = command.depth_clear; + parameters.stencil_clear_value = command.stencil_clear; + parameters.load_pipeline_bind = g17p_bg_resource_spec( + command.background.resource_spec, + resource_spec_mask, + internal.parameters.bg_resource_prefix, + ); + parameters.load_pipeline = background; + parameters.store_pipeline_bind = command.end_of_tile.resource_spec & resource_spec_mask; + parameters.store_pipeline = end_of_tile; + // These high 17 bits are BG shader pass control, not an address class. + // Native derives main and partial masks from their respective shader + // outputs; they need not match. The default diagnostic prefix is zero, + // preserving both full RT0 output masks and no-output masks from userspace. + parameters.partial_load_pipeline_bind = g17p_bg_resource_spec( + command.partial_background.resource_spec, + resource_spec_mask, + internal.parameters.bg_resource_prefix, + ); + parameters.partial_load_pipeline = partial_background; + parameters.partial_store_pipeline_bind = + command.partial_end_of_tile.resource_spec & resource_spec_mask; + parameters.partial_store_pipeline = partial_end_of_tile; + + Ok(TranslatedRenderCommand { + render_order: g17_submission::G17PRenderOrder::default(), + render_barrier: UAPI_BARRIER_NONE, + compute_barrier: UAPI_BARRIER_NONE, + compute_wait_through: None, + flags: command.flags, + vdm_base: command.vdm_base, + parameters, + vertex_attachments, + fragment_attachments, + vertex_timestamps: command.vertex_timestamps, + fragment_timestamps: command.fragment_timestamps, + }) +} + +fn g17p_bg_resource_spec(resource_spec: u64, present_mask: u64, diagnostic_prefix: u64) -> u64 { + diagnostic_prefix | (resource_spec & present_mask) +} + +pub(crate) fn build_render_descriptors( + command: &TranslatedRenderCommand, + objects: RenderDescriptorObjects, + tiling_metadata: RenderDescriptorMetadata, + fragment_metadata: RenderDescriptorMetadata, + tiling: &mut [u8], + fragment: &mut [u8], +) -> Result<(), UapiTranslateError> { + g17_render::build_ta_descriptor(&command.parameters, objects, tiling_metadata, tiling)?; + g17_render::build_fragment_descriptor( + &command.parameters, + objects, + fragment_metadata, + fragment, + )?; + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedComputeCommand { + /// Monotonic logical index, independent of the hardware timestamp. + pub(crate) compute_sequence: u64, + /// Explicit same-engine prefix; only host-published timestamps need host waits. + pub(crate) compute_wait_through: Option, + pub(crate) render_wait_through: Option, + pub(crate) render_wait_last: Option, + pub(crate) usc_exec_base: u64, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) control_stream_terminator: u64, + pub(crate) sampler_heap: u64, + pub(crate) sampler_count: u32, + pub(crate) attachments: UapiAttachmentList, + pub(crate) g17p_add3_buffers: Option<[u64; 3]>, + pub(crate) timestamps: UapiTimestamps, + pub(crate) scs_layout: u64, + /// G18 only: the command carried `DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE`. + /// [`translate_compute_command`] refuses that bit like any other unknown + /// one and leaves this false; the G18 adapter admits the bit, strips it + /// before translation and sets this afterwards. + pub(crate) g18_texture_state: bool, + /// Validated native descriptor inputs; resource/launch integration is + /// separate, so this field does not by itself authorize scratch use. + pub(crate) g18_resource_descriptor: Option<[u32; 23]>, +} + +pub(crate) fn translate_compute_command( + command: UapiComputeCommand, + attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, +) -> Result { + queue.validate()?; + if command.flags & !COMPUTE_G17P_ADD3_PROOF != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + if command.g18_resources != [0; 16] { + return Err(UapiTranslateError::FlagsUnsupported); + } + if command.control_stream_base & 3 != 0 + || command.control_stream_end & 3 != 0 + || command.control_stream_end <= command.control_stream_base + { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.control_stream_base, + command.control_stream_end - command.control_stream_base, + GpuAccess::Read, + )?; + validate_sampler(address_space, command.sampler_heap, command.sampler_count)?; + if command.helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + let g17p_add3_buffers = if command.flags & COMPUTE_G17P_ADD3_PROOF != 0 { + let entries = attachments.as_slice(); + if entries.len() != 3 + || entries + .iter() + .any(|entry| entry.size < G17P_ADD3_BUFFER_BYTES) + { + return Err(UapiTranslateError::Add3ProofAttachments); + } + require_range( + address_space, + entries[0].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Read, + )?; + require_range( + address_space, + entries[1].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Read, + )?; + require_range( + address_space, + entries[2].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Write, + )?; + Some([entries[0].address, entries[1].address, entries[2].address]) + } else { + validate_attachments(address_space, &attachments)?; + None + }; + Ok(TranslatedComputeCommand { + compute_sequence: 0, + compute_wait_through: None, + render_wait_through: None, + render_wait_last: None, + usc_exec_base: queue.base, + control_stream_base: command.control_stream_base, + control_stream_end: command.control_stream_end, + control_stream_terminator: command.control_stream_end - 4, + sampler_heap: command.sampler_heap, + sampler_count: command.sampler_count, + attachments, + g17p_add3_buffers, + timestamps: command.timestamps, + scs_layout: command.scs_layout, + g18_texture_state: false, + g18_resource_descriptor: None, + }) +} + +/// Queue-owned inputs that are independent of one userspace compute command. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeInternalState { + pub(crate) preempt_base: u64, + pub(crate) dispatch_identity: u64, + pub(crate) context_id: u32, + pub(crate) work_ordinal: u32, + pub(crate) robustness: u64, + pub(crate) operand_state_base: u64, + pub(crate) usc_flist_id: u32, +} + +/// GPU addresses resolved from the UAPI timestamp handles while the object +/// table is locked. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeTimestampAddresses { + pub(crate) start: u64, + pub(crate) end: u64, +} + +/// A compute command owns exactly one start/end pair. Both words must be +/// visible before the queue entry and its userspace fence can retire. +pub(crate) const fn completed_compute_timestamps(start: u64, end: u64) -> Option<[u64; 2]> { + if start == 0 || end == 0 { + None + } else { + Some([start, end]) + } +} + +/// Derive the descriptor/RCE checkpoint quartet from the exact command's +/// register program without publishing either object. +pub(crate) fn compute_resume_checkpoints( + command: &TranslatedComputeCommand, + internal: ComputeInternalState, +) -> Result<[u16; 4], UapiTranslateError> { + let registers = g17_compute::build_compute_registers_for_layout( + g17_compute::ComputeRegisterParameters { + preempt_base: internal.preempt_base, + cdm_base: command.control_stream_base, + usc_exec_base: command.usc_exec_base, + helper_binary: 0, + helper_data: 0, + helper_config: 0, + dispatch_identity: internal.dispatch_identity, + context_id: internal.context_id, + work_ordinal: internal.work_ordinal, + robustness: internal.robustness, + operand_state_base: internal.operand_state_base, + usc_flist_id: internal.usc_flist_id, + }, + command.scs_layout, + )?; + Ok(g17_compute::compute_resume_checkpoints(registers.as_slice())?) +} + +/// Build the complete selector-3 descriptor for a translated command. +/// +/// This is the boundary that consumes the command's CDM range, sampler state, +/// USC window, and timestamp destinations. The translated attachment list +/// remains in `command` for the queue's reservation and lifetime tracking. +pub(crate) fn build_compute_descriptor( + command: &TranslatedComputeCommand, + internal: ComputeInternalState, + objects: g17_compute::ComputeDescriptorObjects, + metadata: g17_compute::ComputeDescriptorMetadata, + timestamps: ComputeTimestampAddresses, + raw: &mut [u8], +) -> Result<[u16; 4], UapiTranslateError> { + let registers = g17_compute::build_compute_registers_for_layout( + g17_compute::ComputeRegisterParameters { + preempt_base: internal.preempt_base, + cdm_base: command.control_stream_base, + usc_exec_base: command.usc_exec_base, + helper_binary: 0, + helper_data: 0, + helper_config: 0, + dispatch_identity: internal.dispatch_identity, + context_id: internal.context_id, + work_ordinal: internal.work_ordinal, + robustness: internal.robustness, + operand_state_base: internal.operand_state_base, + usc_flist_id: internal.usc_flist_id, + }, + command.scs_layout, + )?; + let resume_checkpoints = g17_compute::build_compute_descriptor( + ®isters, + objects, + metadata, + g17_compute::ComputeDescriptorCommand { + cdm_terminator: command.control_stream_terminator, + sampler_array: command.sampler_heap, + sampler_count: command.sampler_count, + user_timestamp_start: timestamps.start, + user_timestamp_end: timestamps.end, + }, + raw, + )?; + Ok(resume_checkpoints) +} + +#[derive(Debug, Copy, Clone)] +pub(crate) struct ComputeEntryContext<'a> { + pub(crate) descriptor_flag_4c: bool, + pub(crate) converted_command_timestamp: u64, + pub(crate) barriers: &'a [G17PClBarrierDependency], + pub(crate) mcache: Option, + pub(crate) payload: [u64; 2], + pub(crate) event_mask: [u64; 4], + pub(crate) rce_kind: u8, + pub(crate) rce_bindings: [G17PClRceBinding; 4], + pub(crate) auxiliary: u64, +} + +impl TranslatedComputeCommand { + /// Native `processComputeSetup` writes descriptor +0x5c8 from whether the + /// derived SCS page count is nonzero. `scs_pages_for_layout` has already + /// validated the same layout before an entry can be published; its native + /// zero-page gate is exactly both payload fields being zero. + pub(crate) const fn scs_enabled(&self) -> bool { + self.scs_layout >> 32 != 0 + } + + pub(crate) const fn entry_operands<'a>( + &self, + context: ComputeEntryContext<'a>, + ) -> G17PClKickEntryOperands<'a> { + G17PClKickEntryOperands { + descriptor_flag_4c: context.descriptor_flag_4c, + // `generateCLRegisterListG16Common` uses the same +0x5c8 bit to + // prepend 0x17d9, and encodeCLKickEntry copies it to header bit61. + // Derive both from the command so they cannot diverge. + descriptor_flag_5c8: self.scs_enabled(), + converted_command_timestamp: context.converted_command_timestamp, + barriers: context.barriers, + mcache: context.mcache, + payload: context.payload, + event_mask: context.event_mask, + rce_kind: context.rce_kind, + rce_bindings: context.rce_bindings, + auxiliary: context.auxiliary, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const fn nibble(value: u8) -> u8 { + match value { + b'0'..=b'9' => value - b'0', + b'a'..=b'f' => value - b'a' + 10, + _ => 0xff, + } + } + + const fn hex(source: &[u8; M]) -> [u8; N] { + let mut out = [0u8; N]; + let mut index = 0; + while index < N { + out[index] = (nibble(source[index * 2]) << 4) | nibble(source[index * 2 + 1]); + index += 1; + } + out + } + + const RENDER_FIXTURE: [u8; 312] = hex::<312, 624>(b"02001800ffffffff00000a00000000000001000000000000000000000000000003001800ffffffff00000b0000000000000200000000000000000000000000000000f000ffffffff020000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000088000000000000000000000000000000020200004000300001000000202001040000000000000000040001004000000004000200080000000400010000000000040002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"); + const COMPUTE_FIXTURE: [u8; 104] = hex::<104, 208>(b"04001800ffffffff00000300000000000001000000000000000000000000000001004000ffffffff00000000000000000000010000000000400001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"); + + fn fnv1a64(raw: &[u8]) -> u64 { + let mut value = 0xcbf2_9ce4_8422_2325u64; + for byte in raw { + value ^= *byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + value + } + + struct TestAddressSpace; + + impl GpuAddressSpace for TestAddressSpace { + fn covers(&self, address: u64, size: u64, _access: GpuAccess) -> bool { + let end = match address.checked_add(size) { + Some(value) => value, + None => return false, + }; + (0x1_0000..=0x20_0000).contains(&address) && end <= 0x20_0000 + || (0x11_0000_0000..0x12_0000_0000).contains(&address) && end <= 0x12_0000_0000 + } + } + + fn window() -> QueueUscWindow { + QueueUscWindow { + base: 0x11_0000_0000, + user_start: 0x4000, + user_end: (1 << 39) - 0x8000, + } + } + + fn objects() -> RenderDescriptorObjects { + RenderDescriptorObjects { + record_a: 0xffff_fc20_0150_0000, + shared: 0xffff_fc20_0160_0000, + record_b: 0xffff_fc20_0158_0000, + zero: 0xffff_fc20_0168_0000, + } + } + + #[test] + fn m5_d32_envelope_rejects_wrong_layout_and_incomplete_backing() { + let mut parser = UapiCommandParser::new(&RENDER_FIXTURE); + let ParsedHardwareCommand::Render { mut payload, .. } = + parser.next_hardware().unwrap().unwrap() else { panic!("not render") }; + payload.width = 256; + payload.height = 256; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + payload.depth_dimensions = 0x00ff00ff; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload.depth.base = 0x10000; + payload.depth.stride = 0x3c001; + payload.zls_control = 0x80000; + payload.depth_clear = 0x3f800000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + payload.depth.compression_base = 0x60000; + payload.zls_control = 0x80044; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + let valid = payload; + // The real gears command formerly used G17P's 15-bit dimensions. + payload.depth_dimensions = 0x007f80ff; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload = valid; + payload.depth.compression_base = 0; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload = valid; + payload.depth.base = 0x1fc000; // First byte mapped, remainder absent. + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::AddressRange)); + payload = valid; + payload.depth.compression_base = 0x200000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::AddressRange)); + payload = valid; + // Every combination of the load and store bits is a render the + // application may ask for, including neither: a bound depth buffer + // that this render does not test is not written back. + for transfers in [0u64, 0x8000, 0x80000, 0x88000] { + payload.zls_control = 0x44 | transfers; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Ok(())); + } + // Nothing outside those two bits may move: stencil transfers, the + // twiddled tilings, the 16-bit format and the resolve bits all still + // need their own witnesses. + for stray in [0x2u64, 0x4000, 0x40000, 1 << 25, 1 << 56] { + payload.zls_control = 0x80044 | stray; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + } + payload = valid; + payload.layers = 2; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload = valid; + // The clear value is the application's float and reaches register + // 0x15301 unchanged; only a NaN payload is refused. + for clear in [0u32, 0x3f000000, 0xbf800000, 0x7f800000] { + payload.depth_clear = clear; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Ok(())); + } + payload.depth_clear = 0x7fc00000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + // An attached stencil plane contributes only its compression pair. + payload = valid; + payload.stencil.base = 0x10000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + payload.stencil.compression_base = 0x60000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload.zls_control = 0x80154; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + // Loading or storing it is outside the profile. + for transfer in [0x4000u64, 0x40000] { + payload.zls_control = 0x80154 | transfer; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + } + } + + #[test] + fn m5_resized_depth_validates_tile_padding_and_complete_backing() { + let mut parser = UapiCommandParser::new(&RENDER_FIXTURE); + let ParsedHardwareCommand::Render { mut payload, .. } = + parser.next_hardware().unwrap().unwrap() else { panic!("not render") }; + payload.width = 319; + payload.height = 191; + payload.depth_dimensions = 318 | (190 << 16); + payload.depth.base = 0x10000; + payload.depth.stride = 0x38001; // 5x3 pages, not width*height*4. + payload.depth.compression_base = 0x60000; + payload.depth.compression_stride = 0x7c000; // 512x256 / (16x16) *8. + payload.zls_control = 0x80044; + payload.depth_clear = 0x3f800000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + let valid = payload; + // The array strides are discarded for a single-layer render, so any + // value in them is admitted -- including the underflow a surface + // smaller than one page produces. The backing store below is what + // actually bounds the render. + for stride in [0u32, 0x3c001, 0xffffc001] { + payload.depth.stride = stride; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Ok(())); + } + payload = valid; + payload.depth.compression_stride = 0x3c000; // Old metadata stride. + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + payload = valid; + payload.depth.compression_base = 0; // Metadata stride without metadata. + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload = valid; + payload.depth.base = 0x1c8000; // Only 14 pages remain, but 15 are needed. + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::AddressRange)); + payload = valid; + payload.width = 1024; + payload.height = 1024; + payload.depth_dimensions = 1023 | (1023 << 16); + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::AddressRange)); // Four MiB exceeds this test VM. + } + + #[test] + fn render_fixture_translates_to_full_g17p_programs_and_descriptors() { + assert_eq!(fnv1a64(&RENDER_FIXTURE), 0xf056_ea1d_ef06_32c5); + let mut parser = UapiCommandParser::new(&RENDER_FIXTURE); + let command = parser.next_hardware().unwrap().unwrap(); + let (payload, vertex, fragment) = match command { + ParsedHardwareCommand::Render { + index, + payload, + vertex_attachments, + fragment_attachments, + .. + } => { + assert_eq!(index, 3); + (payload, vertex_attachments, fragment_attachments) + } + _ => panic!("wrong command kind"), + }; + assert!(parser.next_hardware().unwrap().is_none()); + assert_eq!(parser.finish(), Ok(1)); + + let mut internal = G17pRenderParameters::source_rsrc8(); + internal.context_base = 0; + // This frozen descriptor fixture predates userspace high-dword masks; + // retain its explicit legacy diagnostic mask, not a default injection. + internal.bg_resource_prefix = 0x0007_8000_0000_0000; + let translated = translate_render_command( + payload, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: internal, + }, + ) + .unwrap(); + assert_eq!(translated.parameters.load_pipeline, 0x11_0001_0000); + assert_eq!(translated.parameters.store_pipeline, 0x11_0002_0000); + assert_eq!( + translated.parameters.load_pipeline_bind, + 0x0007_8000_0000_0040 + ); + assert_eq!(translated.parameters.store_pipeline_bind, 0x8); + assert_eq!(translated.parameters.partial_load_pipeline, 0x11_0001_0000); + assert_eq!(translated.parameters.partial_store_pipeline, 0x11_0002_0000); + assert_eq!(translated.parameters.utile_config, 0xa000); + assert_eq!(translated.parameters.tib_blocks, 2); + assert_eq!(translated.parameters.tile_config, 0x1_0280); + assert_eq!(translated.parameters.encoder, 0x1_0000); + assert_eq!( + translated.vertex_attachments.as_slice(), + &[UapiAttachment { + address: 0xa_0000, + size: 0x100, + }] + ); + assert_eq!( + translated.fragment_attachments.as_slice(), + &[UapiAttachment { + address: 0xb_0000, + size: 0x200, + }] + ); + + let mut tiling = [0u8; g17_render::TA_DESCRIPTOR_SIZE]; + let mut fragment_raw = [0u8; g17_render::FRAGMENT_DESCRIPTOR_SIZE]; + build_render_descriptors( + &translated, + objects(), + RenderDescriptorMetadata::source(g17_render::RenderDescriptorKind::Tiling), + RenderDescriptorMetadata::source(g17_render::RenderDescriptorKind::Fragment), + &mut tiling, + &mut fragment_raw, + ) + .unwrap(); + assert_eq!(fnv1a64(&tiling), 0x8581_5830_2d5b_e227); + assert_eq!(fnv1a64(&fragment_raw), 0x6688_e9f8_d061_d776); + } + + #[test] + fn background_pass_masks_preserve_userspace_and_explicit_diagnostics() { + let default_prefix = G17pRenderParameters::source_rsrc8().bg_resource_prefix; + assert_eq!(default_prefix, 0); + assert_eq!(g17p_bg_resource_spec(0xffff_8000_0000_0040, u64::MAX, default_prefix), + 0xffff_8000_0000_0040); + assert_eq!(g17p_bg_resource_spec(0x40, u64::MAX, default_prefix), 0x40); + assert_eq!(g17p_bg_resource_spec(0xffff_8000_0000_0040, u32::MAX as u64, default_prefix), + 0x40); + assert_eq!(g17p_bg_resource_spec(0x40, u64::MAX, 0xffff_8000_0000_0000), + 0xffff_8000_0000_0040); + } + + #[test] + fn partial_zls_preserves_depth_and_stencil_across_internal_splits() { + let mut parser = UapiCommandParser::new(&RENDER_FIXTURE); + let (mut payload, vertex, fragment) = match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Render { + payload, + vertex_attachments, + fragment_attachments, + .. + } => (payload, vertex_attachments, fragment_attachments), + _ => panic!("wrong command kind"), + }; + payload.depth.base = 0x1_8000; + payload.stencil.base = 0x1_c000; + payload.zls_control = 0x000c_0000; + + let translated = translate_render_command( + payload, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: G17pRenderParameters::source_rsrc8(), + }, + ) + .unwrap(); + assert_eq!(translated.parameters.depth_flags, 0x000c_0000); + assert_eq!(translated.parameters.partial_depth_flags, 0x000c_c000); + } + + #[test] + fn render_resource_spec_high_dwords_reach_g17_register_programs() { + const RENDER_HEADER: usize = 64; + const PAYLOAD: usize = RENDER_HEADER + UAPI_COMMAND_HEADER_SIZE; + let mut extended = [0u8; RENDER_FIXTURE.len() + 16]; + extended[..RENDER_FIXTURE.len()].copy_from_slice(&RENDER_FIXTURE); + extended[RENDER_HEADER + 2..RENDER_HEADER + 4] + .copy_from_slice(&(UAPI_RENDER_SIZE as u16).to_le_bytes()); + extended[PAYLOAD..PAYLOAD + 4] + .copy_from_slice(&(RENDER_PROCESS_EMPTY_TILES | RENDER_RSRC_SPEC_HI).to_le_bytes()); + + for (offset, value) in [(0xac, 0x42), (0xb4, 0x40), (0xbc, 0x43), (0xc4, 0x41)] { + extended[PAYLOAD + offset..PAYLOAD + offset + 4] + .copy_from_slice(&(value as u32).to_le_bytes()); + } + for (offset, value) in [ + (0xf0, 0x0007_8000), + (0xf4, 0x0007_8001), + (0xf8, 0xffff_8002), + (0xfc, 0), + ] { + extended[PAYLOAD + offset..PAYLOAD + offset + 4] + .copy_from_slice(&(value as u32).to_le_bytes()); + } + + let mut parser = UapiCommandParser::new(&extended); + let (payload, vertex, fragment) = match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Render { + payload, + vertex_attachments, + fragment_attachments, + .. + } => (payload, vertex_attachments, fragment_attachments), + _ => panic!("wrong command kind"), + }; + assert_eq!(parser.finish(), Ok(1)); + let translated = translate_render_command( + payload, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: G17pRenderParameters::source_rsrc8(), + }, + ) + .unwrap(); + assert_eq!( + translated.parameters.store_pipeline_bind, + 0x0007_8000_0000_0040 + ); + assert_eq!( + translated.parameters.partial_store_pipeline_bind, + 0x0007_8001_0000_0041 + ); + assert_eq!( + translated.parameters.load_pipeline_bind, + 0xffff_8002_0000_0042 + ); + assert_eq!( + translated.parameters.partial_load_pipeline_bind, + 0x43 + ); + let registers = g17_render::build_fragment_registers(&translated.parameters).unwrap(); + assert_eq!( + registers + .iter() + .find(|register| register.number == 0x15369) + .unwrap() + .value, + 0xffff_8002_0000_0042 + ); + + // Clearing RSRC_SPEC_HI means "the high dwords are not present", so + // they are masked off -- not grounds to refuse the submission. + let mut no_flag = payload; + no_flag.flags &= !RENDER_RSRC_SPEC_HI; + let masked = translate_render_command( + no_flag, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: G17pRenderParameters::source_rsrc8(), + }, + ) + .unwrap(); + assert_eq!(masked.parameters.load_pipeline_bind, 0x42); + assert_eq!( + masked.parameters.partial_load_pipeline_bind, + 0x43 + ); + + // An explicitly selected diagnostic OR-mask still applies; it is no + // longer silently imposed on ordinary no-output background programs. + let mut diagnostic = G17pRenderParameters::source_rsrc8(); + diagnostic.bg_resource_prefix = 0xffff_8000_0000_0000; + let overridden = translate_render_command( + no_flag, vertex, fragment, window(), &TestAddressSpace, + RenderInternalState { parameters: diagnostic }, + ) + .unwrap(); + assert_eq!(overridden.parameters.load_pipeline_bind, 0xffff_8000_0000_0042); + assert_eq!( + overridden.parameters.partial_load_pipeline_bind, + 0xffff_8000_0000_0043 + ); + } + + #[test] + fn compute_fixture_translates_to_existing_cl_operands() { + assert_eq!(fnv1a64(&COMPUTE_FIXTURE), 0xeb42_2636_cc6a_c274); + let mut parser = UapiCommandParser::new(&COMPUTE_FIXTURE); + let (payload, attachments) = match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Compute { + index, + payload, + attachments, + .. + } => { + assert_eq!(index, 2); + (payload, attachments) + } + _ => panic!("wrong command kind"), + }; + assert!(parser.next_hardware().unwrap().is_none()); + assert_eq!(parser.finish(), Ok(1)); + let translated = + translate_compute_command(payload, attachments, window(), &TestAddressSpace).unwrap(); + assert_eq!(translated.control_stream_base, 0x1_0000); + assert_eq!(translated.control_stream_end, 0x1_0040); + assert_eq!(translated.control_stream_terminator, 0x1_003c); + assert_eq!(translated.usc_exec_base, 0x11_0000_0000); + assert_eq!(translated.g17p_add3_buffers, None); + assert_eq!( + translated.attachments.as_slice(), + &[UapiAttachment { + address: 0x3_0000, + size: 0x100, + }] + ); + + let empty = G17PClRceBinding { address: 0, tag: 0 }; + let operands = translated.entry_operands(ComputeEntryContext { + descriptor_flag_4c: false, + converted_command_timestamp: 1, + barriers: &[], + mcache: None, + payload: [0xffff_fc20_c001_a000, 0xffff_fc20_c000_0480], + event_mask: [1, 2, 3, 4], + rce_kind: 0, + rce_bindings: [empty; 4], + auxiliary: 0x003f_ffff_ffff_ffff, + }); + assert_eq!( + operands.payload, + [0xffff_fc20_c001_a000, 0xffff_fc20_c000_0480] + ); + assert_eq!(operands.event_mask, [1, 2, 3, 4]); + assert_eq!(operands.auxiliary, 0x003f_ffff_ffff_ffff); + assert!(!operands.descriptor_flag_5c8); + + let mut scs_translated = translated; + scs_translated.scs_layout = 0x48u64 << 48; + let scs_operands = scs_translated.entry_operands(ComputeEntryContext { + descriptor_flag_4c: false, + converted_command_timestamp: 1, + barriers: &[], + mcache: None, + payload: [0xffff_fc20_c001_a000, 0xffff_fc20_c000_0480], + event_mask: [1, 2, 3, 4], + rce_kind: 0, + rce_bindings: [empty; 4], + auxiliary: 0x003f_ffff_ffff_ffff, + }); + assert!(scs_operands.descriptor_flag_5c8); + let entry = g17_submission::prepare_g17p_cl_kick_entry( + g17_submission::G17PClKickProducerState { + queue_id: 4, + num_stamps: 4, + entry_stride: 0x200, + completion_selector: 2, + qos: g17_submission::G17PClQosConfig { + queue_byte_32: 0, + class_40: 0, + word_48: 0, + }, + submitted: 0, + current_timestamp: 0, + last_add_kicks_timestamp: 0, + base_timestamp: 0, + previous_valid: false, + previous_timestamp: 0, + }, + operands, + ) + .unwrap(); + assert_eq!(fnv1a64(&entry.entry), 0xbcc4_342d_108e_f9d0); + + let mut descriptor = [0u8; g17_compute::COMPUTE_DESCRIPTOR_SIZE]; + build_compute_descriptor( + &translated, + ComputeInternalState { + preempt_base: 0x100_0404_0000, + dispatch_identity: 0x1234_5678_0000_0001, + context_id: 3, + work_ordinal: 7, + robustness: 0x100_0430_0000, + operand_state_base: 0x70_0022_8000, + usc_flist_id: 2, + }, + g17_compute::ComputeDescriptorObjects { + scheduler_record: 0xffff_fc20_c099_8100, + descriptor_low_alias: 0x70_0034_0000, + dispatch_a: 0xffff_fc20_001c_8028, + dispatch_b: 0xffff_fc20_c07c_0028, + status_a: 0xffff_fc20_0002_4c68, + status_b: 0xffff_fc20_0002_4c70, + shared_control: 0xffff_fc20_c095_0000, + usage_record: 0xffff_fc20_0171_0000, + }, + g17_compute::ComputeDescriptorMetadata { + submit_sequence: 5, + context_id: 3, + grid_index: 2, + work_ordinal: 7, + queue_submission: 8, + queue_ordinal: 7, + submission_index: 9, + support_control: 0xe0a0_0001, + support_flags: 0, + }, + ComputeTimestampAddresses { + start: 0x200_0000_4000, + end: 0x200_0000_4008, + }, + &mut descriptor, + ) + .unwrap(); + assert_eq!( + u64::from_le_bytes(descriptor[0x0ee0..0x0ee8].try_into().unwrap()), + translated.control_stream_terminator + ); + assert_eq!( + u64::from_le_bytes(descriptor[0x0f8c..0x0f94].try_into().unwrap()), + 0x200_0000_4000 + ); + assert_eq!( + u64::from_le_bytes(descriptor[0x8c..0x94].try_into().unwrap()), + translated.usc_exec_base + ); + assert_eq!(completed_compute_timestamps(0, 9), None); + assert_eq!(completed_compute_timestamps(7, 0), None); + assert_eq!(completed_compute_timestamps(7, 9), Some([7, 9])); + } + + #[test] + fn add3_proof_maps_exactly_three_buffers_into_the_retained_table() { + let mut parser = UapiCommandParser::new(&COMPUTE_FIXTURE); + let (mut payload, _) = match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Compute { + payload, + attachments, + .. + } => (payload, attachments), + _ => panic!("wrong command kind"), + }; + payload.flags = COMPUTE_G17P_ADD3_PROOF; + let mut attachments = UapiAttachmentList::EMPTY; + attachments.entries[..3].copy_from_slice(&[ + UapiAttachment { + address: 0x3_0000, + size: G17P_ADD3_BUFFER_BYTES, + }, + UapiAttachment { + address: 0x3_8000, + size: G17P_ADD3_BUFFER_BYTES, + }, + UapiAttachment { + address: 0x4_0000, + size: G17P_ADD3_BUFFER_BYTES, + }, + ]); + attachments.count = 3; + + let translated = + translate_compute_command(payload, attachments, window(), &TestAddressSpace).unwrap(); + assert_eq!( + translated.g17p_add3_buffers, + Some([0x3_0000, 0x3_8000, 0x4_0000]) + ); + + let mut short = attachments; + short.entries[2].size = G17P_ADD3_BUFFER_BYTES - 1; + assert_eq!( + translate_compute_command(payload, short, window(), &TestAddressSpace), + Err(UapiTranslateError::Add3ProofAttachments) + ); + } + + #[test] + fn parser_zero_extends_old_payloads_and_checks_new_tail_bytes() { + let mut command = [0u8; UAPI_COMMAND_HEADER_SIZE + 8]; + command[0..2].copy_from_slice(&COMMAND_COMPUTE.to_le_bytes()); + command[2..4].copy_from_slice(&8u16.to_le_bytes()); + command[4..6].copy_from_slice(&UAPI_BARRIER_NONE.to_le_bytes()); + command[6..8].copy_from_slice(&UAPI_BARRIER_NONE.to_le_bytes()); + let mut parser = UapiCommandParser::new(&command); + match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Compute { payload, .. } => { + assert_eq!( + payload, + UapiComputeCommand { + flags: 0, + sampler_count: 0, + control_stream_base: 0, + control_stream_end: 0, + sampler_heap: 0, + helper: UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }, + timestamps: UapiTimestamps { + start: UapiTimestamp { + handle: 0, + offset: 0, + }, + end: UapiTimestamp { + handle: 0, + offset: 0, + }, + }, + scs_layout: 0, + g18_resources: [0; 16], + prior_vdm: 0, + prior_cdm: 0, + } + ); + } + _ => panic!("wrong command kind"), + } + + let mut extended = [0u8; UAPI_COMPUTE_SIZE + 1]; + extended[UAPI_COMPUTE_SIZE] = 1; + assert_eq!( + UapiComputeCommand::parse(&extended), + Err(UapiTranslateError::PayloadTail) + ); + } + + #[test] + fn translation_checks_sampler_program_and_attachment_ranges() { + let mut parser = UapiCommandParser::new(&RENDER_FIXTURE); + let (mut payload, vertex, fragment) = match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Render { + payload, + vertex_attachments, + fragment_attachments, + .. + } => (payload, vertex_attachments, fragment_attachments), + _ => panic!("wrong command kind"), + }; + payload.sampler_count = 1; + assert_eq!( + translate_render_command( + payload, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: G17pRenderParameters::source_rsrc8(), + }, + ), + Err(UapiTranslateError::SamplerPair) + ); + payload.sampler_count = 0; + payload.end_of_tile.usc = 0; + assert_eq!( + translate_render_command( + payload, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: G17pRenderParameters::source_rsrc8(), + }, + ), + Err(UapiTranslateError::ProgramOffset) + ); + } +} + + +#[cfg(test)] +mod exact_prior_payload_tests { + use super::*; + + #[test] + fn historical_wire_offsets_preserve_scs_and_render_high_words() { + let mut render = [0u8; 272]; + render[0xf0..0xf4].copy_from_slice(&0x12345678u32.to_le_bytes()); + render[256..264].copy_from_slice(&0x123456789abcdef0u64.to_le_bytes()); + render[264..272].copy_from_slice(&0xfedcba9876543210u64.to_le_bytes()); + let p = UapiRenderCommand::parse(&render).unwrap(); + assert_eq!(p.prior_vdm, 0x123456789abcdef0); + assert_eq!(p.prior_cdm, 0xfedcba9876543210); + let old = UapiRenderCommand::parse(&render[..256]).unwrap(); + assert_eq!(old.end_of_tile, p.end_of_tile); + assert_eq!((old.prior_vdm, old.prior_cdm), (0, 0)); + let mut compute = [0u8; 88]; + compute[64..72].copy_from_slice(&0x8877665544332211u64.to_le_bytes()); + compute[72..80].copy_from_slice(&0x0123456789abcdefu64.to_le_bytes()); + compute[80..88].copy_from_slice(&0xfedcba9876543210u64.to_le_bytes()); + let c = UapiComputeCommand::parse(&compute).unwrap(); + assert_eq!(c.scs_layout, 0x8877665544332211); + assert_eq!((c.prior_vdm, c.prior_cdm), (0x0123456789abcdef, 0xfedcba9876543210)); + let old72 = UapiComputeCommand::parse(&compute[..72]).unwrap(); + assert_eq!(old72.scs_layout, c.scs_layout); + assert_eq!((old72.prior_vdm, old72.prior_cdm), (0, 0)); + let old64 = UapiComputeCommand::parse(&compute[..64]).unwrap(); + assert_eq!((old64.scs_layout, old64.prior_vdm, old64.prior_cdm), (0, 0, 0)); + } + + #[test] + fn truncated_extensions_zero_fill_and_unknown_nonzero_tails_reject() { + let mut render = [0u8; 280]; + render[256] = 7; + assert_eq!(UapiRenderCommand::parse(&render[..257]).unwrap().prior_vdm, 7); + assert!(UapiRenderCommand::parse(&render).is_ok()); + render[272] = 1; + assert_eq!(UapiRenderCommand::parse(&render), Err(UapiTranslateError::PayloadTail)); + let mut compute = [0u8; 96]; + compute[80] = 9; + assert_eq!(UapiComputeCommand::parse(&compute[..81]).unwrap().prior_cdm, 9); + assert!(UapiComputeCommand::parse(&compute).is_ok()); + compute[88] = 1; + assert_eq!(UapiComputeCommand::parse(&compute), Err(UapiTranslateError::PayloadTail)); + } +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/compute_inputs.rs b/drivers/gpu/drm/asahi/g18/uapi/compute_inputs.rs new file mode 100644 index 0000000000000..2f6ad2b0cf46f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/compute_inputs.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded G18 compute resource demand, before pool sizing or allocation. +//! +//! Derived from 26A428 HAL400 SpillInfoGen5::checkSpillParamsForCompute and +//! allocateUSCSpillBuffer. It was compared +//! with the original code. This module is not yet +//! connected to submission; it does not admit scratch or allocate GPU memory. + +/// Original Device constructor default (0x1ace020b8 in the exact BaseSystem). +/// Capacity is 64 KiB plus this many 4 KiB units; 63 disables the limit. +pub(crate) const G18_TGMEM_POLICY: u32 = 12; + +/// Inputs in native units. Main and constant-program storage are independent. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ComputeDemand { + pub(crate) scratch_bytes: u32, + pub(crate) constant_scratch_bytes: u32, + pub(crate) registers_32: u32, + pub(crate) ipr_bytes: u32, + pub(crate) constant_ipr_bytes: u32, + pub(crate) threadgroup_bytes: u32, + pub(crate) imageblock_bytes: u32, + /// Zero means unknown local geometry, not an empty global grid. + pub(crate) threads: u32, +} + +/// Effective compiler metadata, including any linked prolog/epilog demand. +/// Counts use Mesa's units. This is an internal type, not a userspace ABI. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ComputeShader { + pub(crate) registers_16: u32, + pub(crate) scratch_bytes: u32, + pub(crate) spectpr_scratch_bytes: u32, + pub(crate) staging_16: u32, + pub(crate) staging_known: bool, + pub(crate) constant_present: bool, + pub(crate) constant_registers_16: u32, + pub(crate) constant_scratch_bytes: u32, + pub(crate) constant_staging_16: u32, + pub(crate) constant_staging_known: bool, + pub(crate) local_bytes: u32, + pub(crate) imageblock_stride: u32, +} + +impl G18ComputeDemand { + /// Legacy metadata v1 and precompiled G17G kernels use byte offsets in + /// element-index fields, requiring a fourfold SPECLM footprint. + pub(crate) fn from_shader(shader: G18ComputeShader, local: Option<[u32; 3]>, + variable_shared_bytes: u32) -> Option { + Self::from_shader_scaled(shader, local, variable_shared_bytes, 4) + } + + /// Metadata v2 describes shaders with correctly lowered element indices. + /// Their static and variable shared storage both count actual bytes. + pub(crate) fn from_shader_element_indices(shader: G18ComputeShader, + local: Option<[u32; 3]>, variable_shared_bytes: u32) -> Option { + Self::from_shader_scaled(shader, local, variable_shared_bytes, 1) + } + + fn from_shader_scaled(shader: G18ComputeShader, local: Option<[u32; 3]>, + variable_shared_bytes: u32, shared_scale: u32) -> Option { + if shader.registers_16 == 0 || shader.registers_16 > 256 + || !shader.staging_known || shader.staging_16 > 0xffff + || shader.imageblock_stride != 0 + { + return None; + } + if shader.constant_present { + if shader.constant_registers_16 == 0 || shader.constant_registers_16 > 256 + || !shader.constant_staging_known || shader.constant_staging_16 > 0xffff + { + return None; + } + } else if shader.constant_registers_16 != 0 || shader.constant_scratch_bytes != 0 + || shader.constant_staging_16 != 0 || shader.constant_staging_known + { + return None; + } + let threads = if let Some(dims) = local { + if dims.contains(&0) { return None; } + dims[0].checked_mul(dims[1])?.checked_mul(dims[2])? + } else { 0 }; + if threads > 1024 { return None; } + let scratch = shader.scratch_bytes.max(shader.spectpr_scratch_bytes); + // Match the qualified static AGX3 TPS frame's 16-byte alignment. + let scratch = scratch.checked_add(15)? & !15; + if scratch > 0xffff * 4 || shader.constant_scratch_bytes > 0xffff * 4 { + return None; + } + let shared = shader.local_bytes.checked_add(variable_shared_bytes)?.checked_mul(shared_scale)?; + let shared = if shared == 0 { 0 } else { (shared.checked_add(63)? & !63).max(256) }; + if shared > 32768 { return None; } + Some(Self { + scratch_bytes: scratch, + constant_scratch_bytes: shader.constant_scratch_bytes, + registers_32: (shader.registers_16 + 1) / 2, + ipr_bytes: shader.staging_16 * 2, + constant_ipr_bytes: shader.constant_staging_16 * 2, + threadgroup_bytes: shared, + imageblock_bytes: 0, + threads, + }) + } +} + +/// Native compute-only accumulator. Keep component maxima until serialization. +/// The two pointer words stay zero; no host or GPU pointer is encoded here. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct G18ComputeInputs { + words: [u32; 58], +} + +impl G18ComputeInputs { + pub(crate) fn new() -> Self { + let mut result = Self { words: [0; 58] }; + result.words[0xc4 / 4] = 4; + result.words[0xc8 / 4] = 1; + result.words[0xcc / 4] = 128; + result + } + + fn maximum(&mut self, offset: usize, value: u64) -> Option<()> { + let value = u32::try_from(value).ok()?; + self.words[offset / 4] = self.words[offset / 4].max(value); + Some(()) + } + + /// Arithmetic domain covers 16-bit SPECTPR word counts, 128 32-bit GPRs, + /// current GL threadgroup memory, and at most 1024 local invocations. + /// Validation is independent of later pool/backing limits. A rejected + /// merge never modifies the caller's accumulated state. + pub(crate) fn merge(&mut self, d: G18ComputeDemand, tgmem_policy: u32) -> Option<()> { + if d.scratch_bytes > 0xffff * 4 + || d.constant_scratch_bytes > 0xffff * 4 + || d.registers_32 > 128 + || d.ipr_bytes > 0xffff * 2 + || d.constant_ipr_bytes > 0xffff * 2 + || d.threadgroup_bytes > 32768 + || d.imageblock_bytes > 32768 + || d.threads > 1024 + || tgmem_policy > 63 + { + return None; + } + + // Calculate in u64, retaining native rounding order. The input bounds + // make every intermediate fit the original unsigned 32-bit domain. + let align = |v: u64, a: u64| (v + a - 1) & !(a - 1); + let pages = |v: u64| (v + 8191) / 8192; + let warps = if d.threads == 0 { 32 } else { (u64::from(d.threads) + 31) / 32 }; + let mut groups = 96 / if d.threads == 0 { 1 } else { warps }; + if d.threadgroup_bytes != 0 && tgmem_policy < 63 { + let capacity = 65536 + u64::from(tgmem_policy) * 4096; + groups = groups.min(capacity / align(u64::from(d.threadgroup_bytes), 64)); + } + let ideal_warps = (groups * warps).min(96); + let minimum_page_warps = warps.min(8); + let ideal_page_warps = (groups * warps).min(8); + + let scratch_warp = align(u64::from(d.scratch_bytes) * 32 + 896, 1024); + let register_warp = align(u64::from(d.registers_32) * 128, 1024); + let ipr_warp = align(u64::from(d.ipr_bytes) * 32, 1024); + let constant_scratch = align(u64::from(d.constant_scratch_bytes) * 32 + 896, 1024); + let constant_ipr = align(u64::from(d.constant_ipr_bytes) * 32, 1024); + let constant_floor = (constant_scratch + constant_ipr) / 1024 + 16; + let warp_kib = (scratch_warp + register_warp + ipr_warp) / 1024; + let local_kib = align(u64::from(d.threadgroup_bytes), 1024) / 1024; + let image_kib = align(u64::from(d.imageblock_bytes), 1024) / 1024; + let scratch_pages = pages(scratch_warp); + let register_pages = pages(register_warp); + let ipr_pages = pages(ipr_warp); + let constant_scratch_pages = pages(constant_scratch); + let constant_ipr_pages = pages(constant_ipr); + let local_pages = pages(u64::from(d.threadgroup_bytes)); + let image_pages = pages(u64::from(d.imageblock_bytes)); + + let mut next = *self; + next.words[0x08 / 4] = 3; + next.maximum(0x44, (warps * warp_kib).max(constant_floor) + local_kib + image_kib)?; + next.maximum(0x84, (ideal_warps * warp_kib).max(constant_floor) + groups * (local_kib + image_kib))?; + next.maximum(0x0c, (minimum_page_warps * register_pages).max(2))?; + next.maximum(0x10, (minimum_page_warps * scratch_pages).max(constant_scratch_pages))?; + next.maximum(0x14, (minimum_page_warps * ipr_pages).max(constant_ipr_pages))?; + next.maximum(0x18, register_pages.max(2))?; + next.maximum(0x1c, scratch_pages.max(constant_scratch_pages))?; + next.maximum(0x20, ipr_pages.max(constant_ipr_pages))?; + next.maximum(0x24, local_pages)?; + next.maximum(0x28, image_pages)?; + next.maximum(0x30, local_pages)?; + next.maximum(0x34, image_pages)?; + next.maximum(0x4c, (ideal_page_warps * register_pages).max(2))?; + next.maximum(0x50, (ideal_page_warps * scratch_pages).max(constant_scratch_pages))?; + next.maximum(0x54, (ideal_page_warps * ipr_pages).max(constant_ipr_pages))?; + next.maximum(0x58, register_pages.max(2))?; + next.maximum(0x5c, scratch_pages.max(constant_scratch_pages))?; + next.maximum(0x60, ipr_pages.max(constant_ipr_pages))?; + next.maximum(0x64, local_pages * groups.min(6))?; + next.maximum(0x68, image_pages * groups.min(6))?; + next.maximum(0x70, local_pages)?; + next.maximum(0x74, image_pages)?; + *self = next; + Some(()) + } + + /// Original endComputePass contributes an all-zero, unknown-geometry job. + pub(crate) fn finish(&mut self) -> Option<()> { + self.merge(G18ComputeDemand::default(), G18_TGMEM_POLICY) + } + + /// Compute-only AGXSMADescRec. Render/ray/protected-memory contributors + /// remain zero and cannot be admitted through this interface. + pub(crate) fn descriptor(&self) -> [u32; 23] { + let w = |off: usize| self.words[off / 4]; + [w(0x44), w(0x0c) + w(0x10) + w(0x14) + w(0x24) + w(0x28), 0, 0, + w(0x84), w(0x4c) + w(0x50) + w(0x54) + w(0x64) + w(0x68), 0, 0, + w(0x58), w(0x5c), w(0x60), w(0x70), w(0x74), 0, 0, 0, + w(0xc8), 0, 0, w(0xcc), w(0xc4), 0, 0] + } + + /// Expose component state for the retained original-instruction audit. + pub(crate) fn component_words(&self) -> &[u32; 58] { + &self.words + } +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/compute_metadata.rs b/drivers/gpu/drm/asahi/g18/uapi/compute_metadata.rs new file mode 100644 index 0000000000000..71eaa91bb705f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/compute_metadata.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Versioned per-dispatch compiler metadata, before any GPU allocation. + +use crate::g18::uapi::compute_inputs::{G18ComputeDemand, G18ComputeInputs, G18ComputeShader, + G18_TGMEM_POLICY}; + +pub(crate) struct G18ComputeMetadata { + pub(crate) demand: G18ComputeDemand, + pub(crate) descriptor: [u32; 23], +} + +impl G18ComputeMetadata { + /// `drm_asahi_g18_compute_resources`, sixteen little-endian u32 words. + /// Unknown local geometry has zero dimensions and its own flag; it must + /// not accidentally become a one-thread group in the native accumulator. + pub(crate) fn decode(words: [u32; 16]) -> Option { + let flags = words[1]; + if !matches!(words[0], 1 | 2) || flags & !0xf != 0 || words[15] != 0 { + return None; + } + let local = if flags & 8 != 0 { + if words[12..15] != [0; 3] { return None; } + None + } else { + Some([words[12], words[13], words[14]]) + }; + let shader = G18ComputeShader { + registers_16: words[2], + scratch_bytes: words[3], + spectpr_scratch_bytes: words[4], + staging_16: words[5], + staging_known: flags & 1 != 0, + constant_present: flags & 2 != 0, + constant_registers_16: words[6], + constant_scratch_bytes: words[7], + constant_staging_16: words[8], + constant_staging_known: flags & 4 != 0, + local_bytes: words[9], + imageblock_stride: words[10], + }; + let demand = if words[0] == 2 { + G18ComputeDemand::from_shader_element_indices(shader, local, words[11])? + } else { + G18ComputeDemand::from_shader(shader, local, words[11])? + }; + let mut inputs = G18ComputeInputs::new(); + inputs.merge(demand, G18_TGMEM_POLICY)?; + inputs.finish()?; + Some(Self { demand, descriptor: inputs.descriptor() }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/compute_plan.rs b/drivers/gpu/drm/asahi/g18/uapi/compute_plan.rs new file mode 100644 index 0000000000000..39222ca5aba43 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/compute_plan.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G18 shared compute pool sizing for the measured J873 profile. +//! +//! Native 26A428 prepareLockedBegin, compute sizing and allocatePagesEnd. +//! This is a pure plan for one active compute job in a fresh/retired pool. +//! It does not allocate memory or enable scratch submission. Compared with +//! the original code. + +pub(crate) const G18_COMPUTE_BLOCK_BYTES: u64 = 0x20_0000; +pub(crate) const G18_COMPUTE_MAX_BYTES: u64 = 0x8000_0000; +const USCS: u64 = 12; +const CHUNKS_PER_USC: u64 = (G18_COMPUTE_MAX_BYTES / 0x1_0000 / USCS) * 64; +pub(crate) const G18_COMPUTE_DIRECTORY_STRIDE: u64 = 0x15_8000; +pub(crate) const G18_COMPUTE_DIRECTORY_PAGE: u64 = 0x4000; + +/// Sparse directory commitment for the measured flags3 / 16-KiB profile. +/// Initial anchor pages remain mapped. Additional pages form the tail of +/// each USC stride; native growth rounds to quarter/half/full stride tiers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18ComputeDirectory { + pub(crate) tail_pages_per_usc: u32, +} + +impl G18ComputeDirectory { + pub(crate) fn for_blocks(blocks: u32) -> Option { + if blocks == 0 || u64::from(blocks) > G18_COMPUTE_MAX_BYTES / G18_COMPUTE_BLOCK_BYTES { + return None; + } + let units = u64::from(blocks) * (G18_COMPUTE_BLOCK_BYTES >> 16); + let per_usc = (units + USCS - 1) / USCS; + let initial = ((G18_COMPUTE_DIRECTORY_PAGE - 0x400) >> 9) + 8; + let excess = per_usc.saturating_sub(initial); + let needed = (excess * 512 + G18_COMPUTE_DIRECTORY_PAGE - 1) + / G18_COMPUTE_DIRECTORY_PAGE; + let stride_pages = G18_COMPUTE_DIRECTORY_STRIDE / G18_COMPUTE_DIRECTORY_PAGE; + let pages = if needed == 0 { 0 } + else if needed <= stride_pages / 4 { stride_pages / 4 } + else if needed <= stride_pages / 2 { stride_pages / 2 } + else { stride_pages - 1 }; + if needed > pages { return None; } + Some(Self { tail_pages_per_usc: pages as u32 }) + } + + pub(crate) fn reservation_bytes(&self) -> u64 { + USCS * G18_COMPUTE_DIRECTORY_STRIDE + G18_COMPUTE_DIRECTORY_PAGE + } + + pub(crate) fn committed_pages(&self) -> u32 { + USCS as u32 + 1 + USCS as u32 * self.tail_pages_per_usc + } + + /// Enumerate anchors first, followed by each USC's contiguous tail. + pub(crate) fn page_offset(&self, index: u32) -> Option { + if index >= self.committed_pages() { return None; } + if index <= USCS as u32 { + return Some(u64::from(index) * G18_COMPUTE_DIRECTORY_STRIDE); + } + let tail = u64::from(self.tail_pages_per_usc); + let index = u64::from(index) - USCS - 1; + let usc = index / tail; + let page = index % tail; + let offset = (usc + 1) * G18_COMPUTE_DIRECTORY_STRIDE + - (tail - page) * G18_COMPUTE_DIRECTORY_PAGE; + if offset + G18_COMPUTE_DIRECTORY_PAGE > self.reservation_bytes() { return None; } + Some(offset) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18ComputeAllocation { + pub(crate) bytes: u64, + pub(crate) block_count: u32, + pub(crate) register_pair: [u32; 2], + pub(crate) chunk_counters: [u32; 2], + pub(crate) chunks_per_usc: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18ComputePlan { + pub(crate) ideal: G18ComputeAllocation, + /// Native minimum is retained for comparison, not an allocation fallback. + pub(crate) minimum: G18ComputeAllocation, + /// Native AGXUMADataRec+0x100, packed above the slot in register0x10241. + pub(crate) sma_units: u32, +} + +impl G18ComputePlan { + /// Serialize the ideal plan only after its backing and aliases are owned. + /// Ownership, address-space validity and retirement are enforced by the + /// pool owner, not this arithmetic helper. The initial hardware slot is + /// unbound; load_hardware_descriptor publishes it in a separate step. + pub(crate) fn pool_state(&self, statistics_id: u64, directory_base: u64, + block_list_gpu: u64, counter_gpu: u64, counter: u64) + -> Option<[u8; crate::g18::fw::uma::G18_POOL_STATE_BYTES]> + { + let directory = G18ComputeDirectory::for_blocks(self.ideal.block_count)?; + if directory_base == 0 || directory_base & 0x3fff != 0 + || block_list_gpu == 0 || block_list_gpu & 0x3fff != 0 + || counter_gpu == 0 || counter_gpu & 7 != 0 + || directory_base.checked_add(directory.reservation_bytes())? > (1 << 48) + || block_list_gpu.checked_add(0x4000)? > (1 << 48) + { + return None; + } + Some(crate::g18::fw::uma::encode_g18_pool_state(&crate::g18::fw::uma::G18UmaPoolState { + statistics_id, grow_engine: 2, priority: 2, + directory_gpu: directory_base + G18_COMPUTE_DIRECTORY_PAGE - 0x400, + directory_stride: G18_COMPUTE_DIRECTORY_STRIDE, + chunk_counters: self.ideal.chunk_counters, + block_list_gpu, populated_blocks: self.ideal.block_count, + block_units_64k: (G18_COMPUTE_BLOCK_BYTES >> 16) as u32, + sma_mode: 0, preempt_counter_gpu: counter_gpu, preempt_counter: counter, + compute: true, policy_value: 2, + })) + } + + /// Bounded compute-only descriptor from G18ComputeInputs::descriptor(). + /// The caller must finish its accumulator and preserve effective linked + /// metadata. Render, protected, alternate-policy and concurrent-job + /// contributions need their own contract and are rejected here. + pub(crate) fn from_descriptor(words: &[u32; 23]) -> Option { + for index in [2, 3, 6, 7, 13, 14, 15, 17, 18, 21, 22] { + if words[index] != 0 { return None; } + } + if words[16] != 1 || words[19] != 128 || words[20] != 4 + || words[0] < 32 || words[1] < 10 + || words[0] > words[4] || words[1] > words[5] + { + return None; + } + let d = words.map(u64::from); + // Original SIMD sequence at 8c916c0..8c91724. Each header + // component rounds separately; summing before rounding is wrong. + let h = &d[8..16]; + let header = h.iter().sum::() + d[16] + + ((h[1] + 131) >> 7) + + ((h[2] + h[3] + 127) >> 7) + + ((h[4] + h[5] + 127) >> 7) + + ((h[6] + h[2] + h[7] + d[16] + 127) >> 7); + // Shared compute reserves its minimum in a separate contribution. + let reserved = d[0] + d[1] + d[19] + d[20]; + let size = |ideal: bool| -> Option { + let registers = if ideal { d[5] } else { d[1] }; + let scratch = if ideal { d[4] } else { d[0] }; + // One active job, up to two native shared-compute slots; native + // helper 8c91ae4 includes the two-slot minimum only for ideal. + let demand = scratch.saturating_sub(d[0] * 2) + + (registers - d[1]) + if ideal { d[1] * 2 } else { 0 }; + let chunks = (demand + reserved + header + 3 + 63) & !63; + if chunks > CHUNKS_PER_USC { return None; } + let bytes = (chunks * USCS * 1024 + G18_COMPUTE_BLOCK_BYTES - 1) + & !(G18_COMPUTE_BLOCK_BYTES - 1); + if bytes == 0 || bytes > G18_COMPUTE_MAX_BYTES { return None; } + // Both launch fields occupy sixteen bits in register 0xa5e1. + if registers > 0xffff || header > 0xffff { return None; } + Some(G18ComputeAllocation { + bytes, + block_count: u32::try_from(bytes / G18_COMPUTE_BLOCK_BYTES).ok()?, + register_pair: [registers as u32, header as u32], + chunk_counters: [0, u32::try_from(demand).ok()?], + chunks_per_usc: u32::try_from(chunks).ok()?, + }) + }; + Some(Self { ideal: size(true)?, minimum: size(false)?, + sma_units: u32::try_from(reserved).ok()? }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/graphics_scratch.rs b/drivers/gpu/drm/asahi/g18/uapi/graphics_scratch.rs new file mode 100644 index 0000000000000..0991e6bcdb5f2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/graphics_scratch.rs @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G18 render (TA/3D) thread-private scratch plan from per-render metadata. +//! +//! Pure arithmetic: no allocation, MMIO, firmware or VM access. Evidence and +//! differential checks: the graphics-scratch design notes. +//! +//! - Per-program accumulator: the native-verified compute arithmetic +//! ([`G18ComputeInputs::merge`]) for one 32-thread warp reproduces all eight +//! native 26A428 graphics shader-only descriptors (closed-form-check-v3). +//! - Fragment pass term: the native RenderContext framebuffer-resource +//! constructor (AGXMetalG18G 1ad040548..1ad040614) and fragment pass merge +//! (1ad027c8c..1ad027e68) for one sample, no user storage and no MSAA +//! extra: 20/20 native comparisons (pass-closed-form-check). +//! - Render sizing: the TA kick, then the 3D kick on the same pool/FList. +//! Native allocatePagesEnd stores the selected per-index parameters in +//! FList+0xbc..0x19c and the next prepareLockedBegin max-merges them +//! (8c91168..8c913f0), so the 3D requirement includes the TA requirement. +//! Component maxima of descriptor words 0..18 then the ideal/minimum pair, +//! chunk counter and bytes below match the original instructions for every +//! in-capacity ideal case and all minimum cases of a 1,220-case grid at +//! 1 and 2 MiB blocks, and all four captured TA/3D register pairs +//! (sequential-kicks, closed-form-check-v3). +//! +//! Not covered (callers must refuse): multisample or layered renders, user +//! (FramebufferDriverConfig+0x904) storage, tile storage above one 8 KiB +//! page, scratch in background/end-of-tile programs, and renders without any +//! scratch (those keep the existing fixed policy unchanged). The native +//! tile-program KiB terms are not fully reproduced (pass-terms-v2); they +//! affect only words 0/4, i.e. chunk counters and bytes, never the pair. + +use crate::g18::uapi::compute_inputs::{G18ComputeDemand, G18ComputeInputs, G18_TGMEM_POLICY}; + +/// `drm_asahi_g18_render_resources.version`. +pub(crate) const G18_RENDER_RESOURCES_VERSION: u32 = 1; +/// Little-endian u32 words in `drm_asahi_g18_render_resources`. +pub(crate) const G18_RENDER_RESOURCE_WORDS: usize = 20; +const FLAG_VERTEX: u32 = 1 << 0; +const FLAG_FRAGMENT: u32 = 1 << 1; +const TILE_PROGRAMS: usize = 4; +const USCS: u64 = 12; +const POOL_MAX_BYTES: u64 = 0x8000_0000; +/// Native per-USC chunk capacity of a 2 GiB, twelve-USC pool +/// (g18_compute_plan.rs uses the same measured geometry). +const CHUNKS_PER_USC: u64 = (POOL_MAX_BYTES / 0x1_0000 / USCS) * 64; +/// Largest tile storage whose pass term is covered by native evidence. +const MAX_TILE_BYTES: u32 = 8192; + +/// One program's demand in Mesa units (16-bit registers, IPR halfwords). +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct G18GraphicsProgram { + pub(crate) registers_16: u32, + pub(crate) scratch_bytes: u32, + pub(crate) staging_16: u32, +} + +impl G18GraphicsProgram { + fn valid(&self) -> bool { + (1..=256).contains(&self.registers_16) + && self.scratch_bytes & 15 == 0 + && self.scratch_bytes <= 0xffff * 4 + && self.staging_16 <= 0xffff + } + + /// Accumulator components of this program alone (one 32-thread warp). + fn components(&self) -> Option<[u32; 58]> { + let mut inputs = G18ComputeInputs::new(); + inputs.merge(G18ComputeDemand { + scratch_bytes: self.scratch_bytes, + constant_scratch_bytes: 0, + registers_32: (self.registers_16 + 1) / 2, + ipr_bytes: self.staging_16 * 2, + constant_ipr_bytes: 0, + threadgroup_bytes: 0, + imageblock_bytes: 0, + threads: 32, + }, G18_TGMEM_POLICY)?; + Some(*inputs.component_words()) + } +} + +/// Decoded `drm_asahi_g18_render_resources` plus the command's tile storage. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderMetadata { + pub(crate) vertex: G18GraphicsProgram, + pub(crate) fragment: G18GraphicsProgram, + /// Background, end-of-tile, partial background, partial end-of-tile. + pub(crate) tiles: [Option; TILE_PROGRAMS], + pub(crate) tile_bytes: u32, +} + +impl G18RenderMetadata { + /// `tile_bytes` comes from the validated render command (utile area x + /// bytes per sample x samples), never from userspace metadata. + pub(crate) fn decode(words: &[u32; G18_RENDER_RESOURCE_WORDS], tile_bytes: u32) -> Option { + if words[0] != G18_RENDER_RESOURCES_VERSION + || words[1] != (FLAG_VERTEX | FLAG_FRAGMENT) + || words[16..].iter().any(|&w| w != 0) + || tile_bytes == 0 || tile_bytes > MAX_TILE_BYTES + { + return None; + } + let stage = |at: usize| G18GraphicsProgram { + registers_16: words[at], scratch_bytes: words[at + 1], staging_16: words[at + 2], + }; + let (vertex, fragment) = (stage(2), stage(5)); + if !vertex.valid() || !fragment.valid() + || (vertex.scratch_bytes == 0 && fragment.scratch_bytes == 0) + { + return None; + } + let mut tiles = [None; TILE_PROGRAMS]; + for (index, slot) in tiles.iter_mut().enumerate() { + let program = G18GraphicsProgram { + registers_16: words[8 + index], scratch_bytes: 0, staging_16: words[12 + index], + }; + if program.registers_16 == 0 { + if program.staging_16 != 0 { return None; } + } else if program.valid() { + *slot = Some(program); + } else { + return None; + } + } + Some(Self { vertex, fragment, tiles, tile_bytes }) + } + + /// TA (vertex stage) descriptor: the vertex program alone. + pub(crate) fn ta_descriptor(&self) -> Option<[u32; 23]> { + Some(serialize(&self.vertex.components()?)) + } + + /// 3D (fragment stage) descriptor: component maxima of the fragment and + /// tile programs, then the native pass merge of (fragment + pass terms). + pub(crate) fn fragment_descriptor(&self) -> Option<[u32; 23]> { + let main = self.fragment.components()?; + let mut acc = main; + for program in self.tiles.iter().flatten() { + let words = program.components()?; + for (a, b) in acc.iter_mut().zip(words.iter()) { *a = (*a).max(*b); } + } + let kib = (self.tile_bytes + 1023) / 1024; + let pages = (self.tile_bytes + 8191) / 8192; + let mut passed = main; + passed[0x44 / 4] = passed[0x44 / 4].checked_add(kib)?; + passed[0x84 / 4] = passed[0x84 / 4].checked_add(kib.checked_mul(11)?)?; + for (offset, value) in [(0x28, pages), (0x68, pages * 6), (0x74, pages)] { + passed[offset / 4] = passed[offset / 4].max(value); + } + for (a, b) in acc.iter_mut().zip(passed.iter()) { *a = (*a).max(*b); } + Some(serialize(&acc)) + } + + /// Per-render plan for a pool of `block_bytes` blocks. + pub(crate) fn plan(&self, block_bytes: u64) -> Option { + if !matches!(block_bytes, 0x10_0000 | 0x20_0000) { return None; } + let ta = self.ta_descriptor()?; + let fragment = self.fragment_descriptor()?; + // The 3D kick follows the TA kick on the same FList: its requirement + // is the component maximum of both descriptors (words 0..18). + let mut d3 = fragment; + for (a, b) in d3[..19].iter_mut().zip(ta[..19].iter()) { *a = (*a).max(*b); } + Some(G18RenderScratchPlan { + ideal: [size(&ta, false, block_bytes)?, size(&d3, false, block_bytes)?], + minimum: [size(&ta, true, block_bytes)?, size(&d3, true, block_bytes)?], + }) + } +} + +/// Native AGXSMADescRec serialization of the 58-word accumulator +/// (identical to `G18ComputeInputs::descriptor`). +fn serialize(c: &[u32; 58]) -> [u32; 23] { + let w = |off: usize| c[off / 4]; + [w(0x44), w(0x0c) + w(0x10) + w(0x14) + w(0x24) + w(0x28), 0, 0, + w(0x84), w(0x4c) + w(0x50) + w(0x54) + w(0x64) + w(0x68), 0, 0, + w(0x58), w(0x5c), w(0x60), w(0x70), w(0x74), 0, 0, 0, + w(0xc8), 0, 0, w(0xcc), w(0xc4), 0, 0] +} + +/// One kick's selected native result. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderStagePlan { + /// Launch pair, packed `[0] | [1] << 16` in 0xa5e9 (TA) / 0xa5f1 (3D). + pub(crate) register_pair: [u32; 2], + /// Native FList chunk counter (+0xb0) for this kick. + pub(crate) chunk_counter: u32, + /// Native per-USC chunk reservation (64-aligned). + pub(crate) chunks_per_usc: u32, + pub(crate) bytes: u64, +} + +impl G18RenderStagePlan { + pub(crate) fn resource_word(&self) -> u64 { + u64::from(self.register_pair[0]) | u64::from(self.register_pair[1]) << 16 + } +} + +fn size(d: &[u32; 23], minimum: bool, block_bytes: u64) -> Option { + let d = d.map(u64::from); + // Original SIMD sequence at 8c916c0..8c91724 (see g18_compute_plan.rs): + // each header component rounds separately. + let h = &d[8..16]; + let header = h.iter().sum::() + d[16] + + ((h[1] + 131) >> 7) + + ((h[2] + h[3] + 127) >> 7) + + ((h[4] + h[5] + 127) >> 7) + + ((h[6] + h[2] + h[7] + d[16] + 127) >> 7); + let (first, counter) = if minimum { (d[1], d[0] + d[19]) } else { (d[5], d[4] + d[1] + d[19]) }; + let chunks = (counter + header + 3 + 63) & !63; + if chunks > CHUNKS_PER_USC || first > 0xffff || header > 0xffff { return None; } + let bytes = (chunks * USCS * 1024 + block_bytes - 1) & !(block_bytes - 1); + if bytes == 0 || bytes > POOL_MAX_BYTES { return None; } + Some(G18RenderStagePlan { + register_pair: [u32::try_from(first).ok()?, u32::try_from(header).ok()?], + chunk_counter: u32::try_from(counter).ok()?, + chunks_per_usc: u32::try_from(chunks).ok()?, + bytes, + }) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G18RenderScratchMode { + Ideal, + /// Native's below-ideal selection (allocatePagesEnd minimum flag). + Minimum, +} + +/// Both native results for `[TA, 3D]`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderScratchPlan { + pub(crate) ideal: [G18RenderStagePlan; 2], + pub(crate) minimum: [G18RenderStagePlan; 2], +} + +/// The selection the job applies to its TA/3D resource words. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderScratchSelection { + pub(crate) mode: G18RenderScratchMode, + pub(crate) stages: [G18RenderStagePlan; 2], +} + +impl G18RenderScratchPlan { + /// Ideal when both kicks fit the owned pool, else the native minimum + /// when both fit, else none. `owned_chunks_per_usc` is the chunk counter + /// already published in the owned pool state; neither the pool nor its + /// state is resized by this selection. + pub(crate) fn select(&self, owned_bytes: u64, owned_chunks_per_usc: u32) + -> Option + { + let fits = |stages: &[G18RenderStagePlan; 2]| stages.iter() + .all(|s| s.bytes <= owned_bytes && s.chunks_per_usc <= owned_chunks_per_usc); + if fits(&self.ideal) { + Some(G18RenderScratchSelection { mode: G18RenderScratchMode::Ideal, stages: self.ideal }) + } else if fits(&self.minimum) { + Some(G18RenderScratchSelection { mode: G18RenderScratchMode::Minimum, stages: self.minimum }) + } else { + None + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/mod.rs b/drivers/gpu/drm/asahi/g18/uapi/mod.rs new file mode 100644 index 0000000000000..4a1095e31ca92 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/mod.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! User command parsing, admission and per-command planning for G18 (T8152). + +#[allow(dead_code)] +pub(crate) mod application; +#[allow(dead_code)] +pub(crate) mod application_runtime; +#[allow(dead_code)] +pub(crate) mod command; +#[allow(dead_code)] +pub(crate) mod compute_inputs; +#[allow(dead_code)] +pub(crate) mod compute_metadata; +#[allow(dead_code)] +pub(crate) mod compute_plan; +#[allow(dead_code)] +pub(crate) mod graphics_scratch; +#[allow(dead_code)] +pub(crate) mod render_policy; diff --git a/drivers/gpu/drm/asahi/g18/uapi/render_policy.rs b/drivers/gpu/drm/asahi/g18/uapi/render_policy.rs new file mode 100644 index 0000000000000..84170ed8be79c --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/render_policy.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded H1/application policy shared by the kernel and host. +//! Target kinds 1/2 use native geometry; mode 0 retains the stage33d policy. +use crate::{g18::fw::render_params::G17pRenderParameters, g18::address_map as va, + g18::fw::render::G18RenderControls, g18::fw::render_command::*, g18::fw::render_assembly as g18_render_assembly}; + +pub(crate) const PB_BLOCKS: u32 = 8; // plan: overprovision >=8; no PB growth. +pub(crate) const PB_CAPACITY: u32 = 1650; +pub(crate) const UMA_BLOCKS: u32 = 24; // 24 MiB > sum of both M5 budgets. +pub(crate) const USC_REQUIRED: [u64; 2] = [0x7d4000, 0x934000]; // M5 DRM hardware path. +// G18 native SMAV1 first-use unit pair, proven by the compute fixture. The +// graphics byte budgets above remain M5 policy; no claim of derived sizing. +pub(crate) const RESOURCE_PAIR: [u64; 2] = [16, 6]; +pub(crate) const UMA_CHUNKS: [u32; 2] = [0, + ((USC_REQUIRED[0] + USC_REQUIRED[1] + 12 * 1024 - 1) / (12 * 1024)) as u32]; + +// Native SMAV1 minimum-result serialization for the captured witness. Each +// stage needs 3 MiB, below the unchanged H1 backing/accounting. This probes the +// two resource words only; it does not claim native ideal allocation policy. +pub(crate) const NATIVE_MINIMUM_PAIRS: [[u64; 2]; 2] = [[4, 8], [9, 11]]; +#[cfg(not(CONFIG_RUST))] +pub(crate) static HOST_SMA_MINIMUM: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false); + +// Fixed first-draw witness policy, from the original M6 SMAV1 methods. This +// experiment is admitted only for stage9/variant0/profile4 before MMIO. +pub(crate) const NATIVE_IDEAL_PAIRS: [[u64; 2]; 2] = [[24, 8], [42, 11]]; +pub(crate) const IDEAL_GROWTH_BYTES: u64 = 21 * va::PAGE; +pub(crate) const IDEAL_GROWTH_OFFSET: u64 = 0x158000 - IDEAL_GROWTH_BYTES; +#[cfg(not(CONFIG_RUST))] +pub(crate) static HOST_SMA_IDEAL: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false); + +pub(crate) fn ideal_enabled() -> bool { + #[cfg(CONFIG_RUST)] { crate::g18::params::g18_sma_ideal.value() == 1 } + #[cfg(not(CONFIG_RUST))] { HOST_SMA_IDEAL.load(core::sync::atomic::Ordering::Relaxed) } +} +pub(crate) fn ta_barrier_flag() -> u32 { + #[cfg(CONFIG_RUST)] { crate::g18::params::g18_native_barrier.value() } + #[cfg(not(CONFIG_RUST))] { 0 } +} +pub(crate) fn uma_blocks() -> u32 { if ideal_enabled() { 40 } else { UMA_BLOCKS } } +/// Block size of the owned render pool (prepare_render_backing). +pub(crate) const RENDER_POOL_BLOCK_BYTES: u64 = 0x10_0000; +/// Capacity of the owned render pool as published in its state: backing +/// bytes and the per-USC chunk counter. Graphics scratch selections must fit +/// both; they never resize the pool (g18_graphics_scratch.rs). +pub(crate) fn owned_scratch_capacity() -> (u64, u32) { + (u64::from(uma_blocks()) * RENDER_POOL_BLOCK_BYTES, uma_chunks()[1]) +} +pub(crate) fn uma_chunks() -> [u32; 2] { if ideal_enabled() { [0, 708 + 2305] } else { UMA_CHUNKS } } +pub(crate) fn block_base() -> u64 { if ideal_enabled() { va::GRAPHICS_IDEAL_BLOCKS } else { va::POOL_BLOCKS } } +pub(crate) fn usc_required() -> [u64; 2] { if ideal_enabled() { [9 * 0x100000, 28 * 0x100000] } else { USC_REQUIRED } } + +pub(crate) fn resource_pairs() -> [[u64; 2]; 2] { + #[cfg(CONFIG_RUST)] + let minimum = crate::g18::params::g18_sma_minimum.value() == 1; + #[cfg(not(CONFIG_RUST))] + let minimum = HOST_SMA_MINIMUM.load(core::sync::atomic::Ordering::Relaxed); + if ideal_enabled() { NATIVE_IDEAL_PAIRS } else if minimum { NATIVE_MINIMUM_PAIRS } else { [RESOURCE_PAIR; 2] } +} + +// Keep registration in this owned file. This is the same KernelParam ABI as +// kernel::module!, with load-time-only access (perm=0), so a live job cannot +// change layout under its mappings. Default 1 matches the parsed colour/gears +// force-kind-1 flag. Invalid selectors fail target_plan before publication. +#[cfg(CONFIG_RUST)] +mod parameter { + use kernel::{bindings, module_param::{KernelParam, ModuleParamAccess, PARAM_OPS_U32}}; + pub(super) static VALUE: ModuleParamAccess = ModuleParamAccess::new(1); + #[link_section = "__param"] + #[used(compiler)] + static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g18_render_target_kind" + } else { c"aurora.g18_render_target_kind" }), + // SAFETY: module lifetime and parameter serialization are managed by + // the kernel, identically to the kernel::module! registration. + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PARAM_OPS_U32), perm: 0, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: VALUE.as_void_ptr() }, + }); + #[link_section = ".modinfo"] + #[used(compiler)] + static TYPE: [u8; b"parmtype=g18_render_target_kind:uint\0".len()] = *b"parmtype=g18_render_target_kind:uint\0"; + #[link_section = ".modinfo"] + #[used(compiler)] + static DESCRIPTION: [u8; b"parm=g18_render_target_kind:0 legacy, 1 native single target (default), 2 native clustered target\0".len()] = + *b"parm=g18_render_target_kind:0 legacy, 1 native single target (default), 2 native clustered target\0"; +} + +#[cfg(not(CONFIG_RUST))] +pub(crate) static HOST_TARGET_KIND: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(1); + +// Fragment resource word 0xa5f1 for the admitted 4x render. Apple's run112 +// renders use 0xb0022 (34,11) at 1x and 0xd002e (46,13) at 4x: the size class +// grows by log2(samples). With the 1x minimum word (9,11) the 4x draw faulted +// into firmware recovery (run164). Default (9,13): the minimum count at the 4x +// size class. 0 keeps the 1x word. 0644 (C uint ops, like the g18_app_* +// knobs), read at every 4x render. +#[cfg(CONFIG_RUST)] +mod msaa4_fragment { + use kernel::{bindings, module_param::KernelParam}; + pub(super) static VALUE: core::sync::atomic::AtomicU32 = + core::sync::atomic::AtomicU32::new(0xd_0009); + #[link_section = "__param"] + #[used(compiler)] + static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g18_msaa4_fragment_word" + } else { c"aurora.g18_msaa4_fragment_word" }), + #[cfg(MODULE)] + mod_: &raw mut bindings::__this_module, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + // SAFETY: exported vmlinux ops; they access `arg` as an unsigned int + // under the parameter lock, which AtomicU32 represents exactly. + ops: unsafe { core::ptr::from_ref(&bindings::param_ops_uint) }, + perm: 0o644, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { + arg: core::ptr::from_ref(&VALUE).cast_mut().cast(), + }, + }); + #[link_section = ".modinfo"] + #[used(compiler)] + static TYPE: [u8; b"parmtype=g18_msaa4_fragment_word:uint\0".len()] = + *b"parmtype=g18_msaa4_fragment_word:uint\0"; + #[link_section = ".modinfo"] + #[used(compiler)] + static DESCRIPTION: [u8; b"parm=g18_msaa4_fragment_word:0xa5f1 for the admitted 4x render, count|log2<<16 (default 0xd0009; native 0xd002e; 0 = 1x word)\0".len()] = + *b"parm=g18_msaa4_fragment_word:0xa5f1 for the admitted 4x render, count|log2<<16 (default 0xd0009; native 0xd002e; 0 = 1x word)\0"; +} + +pub(crate) fn msaa4_fragment_word() -> u64 { + #[cfg(CONFIG_RUST)] + { u64::from(msaa4_fragment::VALUE.load(core::sync::atomic::Ordering::Relaxed)) } + #[cfg(not(CONFIG_RUST))] + { 0xd_0009 } +} + +pub(crate) fn target_kind() -> u32 { + #[cfg(CONFIG_RUST)] { parameter::VALUE.value() } + #[cfg(not(CONFIG_RUST))] { HOST_TARGET_KIND.load(core::sync::atomic::Ordering::Relaxed) } +} + +/// All non-capture fields are either assigned here or disabled explicitly by +/// the default zero optional-feature policy. No value is taken from post-run +/// firmware memory or a captured kernel pointer. +pub(crate) fn h1_inputs(command: &[u8], context: u32, + p: &mut G17pRenderParameters, c: &mut G18RenderControls, +) -> G18RenderCommandInputs { + let (i, _) = crate::g18::fw::render_passthrough::capture_fields(command); + owned_inputs(i, context, p, c) +} + +/// Shared first-use policy; application callers supply their own descriptor +/// fields and then replace frame identity and PB addresses via owner accessors. +pub(crate) fn owned_inputs(mut i: G18RenderCommandInputs, context: u32, + p: &mut G17pRenderParameters, c: &mut G18RenderControls) -> G18RenderCommandInputs { + // Linux first-pass lifecycle and work identity, shared with command copies. + p.lifecycle = 1u64 << 32; p.fragment_lifecycle = p.lifecycle; + c.ta.work_key = u64::from(context) << 8; c.fragment.work_key = c.ta.work_key; + let pairs = resource_pairs(); + c.ta.resource_word = pairs[0][0] | pairs[0][1] << 16; + c.fragment.resource_word = pairs[1][0] | pairs[1][1] << 16; + // Only the admitted 4x render carries a sample control (msaa163). + let msaa4_word = msaa4_fragment_word(); + if c.fragment_sample_control != 0 && msaa4_word != 0 { + c.fragment.resource_word = msaa4_word; + } + i.context_id = context; i.scheduler_record = va::GRAPHICS_SCHEDULER_START + 0x4000; + i.fragment.sequence = 0; i.ta.sequence = 1; i.fragment.pass = 1; + for stage in 0..2 { + let qid = va::QUEUE_IDS[stage]; + let completion = G18StageCompletion { + stamp_addresses: [va::GRAPHICS_SIGNALS + 0x4000 + u64::from(qid)*4, + va::GRAPHICS_SIGNALS + 0x8000 + u64::from(qid)*4], + stamp: 0x100, queue_index: u32::from(qid), ordinal: 0, + has_wait: stage == 1, completion: (1u64 << 40) | 1, + key: if stage == 0 {2} else {1}, + }; + if stage == 0 { i.ta.completion = completion; } else { i.fragment.completion = completion; } + } + i.ta.linked_3d = G18TaLinked3d { queue_index: 10, stamp: 0x100, flag: 0, + completion: i.fragment.completion.completion }; + i.ta.pb_slot = 0; i.ta.pm_1d4 = 0; // M5 first-pass PM policy. + i.ta.pb_state = va::GRAPHICS_PB_HIGH; i.fragment.pm_state = i.ta.pb_state; + i.ta.pm_scene = va::PB_GRAPH + 0x1080; i.fragment.pm_scene = i.ta.pm_scene; + i.ta.pm_b8 = va::PB_GRAPH + 0x3800; i.fragment.pm_b8 = i.ta.pm_b8; + i.ta.uma_state = va::GRAPHICS_UMA_STATE; i.fragment.uma_state = i.ta.uma_state; + // Native render-target-loader fields with the M5 geometry of a 256x256, + // 32x32, single-layer target, sized with the register worker's 2x policy. + let plan = g18_render_assembly::target_plan(p).expect("admitted H1 geometry"); + g18_render_assembly::apply_target_controls(p, c, &plan); + i.ta.d11b8 = p.tpc; + i.ta.d11d8 = plan.native.map_or(plan.layout.tpc_bytes, |n| n.slice_sizes[0]) as u64; + i.ta.d10f8 = p.deflake_1; + i.fragment.d08d8 = p.tilemap; + i.fragment.d08f0 = p.multisample_control; + i.fragment.d08f8 = u32::from(p.samples); + i.fragment.d0900 = 1; + i.fragment.d0908 = 0x40004; + i.fragment.d0910 = [0, 0]; + i.fragment.d0920 = 0; + i.fragment.d0930 = 64; + if plan.native.is_some() { + let tx = (p.width + 31) / 32; + let ty = (p.height + 31) / 32; + i.fragment.d08f8 = 0; // target +0x20: native single-sample value + // msaa163: same values as g18_application_runtime (4 / 0x40008 for + // the admitted 4x render; unchanged otherwise). + i.fragment.d0900 = crate::g18::fw::render::g18_sample_count(p); + i.fragment.d0908 = crate::g18::fw::render::g18_macro_size(p); + i.fragment.d0910 = [0, u64::from(p.merge_upper_x_bits) | (u64::from(p.merge_upper_y_bits) << 32)]; + i.fragment.d0930 = tx * ty; + } + i.ta.d08a4 = 1; i.ta.d08a8 = 0; i.ta.d08ac = 0; + i.fragment.d08a4 = 1; i.fragment.d08a8 = 0; i.fragment.d08ac = 0; + // Passes per scene, as native (by the H1 parameter-buffer audit): + // dc98 = pass count, dc9c = 1 for a single pass. The encoder copies dc98 + // to 3D +0x21f9; firmware adds it to PB state +0x00 per 3D completion and + // adds 1 to +0x14 per TA completion (b000 0xcc1c, 0xd214). The PB is + // idle, and one a recovery flagged can be reset, only while the two + // balance (0xcca8, 0x11c94). Zero here left every flushed scene's PB + // blocking the render queues for the rest of the boot. + i.fragment.dc98 = 1; i.fragment.dc9c = 1; + // M5 TA activation bit; no optional policy/event/history payloads. + i.ta.d11a4 = 1; + i.ta.d08b0 = va::GRAPHICS_SIGNALS + 0xc000; + i.fragment.d08b0 = [va::GRAPHICS_SIGNALS + 0xc010, va::GRAPHICS_SIGNALS + 0xc018]; + // Native UMA data-record tail: required/allocated byte accounting. + let required = usc_required(); + i.ta.d1418 = [0, 0]; i.ta.d1428 = required[0]; i.ta.d1430 = 0; + i.fragment.dad0 = [0, 0]; i.fragment.dae0 = required[1]; i.fragment.dae8 = 0; + i +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index f100820a0c75e..551e801ba2a29 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -210,6 +210,21 @@ impl ObjectRef { vm.map_at(addr, self.gem.size(), &self.gem, prot, guard) } + + /// Map at a specific address, reserving `guard_pages` unmapped pages + /// after the object in that Vm's allocator (T8152, module 1134). + #[cfg(CONFIG_DRM_ASAHI_G18)] + pub(crate) fn map_at_with_guards( + &mut self, + vm: &crate::mmu::Vm, + addr: u64, + prot: mmu::Prot, + guard_pages: usize, + ) -> Result { + validate_vm_binding(&self.gem, vm)?; + + vm.map_at_with_guards(addr, self.gem.size(), &self.gem, prot, guard_pages) + } } pub(crate) struct AsahiObjConfig { @@ -395,6 +410,12 @@ impl Drop for KernelMappingPin { } impl AsahiObject { + /// Whether this is a kernel-created object. + #[cfg(CONFIG_DRM_ASAHI_G18)] + pub(crate) fn is_kernel(&self) -> bool { + self.kernel + } + /// Whether CPU mappings use the cacheable WRITEBACK policy. pub(crate) fn is_writeback(&self) -> bool { self.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK != 0 diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index c54ffb5c208fe..438d071de971c 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -19,6 +19,8 @@ mod t604x; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; +#[cfg(CONFIG_DRM_ASAHI_G18)] +pub(crate) mod t8152; /// GPU generation enumeration. Note: Part of the UABI. #[derive(Debug, PartialEq, Copy, Clone)] diff --git a/drivers/gpu/drm/asahi/hw/t8152.rs b/drivers/gpu/drm/asahi/hw/t8152.rs new file mode 100644 index 0000000000000..a50477ee21c38 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8152.rs @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8152 platforms (M6, G18G). +//! +//! The M6 GPU runs on the G18 backend (`crate::g18`), which uses its own +//! firmware interface and therefore an identification-only SoC entry rather +//! than a [`super::HwConfig`]. + +use crate::g18::identity::{GpuGen, GpuVariant}; + +/// Identification-only configuration for one AGX3 SoC. +#[allow(dead_code)] +#[derive(Debug)] +pub(crate) struct SocConfig { + /// Chip ID in hex format (e.g. 0x6050 for t6050). + pub(crate) chip_id: u32, + /// Decoded design generation for this chip. + pub(crate) gpu_gen: GpuGen, + /// Decoded variant for this chip. + pub(crate) gpu_variant: GpuVariant, + /// Hardware family byte expected at `0xd04000[31:24]`. + pub(crate) hw_family: u8, + /// Hardware variant byte expected at `0xd04000[23:16]`. + pub(crate) hw_variant: u8, + /// Die count expected at `0xd04010[19:16]`. + pub(crate) num_dies: u32, + /// Input-address bits per UAT root (42 on every AGX3 part). + pub(crate) uat_ias: u8, + /// Output address space (42-bit on every AGX3 part). + pub(crate) uat_oas: u32, +} + +const fn agx3( + chip_id: u32, + gpu_gen: GpuGen, + gpu_variant: GpuVariant, + hw_family: u8, + hw_variant: u8, + num_dies: u32, +) -> SocConfig { + SocConfig { + chip_id, + gpu_gen, + gpu_variant, + hw_family, + hw_variant, + num_dies, + uat_ias: 42, + uat_oas: 42, + } +} + +// G18 (M6 family). Family byte 0xc, variant 2: 26A428 +// `AGX·PI_400·G·A0·Accelerator::readChipInfo` @ 0xfffffe0008c3d3a0 compares +// the family byte with 0xc @ 0xfffffe0008c3d49c and gives variant 2 the +// 12-core G configuration @ 0xfffffe0008c3d4bc..0x4c8. Native J873 registry +// (AGXAcceleratorG18G): gpu_gen=18, gpu_var=G, usc_gen=3, is_sksm=1, +// num_cores=12, num_gps=6, num_mgpus=2, core_mask_list=(63,63), +// num_frags=12, kickid_qid_shift=40, kickid_qid_mask=127. The live ID +// registers have not been read yet (the GPU PSD reads idle at cold boot), so +// the T8152 probe identifies the part by chip id and compatible, and only +// logs a mismatching register decode. The ADT uat-vaddr-size is 43 on both +// J813 and J873: 42 bits per root plus the TTBR-select bit, as on T8142. +/// Apple M6 (G18G, J873 Mac mini). +pub(crate) const T8152: SocConfig = agx3(0x8152, GpuGen::G18, GpuVariant::G, 0xc, 2, 1); + +/// Native J873 `AGXAcceleratorG18G` registry facts (macOS 26A428 ioreg). +/// +/// These are identity facts, not firmware tuning: they size the core-mask +/// decode and cross-check the live ID registers once the GPU is powered. +#[allow(dead_code)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RegistryFacts { + pub(crate) num_cores: u32, + pub(crate) num_gps: u32, + pub(crate) num_mgpus: u32, + /// Per-mGPU (cluster) core masks, `core_mask_list`. + pub(crate) core_mask_list: [u32; 2], + pub(crate) num_frags: u32, + pub(crate) usc_generation: u32, + pub(crate) kickid_qid_shift: u32, + pub(crate) kickid_qid_mask: u32, +} + +/// J873 registry facts (PROVEN from the native ioreg). +#[allow(dead_code)] +pub(crate) const T8152_REGISTRY: G18RegistryFacts = G18RegistryFacts { + num_cores: 12, + num_gps: 6, + num_mgpus: 2, + core_mask_list: [0x3f, 0x3f], + num_frags: 12, + usc_generation: 3, + kickid_qid_shift: 40, + kickid_qid_mask: 127, +}; diff --git a/drivers/gpu/drm/asahi/m5/channel.rs b/drivers/gpu/drm/asahi/m5/channel.rs new file mode 100644 index 0000000000000..259d79209a0f7 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/channel.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8142/25G76 channel configuration, separate from Neo's HAL200 path. +//! +//! Native AGXCLChannelSKSM::submitBuffer inlines this packed 0x86-byte +//! record at 0xfffffe000898488c..0x8984bd8. The controlled native sequence +//! is a selector-3 register program, this tag-15 record, then a tag-14 kick. +//! These builders do not map memory, install a queue or publish a producer. +//! Every pointer and event identifier must belong to the retained session. + +pub(crate) const M5_CHANNEL_CONFIG_BYTES: usize = 0x86; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5ChannelError { + QueueId, + Priority, + Address, + Context, + DataMaster, +} + +/// Optional TA parameter-buffer event, separate from the first event tuple. +/// The native TA writer at 25G76 0x8a249ac..0x8a249e4 obtains these values +/// from its retained PM manager and command descriptor. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ParameterBufferEvent { + pub(crate) address: u64, + pub(crate) timestamp: u64, + /// TA descriptor +0x1218, independently selected by the owned PB table. + pub(crate) hardware_buffer_id: u32, + /// Native PM manager +0x79c. This is not the SKSM queue ID. + pub(crate) manager_id: u32, +} + +/// Optional retained event-machine object, with the native timestamp and +/// index. None emits the native disabled branch's full-width sentinels. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ChannelEvent { + pub(crate) address: u64, + pub(crate) timestamp: u64, + pub(crate) index: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ChannelConfig { + pub(crate) ring_low: u64, + pub(crate) ring_high: u64, + pub(crate) queue_id: u8, + pub(crate) priority: u8, + /// Native descriptor +0x181 bit 0. Mode 3 validates the first queue + /// installation on M5; an already-installed native capture has 0. + pub(crate) install_queue: bool, + /// Native descriptor +0x144, independently matched to the UAT context. + pub(crate) gpu_context: u8, + pub(crate) event: Option, + /// AGXCommandQueue +0x8b8: retained scheduler/control object. + pub(crate) channel_control: u64, + /// Native descriptor +0x2dc bit 0. + pub(crate) update_scheduler: bool, + /// AGXCommandQueue +0x898 object's +0x10. Not the GPU context ID. + pub(crate) scheduler_index: u32, + /// getStampIndex on descriptor +0x170. Not a fixed target constant. + pub(crate) channel_stamp_index: u32, + /// Native descriptor +0x180 bit 0. + pub(crate) update_context: bool, + /// Descriptor +0x58 owner's virtual context value. Distinct from UAT. + pub(crate) owner_context: u32, + /// Native channel-state +0x84 is nonzero. + pub(crate) extended_command: bool, +} + +fn firmware_address(address: u64) -> bool { + address >= 0xffff_fc00_0000_0000 && address & 7 == 0 +} + +fn put32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +pub(crate) fn encode_m5_channel_config( + p: M5ChannelConfig, +) -> Result<[u8; M5_CHANNEL_CONFIG_BYTES], M5ChannelError> { + encode_m5_channel_config_for(p, 2, 2, None, None) +} + +/// The two selectors are independent: native TA/fragment use 0/1 at +0x22 +/// while both retain 2 at +0x5e. Changing both by stage breaks that contract. +/// gpu_state_update retains the separate +0x26 flag/+0x2a value pair; +/// the native TA writer copies descriptor +0xe41/+0xe48 to these fields. +pub(crate) fn encode_m5_channel_config_for( + p: M5ChannelConfig, + data_master: u32, + channel_data_master: u32, + gpu_state_update: Option, + parameter_buffer: Option, +) -> Result<[u8; M5_CHANNEL_CONFIG_BYTES], M5ChannelError> { + if data_master > 2 + || channel_data_master > 2 + || (parameter_buffer.is_some() && data_master != 0) + { + return Err(M5ChannelError::DataMaster); + } + if p.queue_id > 127 { + return Err(M5ChannelError::QueueId); + } + if p.priority > 3 { + return Err(M5ChannelError::Priority); + } + if p.gpu_context == 0 { + return Err(M5ChannelError::Context); + } + if p.ring_low == 0 + || p.ring_low >= (1 << 42) + || p.ring_low & 0x3fff != 0 + || !firmware_address(p.ring_high) + || p.ring_high & 0x3fff != 0 + || !firmware_address(p.channel_control) + { + return Err(M5ChannelError::Address); + } + if let Some(event) = p.event { + if !firmware_address(event.address) { + return Err(M5ChannelError::Address); + } + } + if let Some(event) = parameter_buffer { + if !firmware_address(event.address) { + return Err(M5ChannelError::Address); + } + } + let mut bytes = [0; M5_CHANNEL_CONFIG_BYTES]; + put32(&mut bytes, 0, 15); + put64(&mut bytes, 8, p.ring_low); + put64(&mut bytes, 0x10, p.ring_high); + bytes[0x18..0x1a].copy_from_slice(&u16::from(p.queue_id).to_le_bytes()); + for (offset, value) in [ + (0x1a, u32::from(p.install_queue)), + (0x1e, u32::from(p.priority)), + (0x22, data_master), + (0x32, u32::from(p.gpu_context)), + (0x52, u32::from(p.update_scheduler)), + (0x56, p.scheduler_index), + (0x5a, p.channel_stamp_index), + (0x5e, channel_data_master), + (0x62, u32::from(p.update_context)), + (0x66, p.owner_context), + (0x6a, u32::from(p.extended_command)), + ] { + put32(&mut bytes, offset, value); + } + if let Some(value) = gpu_state_update { + put32(&mut bytes, 0x26, 1); + put64(&mut bytes, 0x2a, value); + } + match p.event { + Some(event) => { + put64(&mut bytes, 0x36, event.address); + put64(&mut bytes, 0x3e, event.timestamp); + put32(&mut bytes, 0x46, event.index); + } + None => { + put64(&mut bytes, 0x3e, u64::MAX); + put32(&mut bytes, 0x46, u32::MAX); + } + } + put64(&mut bytes, 0x4a, p.channel_control); + if let Some(event) = parameter_buffer { + put64(&mut bytes, 0x6e, event.address); + put64(&mut bytes, 0x76, event.timestamp); + put32(&mut bytes, 0x7e, event.hardware_buffer_id); + put32(&mut bytes, 0x82, event.manager_id); + } else { + bytes[0x76..0x86].fill(0xff); + } + Ok(bytes) +} + diff --git a/drivers/gpu/drm/asahi/m5/command.rs b/drivers/gpu/drm/asahi/m5/command.rs new file mode 100644 index 0000000000000..58e0ca0eb6670 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/command.rs @@ -0,0 +1,1203 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +use crate::m5::{render_desc, submission}; + +use render_desc::{ + G17pRenderParameters, RenderBuildError, +}; + +pub(crate) const UAPI_COMMAND_HEADER_SIZE: usize = 8; + +pub(crate) const UAPI_RENDER_SIZE: usize = 272; + +pub(crate) const UAPI_COMPUTE_SIZE: usize = 88; + +pub(crate) const UAPI_G18_COMPUTE_SIZE: usize = 136; + +pub(crate) const COMPUTE_G18_RESOURCES: u32 = 1 << 2; + +/// `drm_asahi_cmd_render_g18`: the render payload plus its 80-byte suffix. +pub(crate) const UAPI_G18_RENDER_SIZE: usize = UAPI_RENDER_SIZE + 80; + +/// `DRM_ASAHI_RENDER_G18_RESOURCES`. Only the G18 adapter may consume and +/// strip it; the shared translator's RENDER_FLAGS keeps refusing it. +pub(crate) const RENDER_G18_RESOURCES: u32 = 1 << 21; + +pub(crate) const UAPI_ATTACHMENT_SIZE: usize = 24; + +pub(crate) const UAPI_MAX_ATTACHMENTS: usize = 16; + +pub(crate) const UAPI_MAX_HARDWARE_COMMANDS: u32 = 64; + +pub(crate) const M5_MAX_RENDER_DIMENSION: u32 = 16384; + +pub(crate) const UAPI_BARRIER_NONE: u16 = 0xffff; + +pub(crate) const USC_WINDOW_SIZE: u64 = 1 << 32; + +pub(crate) const COMPUTE_G17P_ADD3_PROOF: u32 = + kernel::uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF as u32; + +const G17P_ADD3_BUFFER_BYTES: u64 = 64 * 4; + +const COMMAND_RENDER: u16 = 0; + +const COMMAND_COMPUTE: u16 = 1; + +const SET_VERTEX_ATTACHMENTS: u16 = 2; + +const SET_FRAGMENT_ATTACHMENTS: u16 = 3; + +const SET_COMPUTE_ATTACHMENTS: u16 = 4; + +const RENDER_VERTEX_SCRATCH: u32 = 1 << 0; + +const RENDER_PROCESS_EMPTY_TILES: u32 = 1 << 1; + +const RENDER_NO_VERTEX_CLUSTERING: u32 = 1 << 2; + +const RENDER_RSRC_SPEC_HI: u32 = 1 << 4; + +const RENDER_DBIAS_IS_INT: u32 = 1 << 18; + +const RENDER_FLAGS: u32 = RENDER_VERTEX_SCRATCH + | RENDER_PROCESS_EMPTY_TILES + | RENDER_NO_VERTEX_CLUSTERING + | RENDER_RSRC_SPEC_HI + | RENDER_DBIAS_IS_INT + // Private, opt-in matched-workload diagnostic. Ordinary builds reject it. + | if cfg!(g17p_aux_fb_small_probe) { 1 << 31 } else { 0 } + | if cfg!(g17p_mcache_aperture_probe) { 1 << 30 } else { 0 }; + +const SAMPLER_SIZE: u64 = 8; + +const MAX_SAMPLERS: u32 = 1024; + +const PAGE_SIZE: u64 = 0x4000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum UapiTranslateError { + HeaderTruncated, + PayloadTruncated, + PayloadTail, + CommandType, + AttachmentPayloadSize, + AttachmentCount, + AttachmentFields, + AttachmentBarrier, + HardwareCommandCount, + BarrierFuture, + QueueUscAlignment, + QueueUscRange, + AddressRange, + AddressAlignment, + FlagsUnknownBits, + FlagsUnsupported, + Dimensions, + Layers, + UtileDimensions, + Samples, + TilebufferSize, + SamplerCount, + SamplerPair, + HelperProgram, + Add3ProofAttachments, + ProgramOffset, + ZlsFields, + RenderBuild(RenderBuildError), +} + +impl From for UapiTranslateError { + fn from(value: RenderBuildError) -> Self { + Self::RenderBuild(value) + } +} + +impl UapiTranslateError { +} + +fn get_u16(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) +} + +fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) +} + +fn padded_payload(raw: &[u8]) -> Result<[u8; N], UapiTranslateError> { + if raw.len() > N && raw[N..].iter().any(|byte| *byte != 0) { + return Err(UapiTranslateError::PayloadTail); + } + let mut out = [0u8; N]; + let count = core::cmp::min(raw.len(), N); + out[..count].copy_from_slice(&raw[..count]); + Ok(out) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiCommandHeader { + pub(crate) command_type: u16, + pub(crate) size: u16, + pub(crate) render_barrier: u16, + pub(crate) compute_barrier: u16, +} + +impl UapiCommandHeader { + fn parse(raw: &[u8]) -> Result { + if raw.len() < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + Ok(Self { + command_type: get_u16(raw, 0), + size: get_u16(raw, 2), + render_barrier: get_u16(raw, 4), + compute_barrier: get_u16(raw, 6), + }) + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct UapiAttachment { + pub(crate) address: u64, + pub(crate) size: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiAttachmentList { + pub(crate) entries: [UapiAttachment; UAPI_MAX_ATTACHMENTS], + pub(crate) count: u8, +} + +impl UapiAttachmentList { + pub(crate) const EMPTY: Self = Self { + entries: [UapiAttachment { + address: 0, + size: 0, + }; UAPI_MAX_ATTACHMENTS], + count: 0, + }; + + pub(crate) fn as_slice(&self) -> &[UapiAttachment] { + &self.entries[..self.count as usize] + } + + fn parse(raw: &[u8]) -> Result { + if raw.len() % UAPI_ATTACHMENT_SIZE != 0 { + return Err(UapiTranslateError::AttachmentPayloadSize); + } + let count = raw.len() / UAPI_ATTACHMENT_SIZE; + if count > UAPI_MAX_ATTACHMENTS { + return Err(UapiTranslateError::AttachmentCount); + } + let mut out = Self::EMPTY; + for index in 0..count { + let offset = index * UAPI_ATTACHMENT_SIZE; + if get_u32(raw, offset + 0x10) != 0 || get_u32(raw, offset + 0x14) != 0 { + return Err(UapiTranslateError::AttachmentFields); + } + out.entries[index] = UapiAttachment { + address: get_u64(raw, offset), + size: get_u64(raw, offset + 8), + }; + } + out.count = count as u8; + Ok(out) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiHelperProgram { + pub(crate) binary: u32, + pub(crate) config: u32, + pub(crate) data: u64, +} + +fn helper(raw: &[u8], offset: usize) -> UapiHelperProgram { + UapiHelperProgram { + binary: get_u32(raw, offset), + config: get_u32(raw, offset + 4), + data: get_u64(raw, offset + 8), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiProgram { + pub(crate) usc: u32, + pub(crate) resource_spec: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamp { + pub(crate) handle: u32, + pub(crate) offset: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamps { + pub(crate) start: UapiTimestamp, + pub(crate) end: UapiTimestamp, +} + +fn timestamps(raw: &[u8], offset: usize) -> UapiTimestamps { + UapiTimestamps { + start: UapiTimestamp { + handle: get_u32(raw, offset), + offset: get_u32(raw, offset + 4), + }, + end: UapiTimestamp { + handle: get_u32(raw, offset + 8), + offset: get_u32(raw, offset + 12), + }, + } +} + +fn program(raw: &[u8], offset: usize, high_offset: usize) -> UapiProgram { + UapiProgram { + usc: get_u32(raw, offset), + resource_spec: get_u32(raw, offset + 4) as u64 + | (get_u32(raw, high_offset) as u64) << 32, + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiZlsBuffer { + pub(crate) base: u64, + pub(crate) compression_base: u64, + pub(crate) stride: u32, + pub(crate) compression_stride: u32, +} + +fn zls(raw: &[u8], offset: usize) -> UapiZlsBuffer { + UapiZlsBuffer { + base: get_u64(raw, offset), + compression_base: get_u64(raw, offset + 8), + stride: get_u32(raw, offset + 0x10), + compression_stride: get_u32(raw, offset + 0x14), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiRenderCommand { + pub(crate) flags: u32, + pub(crate) depth_dimensions: u32, + pub(crate) vdm_base: u64, + pub(crate) vertex_helper: UapiHelperProgram, + pub(crate) fragment_helper: UapiHelperProgram, + pub(crate) scissor_base: u64, + pub(crate) depth_bias_base: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth: UapiZlsBuffer, + pub(crate) stencil: UapiZlsBuffer, + pub(crate) zls_control: u64, + pub(crate) multisample_control: u64, + pub(crate) sampler_heap: u64, + pub(crate) ppp_control: u32, + pub(crate) width: u16, + pub(crate) height: u16, + pub(crate) layers: u16, + pub(crate) sampler_count: u16, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) samples: u8, + pub(crate) sample_size: u8, + pub(crate) merge_upper_x: u32, + pub(crate) merge_upper_y: u32, + pub(crate) background: UapiProgram, + pub(crate) end_of_tile: UapiProgram, + pub(crate) partial_background: UapiProgram, + pub(crate) partial_end_of_tile: UapiProgram, + pub(crate) depth_clear: u32, + pub(crate) stencil_clear: u32, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, + pub(crate) prior_vdm: u64, + pub(crate) prior_cdm: u64, + /// `drm_asahi_g18_render_resources` words; zero unless flagged. + pub(crate) g18_resources: [u32; 20], +} + +impl UapiRenderCommand { + fn parse(raw: &[u8]) -> Result { + // Legacy payloads are zero-extended exactly as before. A flagged G18 + // suffix must be complete; padding could hide a truncated field. + let has_resources = raw.len() >= 4 && get_u32(raw, 0) & RENDER_G18_RESOURCES != 0; + if has_resources && raw.len() < UAPI_G18_RENDER_SIZE { + return Err(UapiTranslateError::PayloadTruncated); + } + if !has_resources && raw.len() > UAPI_RENDER_SIZE + && raw[UAPI_RENDER_SIZE..].iter().any(|byte| *byte != 0) + { + return Err(UapiTranslateError::PayloadTail); + } + let raw = padded_payload::(raw)?; + let mut g18_resources = [0; 20]; + if has_resources { + for (i, word) in g18_resources.iter_mut().enumerate() { + *word = get_u32(&raw, UAPI_RENDER_SIZE + i * 4); + } + } + Ok(Self { + flags: get_u32(&raw, 0x00), + depth_dimensions: get_u32(&raw, 0x04), + vdm_base: get_u64(&raw, 0x08), + vertex_helper: helper(&raw, 0x10), + fragment_helper: helper(&raw, 0x20), + scissor_base: get_u64(&raw, 0x30), + depth_bias_base: get_u64(&raw, 0x38), + occlusion_query_base: get_u64(&raw, 0x40), + depth: zls(&raw, 0x48), + stencil: zls(&raw, 0x60), + zls_control: get_u64(&raw, 0x78), + multisample_control: get_u64(&raw, 0x80), + sampler_heap: get_u64(&raw, 0x88), + ppp_control: get_u32(&raw, 0x90), + width: get_u16(&raw, 0x94), + height: get_u16(&raw, 0x96), + layers: get_u16(&raw, 0x98), + sampler_count: get_u16(&raw, 0x9a), + utile_width: raw[0x9c], + utile_height: raw[0x9d], + samples: raw[0x9e], + sample_size: raw[0x9f], + merge_upper_x: get_u32(&raw, 0xa0), + merge_upper_y: get_u32(&raw, 0xa4), + background: program(&raw, 0xa8, 0xf8), + end_of_tile: program(&raw, 0xb0, 0xf0), + partial_background: program(&raw, 0xb8, 0xfc), + partial_end_of_tile: program(&raw, 0xc0, 0xf4), + depth_clear: get_u32(&raw, 0xc8), + stencil_clear: get_u32(&raw, 0xcc), + vertex_timestamps: timestamps(&raw, 0xd0), + fragment_timestamps: timestamps(&raw, 0xe0), + prior_vdm: get_u64(&raw, 0x100), + prior_cdm: get_u64(&raw, 0x108), + g18_resources, + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiComputeCommand { + pub(crate) flags: u32, + pub(crate) sampler_count: u32, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) sampler_heap: u64, + pub(crate) helper: UapiHelperProgram, + pub(crate) timestamps: UapiTimestamps, + pub(crate) scs_layout: u64, + pub(crate) g18_resources: [u32; 16], + pub(crate) prior_vdm: u64, + pub(crate) prior_cdm: u64, +} + +impl UapiComputeCommand { + fn parse(raw: &[u8]) -> Result { + // Legacy compute payloads are zero-extended as before. A flagged + // metadata payload must be complete; padding it could hide truncation + // of a valid zero-valued field or reserved word. + let has_resources = raw.len() >= 4 && get_u32(raw, 0) & COMPUTE_G18_RESOURCES != 0; + if has_resources && raw.len() < UAPI_G18_COMPUTE_SIZE { + return Err(UapiTranslateError::PayloadTruncated); + } + if !has_resources && raw.len() > UAPI_COMPUTE_SIZE + && raw[UAPI_COMPUTE_SIZE..].iter().any(|byte| *byte != 0) + { + return Err(UapiTranslateError::PayloadTail); + } + let raw = padded_payload::(raw)?; + let mut g18_resources = [0; 16]; + if has_resources { + for (i, word) in g18_resources.iter_mut().enumerate() { + *word = get_u32(&raw, 72 + i * 4); + } + } + Ok(Self { + flags: get_u32(&raw, 0), + sampler_count: get_u32(&raw, 4), + control_stream_base: get_u64(&raw, 8), + control_stream_end: get_u64(&raw, 0x10), + sampler_heap: get_u64(&raw, 0x18), + helper: helper(&raw, 0x20), + timestamps: timestamps(&raw, 0x30), + scs_layout: get_u64(&raw, 0x40), + g18_resources, + prior_vdm: if has_resources { 0 } else { get_u64(&raw, 0x48) }, + prior_cdm: if has_resources { 0 } else { get_u64(&raw, 0x50) }, + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ParsedHardwareCommand { + Render { + index: u32, + header: UapiCommandHeader, + payload: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + }, + Compute { + index: u32, + header: UapiCommandHeader, + payload: UapiComputeCommand, + attachments: UapiAttachmentList, + }, +} + +pub(crate) struct UapiCommandParser<'a> { + raw: &'a [u8], + offset: usize, + command_index: u32, + hardware_count: u32, + render_count: u16, + compute_count: u16, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + compute_attachments: UapiAttachmentList, +} + +impl<'a> UapiCommandParser<'a> { + pub(crate) const fn new(raw: &'a [u8]) -> Self { + Self { + raw, + offset: 0, + command_index: 0, + hardware_count: 0, + render_count: 0, + compute_count: 0, + vertex_attachments: UapiAttachmentList::EMPTY, + fragment_attachments: UapiAttachmentList::EMPTY, + compute_attachments: UapiAttachmentList::EMPTY, + } + } + + pub(crate) fn finish(&self) -> Result { + if self.offset != self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + if self.hardware_count == 0 || self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + Ok(self.hardware_count) + } + + pub(crate) fn next_hardware( + &mut self, + ) -> Result, UapiTranslateError> { + loop { + if self.offset == self.raw.len() { + return Ok(None); + } + if self.raw.len() - self.offset < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + let header = UapiCommandHeader::parse(&self.raw[self.offset..])?; + self.offset += UAPI_COMMAND_HEADER_SIZE; + let end = self + .offset + .checked_add(header.size as usize) + .ok_or(UapiTranslateError::PayloadTruncated)?; + if end > self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + let payload = &self.raw[self.offset..end]; + self.offset = end; + self.command_index += 1; + + match header.command_type { + COMMAND_RENDER | COMMAND_COMPUTE => { + if header.render_barrier != UAPI_BARRIER_NONE + && header.render_barrier > self.render_count + { + return Err(UapiTranslateError::BarrierFuture); + } + if header.compute_barrier != UAPI_BARRIER_NONE + && header.compute_barrier > self.compute_count + { + return Err(UapiTranslateError::BarrierFuture); + } + self.hardware_count += 1; + if self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + if header.command_type == COMMAND_RENDER { + self.render_count += 1; + return Ok(Some(ParsedHardwareCommand::Render { + index: self.command_index, + header, + payload: UapiRenderCommand::parse(payload)?, + vertex_attachments: self.vertex_attachments, + fragment_attachments: self.fragment_attachments, + })); + } + self.compute_count += 1; + return Ok(Some(ParsedHardwareCommand::Compute { + index: self.command_index, + header, + payload: UapiComputeCommand::parse(payload)?, + attachments: self.compute_attachments, + })); + } + SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS | SET_COMPUTE_ATTACHMENTS => { + if header.render_barrier != UAPI_BARRIER_NONE + || header.compute_barrier != UAPI_BARRIER_NONE + { + return Err(UapiTranslateError::AttachmentBarrier); + } + let attachments = UapiAttachmentList::parse(payload)?; + match header.command_type { + SET_VERTEX_ATTACHMENTS => self.vertex_attachments = attachments, + SET_FRAGMENT_ATTACHMENTS => self.fragment_attachments = attachments, + SET_COMPUTE_ATTACHMENTS => self.compute_attachments = attachments, + _ => unreachable!(), + } + } + _ => return Err(UapiTranslateError::CommandType), + } + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum GpuAccess { + Read, + Write, + ReadWrite, +} + +pub(crate) trait GpuAddressSpace { + fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool; +} + +fn require_range( + address_space: &A, + address: u64, + size: u64, + access: GpuAccess, +) -> Result<(), UapiTranslateError> { + let end = address + .checked_add(size) + .ok_or(UapiTranslateError::AddressRange)?; + if size == 0 || end <= address || !address_space.covers(address, size, access) { + return Err(UapiTranslateError::AddressRange); + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueueUscWindow { + pub(crate) base: u64, + pub(crate) user_start: u64, + pub(crate) user_end: u64, +} + +impl QueueUscWindow { + pub(crate) fn validate(self) -> Result<(), UapiTranslateError> { + if self.base & (USC_WINDOW_SIZE - 1) != 0 { + return Err(UapiTranslateError::QueueUscAlignment); + } + let end = self + .base + .checked_add(USC_WINDOW_SIZE) + .ok_or(UapiTranslateError::QueueUscRange)?; + if self.base < self.user_start || end > self.user_end || end <= self.base { + return Err(UapiTranslateError::QueueUscRange); + } + Ok(()) + } + + fn full_program( + self, + address_space: &A, + tagged_offset: u32, + ) -> Result { + self.validate()?; + if tagged_offset == 0 { + return Err(UapiTranslateError::ProgramOffset); + } + let allocation = self + .base + .checked_add((tagged_offset & !7) as u64) + .ok_or(UapiTranslateError::ProgramOffset)?; + require_range(address_space, allocation, 4, GpuAccess::Read)?; + // Honeykrisp carries resource metadata in the low three UAPI bits, + // but G17P's native 3D command strips those bits before programming + // every BG/EOT/partial USC address. Preserving bit 2 launched four + // bytes into the instruction stream and contradicted all complete + // Metal register-list captures. + Ok(allocation) + } +} + +fn validate_sampler( + address_space: &A, + heap: u64, + count: u32, +) -> Result<(), UapiTranslateError> { + if count > MAX_SAMPLERS { + return Err(UapiTranslateError::SamplerCount); + } + if (heap == 0) != (count == 0) { + return Err(UapiTranslateError::SamplerPair); + } + if count != 0 { + if heap & (SAMPLER_SIZE - 1) != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + heap, + count as u64 * SAMPLER_SIZE, + GpuAccess::Read, + )?; + } + Ok(()) +} + +fn validate_attachments( + address_space: &A, + attachments: &UapiAttachmentList, +) -> Result<(), UapiTranslateError> { + for attachment in attachments.as_slice() { + require_range( + address_space, + attachment.address, + attachment.size, + GpuAccess::Write, + )?; + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderInternalState { + pub(crate) parameters: G17pRenderParameters, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedRenderCommand { + /// Exact logical render prefix assigned by the DRM ioctl dispatcher. + pub(crate) render_order: submission::G17PRenderOrder, + /// Original UAPI render index, retained through admission. + pub(crate) render_barrier: u16, + /// Original UAPI compute index, retained through admission. + pub(crate) compute_barrier: u16, + /// Exact compute prefix resolved at ioctl admission. Later independent + /// compute commands must not become parents of this render. + pub(crate) compute_wait_through: Option, + pub(crate) flags: u32, + pub(crate) vdm_base: u64, + pub(crate) parameters: G17pRenderParameters, + pub(crate) vertex_attachments: UapiAttachmentList, + pub(crate) fragment_attachments: UapiAttachmentList, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, +} + +fn validate_zls( + address_space: &A, + value: UapiZlsBuffer, + layers: u16, + g18_subpage: bool, +) -> Result<(), UapiTranslateError> { + if value.base == 0 + && (value.compression_base != 0 || value.stride != 0 || value.compression_stride != 0) + { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_base == 0 && value.compression_stride != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if !g18_subpage && layers > 1 && value.base != 0 && value.stride == 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.stride != 0 && value.stride & 0x3fff != 1 + && !(g18_subpage && value.stride & 0x3fff == 0) { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_stride & 0x3fff != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.base != 0 { + let layer_stride = if g18_subpage && layers > 1 { + let unit = if value.stride & 1 != 0 { PAGE_SIZE } else { 0x80 }; + ((value.stride as u64 >> 14) + 1) * unit + } else if value.stride == 0 { + 0 + } else { + ((value.stride as u64 >> 14) + 1) * PAGE_SIZE + }; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range(address_space, value.base, size, GpuAccess::ReadWrite)?; + } + if value.compression_base != 0 { + let layer_stride = ((value.compression_stride as u64 >> 14) + 1) * 0x80; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range( + address_space, + value.compression_base, + size, + GpuAccess::ReadWrite, + )?; + } + Ok(()) +} + +/// Validate the complete single-layer M5 Z/S allocations before their unused +/// array strides are discarded. AIL stores samples in an expanded 2D surface: +/// 2x doubles Y, and 4x doubles both axes. The dimensions register stays in pixels. +pub(crate) fn validate_m5_reference_depth( + command: &UapiRenderCommand, + address_space: &A, +) -> Result<(), UapiTranslateError> { + let width = u32::from(command.width); + let height = u32::from(command.height); + if !(1..=M5_MAX_RENDER_DIMENSION).contains(&width) + || !(1..=M5_MAX_RENDER_DIMENSION).contains(&height) + || command.layers != 1 || !matches!(command.samples, 1 | 2 | 4) { + return Err(UapiTranslateError::ZlsFields); + } + let sample_width = width * if command.samples == 4 { 2 } else { 1 }; + let sample_height = height * if command.samples > 1 { 2 } else { 1 }; + let depth = command.depth; + let stencil = command.stencil; + let z16 = command.zls_control & (3 << 25) == (2 << 25); + let mut expected_control = 0; + for (plane, bytes_per_sample, transfers, compression) in [ + (depth, if z16 { 2u64 } else { 4u64 }, 0x88000u64, 0x44u64), + (stencil, 1u64, 0x44000u64, 0x110u64), + ] { + if plane.base == 0 { + if plane.compression_base != 0 || plane.stride != 0 + || plane.compression_stride != 0 { + return Err(UapiTranslateError::ZlsFields); + } + continue; + } + let compressed = plane.compression_base != 0; + // Compressed Z16 remains outside the captured layout. Both Z32 and + // S8 compression use 8-byte metadata records for 16x16 sample tiles. + if plane.base & 0x7f != 0 || plane.compression_base & 0x7f != 0 + || (!compressed && plane.compression_stride != 0) + || (compressed && (bytes_per_sample == 2 + || sample_width < 16 || sample_height < 16)) { + return Err(UapiTranslateError::ZlsFields); + } + expected_control |= command.zls_control & transfers; + if compressed { expected_control |= compression; } + // AIL tiles contain 16 KiB: 64x64 Z32, 128x64 Z16, 128x128 S8. + // A miptail smaller than either tile axis is power-of-two padded and + // cacheline aligned, rather than rounded to a whole 16-KiB page. + let (tile_width, tile_height) = match bytes_per_sample { + 4 => (64u32, 64u32), + 2 => (128, 64), + _ => (128, 128), + }; + let data_bytes = if sample_width >= tile_width && sample_height >= tile_height { + u64::from(sample_width.div_ceil(tile_width)) + * u64::from(sample_height.div_ceil(tile_height)) * 0x4000 + } else { + (u64::from(sample_width.next_power_of_two()) + * u64::from(sample_height.next_power_of_two()) * bytes_per_sample) + .div_ceil(128) * 128 + }; + require_range(address_space, plane.base, data_bytes, GpuAccess::ReadWrite)?; + if compressed { + let metadata_bytes = (u64::from(sample_width.next_power_of_two().div_ceil(16)) + * u64::from(sample_height.next_power_of_two().div_ceil(16)) * 8) + .div_ceil(128) * 128; + require_range(address_space, plane.compression_base, metadata_bytes, + GpuAccess::ReadWrite)?; + } + } + if depth.base != 0 { + if z16 { expected_control |= 2 << 25; } + if (command.depth_clear & 0x7fffffff) > 0x7f800000 { + return Err(UapiTranslateError::ZlsFields); + } + } + let dimensions = if depth.base != 0 || stencil.base != 0 { + (width - 1) | ((height - 1) << 16) + } else { 0 }; + // Exact matching rejects unknown format bits and transfer requests for + // an absent plane, as well as mismatched compression/base-address pairs. + if command.zls_control != expected_control || command.depth_dimensions != dimensions { + return Err(UapiTranslateError::ZlsFields); + } + Ok(()) +} + +pub(crate) fn translate_render_command( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, +) -> Result { + translate_render_command_with_capacity(command, vertex_attachments, + fragment_attachments, queue, address_space, internal, 32_768, false) +} + +fn translate_render_command_with_capacity( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, + tile_capacity: u64, + g18_subpage: bool, +) -> Result { + if command.flags & !RENDER_FLAGS != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + // Genuinely unimplemented on G17P rather than merely unvalidated: + // `G17pRenderParameters` carries no helper-program fields at all, so + // VERTEX_SCRATCH has nowhere to put the scratch binding, and there is no + // cluster-count control for NO_VERTEX_CLUSTERING. A clean EINVAL beats + // rendering with the request silently dropped. + if command.flags & (RENDER_VERTEX_SCRATCH | RENDER_NO_VERTEX_CLUSTERING) != 0 { + return Err(UapiTranslateError::FlagsUnsupported); + } + // RSRC_SPEC_HI is *declarative*, and honouring it must not mean rejecting. + // The UAPI defines it as "the appended resource-specifier high dwords are + // present", and states its purpose: "This flag lets old kernels report an + // unsupported render payload instead of silently truncating 64-bit G17 + // values." It exists for kernels that do NOT understand the dwords. + // + // This kernel does understand them -- `program()` ORs the high dword in + // unconditionally, with no reference to the flag -- so the flag cannot + // change what we do with a payload that sets it. The old check therefore + // refused payloads it would otherwise have handled identically, protecting + // nothing; it is the same bring-up leftover as the stencil-clear mask. + // + // Honour the documented meaning instead: when the flag is clear the high + // dwords are "not present", so mask them off rather than refuse the + // submission. A userspace built against the 240-byte upstream struct (whose + // payload `padded_payload` zero-fills) and one that clears the flag now + // behave identically, which is what "not present" has to mean. + let resource_spec_mask = if command.flags & RENDER_RSRC_SPEC_HI != 0 { + u64::MAX + } else { + u32::MAX as u64 + }; + if !(1..=16_384).contains(&command.width) || !(1..=16_384).contains(&command.height) { + return Err(UapiTranslateError::Dimensions); + } + if !(1..=2048).contains(&command.layers) { + return Err(UapiTranslateError::Layers); + } + if !matches!( + (command.utile_width, command.utile_height), + (32, 32) | (32, 16) | (16, 16) + ) { + return Err(UapiTranslateError::UtileDimensions); + } + let samples_log2 = match command.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err(UapiTranslateError::Samples), + }; + let utile_bytes = command.sample_size as u64 + * command.utile_width as u64 + * command.utile_height as u64 + * command.samples as u64; + if utile_bytes > tile_capacity { + return Err(UapiTranslateError::TilebufferSize); + } + let blocks_per_utile = utile_bytes.div_ceil(2048); + + if command.vdm_base == 0 || command.vdm_base & 3 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.vdm_base, 4, GpuAccess::Read)?; + + if command.scissor_base == 0 || command.scissor_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.scissor_base, 8, GpuAccess::Read)?; + if command.depth_bias_base != 0 { + if command.depth_bias_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.depth_bias_base, 8, GpuAccess::Read)?; + } + if command.occlusion_query_base != 0 { + if command.occlusion_query_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.occlusion_query_base, + 8, + GpuAccess::Write, + )?; + } + validate_zls(address_space, command.depth, command.layers, g18_subpage)?; + validate_zls(address_space, command.stencil, command.layers, g18_subpage)?; + validate_sampler( + address_space, + command.sampler_heap, + command.sampler_count as u32, + )?; + if command.vertex_helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + || command.fragment_helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + // `isp_bgobjvals` is the *whole* ISP_BGOBJVALS register value, not just + // the stencil clear: the UAPI documents "the bottom 8-bits contain the + // stencil buffer clear value", and every Mesa backend unconditionally + // seeds the upper bits with 0x300 before OR-ing the clear in + // (hk_cmd_draw.c `render->cr.isp_bgobjvals = 0x300;`, + // agx_pipe.c `c->isp_bgobjvals = 0x300;`, d12_queue.c `0x300 | stencil`). + // The legacy queue path passes the field straight through to the firmware + // (queue/render.rs `let load_bgobjvals = cmdbuf.isp_bgobjvals as u64`), so + // rejecting anything above 0xff refused every real render pass while the + // in-tree render selftest -- which leaves the field zero -- was accepted. + // Accept the register value; `build_*_descriptor` re-asserts the 0x300 + // bits with `stencil_clear_value | 0x300`, which is idempotent for the + // value userspace sends and still supplies 0x300 when it sends zero. + + let background = queue.full_program(address_space, command.background.usc)?; + let end_of_tile = queue.full_program(address_space, command.end_of_tile.usc)?; + let partial_background = queue.full_program(address_space, command.partial_background.usc)?; + let partial_end_of_tile = queue.full_program(address_space, command.partial_end_of_tile.usc)?; + validate_attachments(address_space, &vertex_attachments)?; + validate_attachments(address_space, &fragment_attachments)?; + + let utile_config = ((command.utile_width as u64 / 16) << 12) + | ((command.utile_height as u64 / 16) << 14) + | samples_log2; + let mut tile_config = 0x280; + if command.layers > 1 { + tile_config |= 1; + } + if command.flags & RENDER_PROCESS_EMPTY_TILES != 0 { + tile_config |= 0x1_0000; + } + + let mut parameters = internal.parameters; + parameters.width = command.width as u32; + parameters.height = command.height as u32; + parameters.encoder = command.vdm_base; + parameters.layers = command.layers; + parameters.utile_width = command.utile_width; + parameters.utile_height = command.utile_height; + parameters.utile_config = utile_config; + parameters.multisample_control = command.multisample_control; + parameters.ppp_control = command.ppp_control as u64; + parameters.tib_blocks = blocks_per_utile; + parameters.tile_config = tile_config; + parameters.depth_dimensions = command.depth_dimensions as u64; + parameters.depth_buffer = command.depth.base; + parameters.depth_aux_buffer = command.depth.compression_base; + parameters.depth_stride = command.depth.stride as u64; + parameters.depth_aux_stride = command.depth.compression_stride as u64; + parameters.stencil_buffer = command.stencil.base; + parameters.stencil_aux_buffer = command.stencil.compression_base; + parameters.stencil_stride = command.stencil.stride as u64; + parameters.stencil_aux_stride = command.stencil.compression_stride as u64; + parameters.depth_flags = command.zls_control; + // The firmware's internal partial-render phases preserve attached Z/S + // surfaces even when the render-pass boundary is store-only. Mesa already + // computes this distinction for its partial BG/EOT programs, but the + // current 256-byte render UAPI carries only the boundary ZLS word. Derive + // the exact uncompressed enable bits here until the append-only partial + // ZLS UAPI field is plumbed; tiling/compression/format bits remain those + // packed into the original word. + parameters.partial_depth_flags = command.zls_control + | if command.depth.base != 0 { (1 << 15) | (1 << 19) } else { 0 } + | if command.stencil.base != 0 { (1 << 14) | (1 << 18) } else { 0 }; + parameters.occlusion_query_base = command.occlusion_query_base; + parameters.scissor_array = command.scissor_base; + parameters.depth_bias_array = command.depth_bias_base; + parameters.sampler_array = command.sampler_heap; + parameters.sampler_count = command.sampler_count as u32; + parameters.process_empty_tiles = command.flags & RENDER_PROCESS_EMPTY_TILES != 0; + parameters.merge_upper_x_bits = command.merge_upper_x; + parameters.merge_upper_y_bits = command.merge_upper_y; + parameters.depth_clear_value_bits = command.depth_clear; + parameters.stencil_clear_value = command.stencil_clear; + parameters.load_pipeline_bind = g17p_bg_resource_spec( + command.background.resource_spec, + resource_spec_mask, + internal.parameters.bg_resource_prefix, + ); + parameters.load_pipeline = background; + parameters.store_pipeline_bind = command.end_of_tile.resource_spec & resource_spec_mask; + parameters.store_pipeline = end_of_tile; + // These high 17 bits are BG shader pass control, not an address class. + // Native derives main and partial masks from their respective shader + // outputs; they need not match. The default diagnostic prefix is zero, + // preserving both full RT0 output masks and no-output masks from userspace. + parameters.partial_load_pipeline_bind = g17p_bg_resource_spec( + command.partial_background.resource_spec, + resource_spec_mask, + internal.parameters.bg_resource_prefix, + ); + parameters.partial_load_pipeline = partial_background; + parameters.partial_store_pipeline_bind = + command.partial_end_of_tile.resource_spec & resource_spec_mask; + parameters.partial_store_pipeline = partial_end_of_tile; + + Ok(TranslatedRenderCommand { + render_order: submission::G17PRenderOrder::default(), + render_barrier: UAPI_BARRIER_NONE, + compute_barrier: UAPI_BARRIER_NONE, + compute_wait_through: None, + flags: command.flags, + vdm_base: command.vdm_base, + parameters, + vertex_attachments, + fragment_attachments, + vertex_timestamps: command.vertex_timestamps, + fragment_timestamps: command.fragment_timestamps, + }) +} + +fn g17p_bg_resource_spec(resource_spec: u64, present_mask: u64, diagnostic_prefix: u64) -> u64 { + diagnostic_prefix | (resource_spec & present_mask) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedComputeCommand { + /// Monotonic logical index, independent of the hardware timestamp. + pub(crate) compute_sequence: u64, + /// Explicit same-engine prefix; only host-published timestamps need host waits. + pub(crate) compute_wait_through: Option, + pub(crate) render_wait_through: Option, + pub(crate) render_wait_last: Option, + pub(crate) usc_exec_base: u64, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) control_stream_terminator: u64, + pub(crate) sampler_heap: u64, + pub(crate) sampler_count: u32, + pub(crate) attachments: UapiAttachmentList, + pub(crate) g17p_add3_buffers: Option<[u64; 3]>, + pub(crate) timestamps: UapiTimestamps, + pub(crate) scs_layout: u64, + /// G18 only: the command carried `DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE`. + /// [`translate_compute_command`] refuses that bit like any other unknown + /// one and leaves this false; the G18 adapter admits the bit, strips it + /// before translation and sets this afterwards. + pub(crate) g18_texture_state: bool, + /// Validated native descriptor inputs; resource/launch integration is + /// separate, so this field does not by itself authorize scratch use. + pub(crate) g18_resource_descriptor: Option<[u32; 23]>, +} + +pub(crate) fn translate_compute_command( + command: UapiComputeCommand, + attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, +) -> Result { + queue.validate()?; + if command.flags & !COMPUTE_G17P_ADD3_PROOF != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + if command.g18_resources != [0; 16] { + return Err(UapiTranslateError::FlagsUnsupported); + } + if command.control_stream_base & 3 != 0 + || command.control_stream_end & 3 != 0 + || command.control_stream_end <= command.control_stream_base + { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.control_stream_base, + command.control_stream_end - command.control_stream_base, + GpuAccess::Read, + )?; + validate_sampler(address_space, command.sampler_heap, command.sampler_count)?; + if command.helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + let g17p_add3_buffers = if command.flags & COMPUTE_G17P_ADD3_PROOF != 0 { + let entries = attachments.as_slice(); + if entries.len() != 3 + || entries + .iter() + .any(|entry| entry.size < G17P_ADD3_BUFFER_BYTES) + { + return Err(UapiTranslateError::Add3ProofAttachments); + } + require_range( + address_space, + entries[0].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Read, + )?; + require_range( + address_space, + entries[1].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Read, + )?; + require_range( + address_space, + entries[2].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Write, + )?; + Some([entries[0].address, entries[1].address, entries[2].address]) + } else { + validate_attachments(address_space, &attachments)?; + None + }; + Ok(TranslatedComputeCommand { + compute_sequence: 0, + compute_wait_through: None, + render_wait_through: None, + render_wait_last: None, + usc_exec_base: queue.base, + control_stream_base: command.control_stream_base, + control_stream_end: command.control_stream_end, + control_stream_terminator: command.control_stream_end - 4, + sampler_heap: command.sampler_heap, + sampler_count: command.sampler_count, + attachments, + g17p_add3_buffers, + timestamps: command.timestamps, + scs_layout: command.scs_layout, + g18_texture_state: false, + g18_resource_descriptor: None, + }) +} + +impl TranslatedComputeCommand { + +} diff --git a/drivers/gpu/drm/asahi/m5/completion.rs b/drivers/gpu/drm/asahi/m5/completion.rs new file mode 100644 index 0000000000000..b6b1e748df482 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/completion.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Completion ownership for the bounded M5 TA/fragment pair. +//! +//! 25G76 GFX 0x17544 compares scheduler.completed with *scheduler.submitted; +//! 0x1763c unlinks the completed scheduler. The submitted total is host-owned, +//! not a firmware-decremented pending count. 0x17a68/0x17ae0 then write the +//! first stamp pointer and 0x17b98 publishes the corresponding event mask. + +// The 32-bit completion value reserves its low byte. Stop before it wraps; +// storage-ring wrap is independent of this monotonically increasing stamp. +pub(crate) const MAX_GRAPHICS_GENERATION: u32 = 0x00ff_ffff; +pub(crate) const GRAPHICS_POINTER_COUNT: u32 = 0x500; + +pub(crate) const fn graphics_pointer_head(generation: u32) -> u32 { + (1 + generation * 3) % GRAPHICS_POINTER_COUNT +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct M5GraphicsCompletion { + pub(crate) submitted: u32, + pub(crate) completed: u32, + pub(crate) linked: u32, + pub(crate) list: [u64; 2], + pub(crate) stamps: [u32; 2], + pub(crate) signalled: u128, +} + +/// The same native scheduler retirement contract for one compute record. +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct M5ComputeCompletion { + pub(crate) submitted: u32, + pub(crate) completed: u32, + pub(crate) linked: u32, + pub(crate) list: [u64; 2], + pub(crate) stamp: u32, + pub(crate) signalled: u128, +} + +impl M5ComputeCompletion { + pub(crate) fn complete_at(self, list_address: u64, queue_id: u8, stamp: u32) -> bool { + list_address != 0 && queue_id < 128 && stamp != 0 && stamp & 0xff == 0 + && self.submitted == 1 && self.completed == 1 && self.linked == 0 + && self.list == [0, list_address] && self.stamp == stamp + && self.signalled & (1u128 << queue_id) != 0 + } +} + +impl M5GraphicsCompletion { + + pub(crate) fn pair_complete_at(self, list_address: u64, stamp: u32) -> bool { + list_address != 0 + && stamp != 0 && stamp & 0xff == 0 + && self.submitted == 2 + && self.completed == self.submitted + && self.linked == 0 + && self.list == [0, list_address] + && self.stamps == [stamp; 2] + && self.signalled & 0xc00 == 0xc00 + } +} + diff --git a/drivers/gpu/drm/asahi/m5/compute.rs b/drivers/gpu/drm/asahi/m5/compute.rs new file mode 100644 index 0000000000000..71e1384b070bc --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/compute.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8142/G17G C0 J813 compute register program for the 25G76 native profile. +//! +//! The native SKSM payload has selector 3 and its register array begins at +//! +0x40. The 40-write program below is separately captured on M5; it is not +//! the G17P program with a new chip ID. Queue registration, descriptor-owned +//! pointers, shader/CDM construction and completion still belong to the M5 +//! runtime. Encoding these registers does not admit a Linux GPU job. + +pub(crate) const M5_COMPUTE_REGISTER_COUNT: usize = 40; +pub(crate) const M5_COMPUTE_REGISTER_BYTES: usize = 12 * M5_COMPUTE_REGISTER_COUNT; +/// Measured descriptor prefix, not a claim about the host allocation size. +pub(crate) const M5_COMPUTE_DESCRIPTOR_BYTES: usize = 0x1000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5ComputeError { + AddressOverflow, + BufferTooSmall, + CdmTerminatorBeforeStream, +} + +/// Retained objects and per-command state for the measured simple CL path. +/// Samplers, user timestamps, indirect dispatch and extra spill modes are not +/// part of this profile. Pointer ownership belongs to the runtime caller. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ComputeDescriptor { + pub(crate) registers: M5ComputeParameters, + pub(crate) submit_sequence: u64, + /// Host descriptor +0x188, copied to native command +0x10. + pub(crate) scheduler_record: u64, + pub(crate) descriptor_low_alias: u64, + pub(crate) cdm_terminator: u64, + pub(crate) dispatch_stamps: [u64; 2], + pub(crate) event_machine_stamp: u32, + pub(crate) queue_id: u32, + pub(crate) queue_ordinal: u32, + pub(crate) has_event: bool, + pub(crate) submission_index: u32, + pub(crate) status: [u64; 2], + pub(crate) shared_control: u64, + pub(crate) shared_control_enabled: bool, + /// Sixteen bytes copied by the native host from the event machine's + /// +0x70 record. Their bit meanings are not all qualified; keep opaque. + pub(crate) event_metadata: [u8; 16], + pub(crate) zero_page: u64, + /// Host descriptor +0x148; not a fixed target constant. + pub(crate) owner_id: u8, + /// Full host descriptor +0x140. Register context packing uses only its + /// low byte; the descriptor copy at +0xf70 must retain all 32 bits. + pub(crate) descriptor_work_ordinal: u32, +} + +/// Encode all 4096 bytes of the native simple-compute descriptor prefix. +/// The register arrays and their aliases are generated from owned inputs. +/// This function performs no allocation, cache operation or publication. +pub(crate) fn encode_m5_compute_descriptor( + p: M5ComputeDescriptor, + raw: &mut [u8], +) -> Result<(), M5ComputeError> { + if raw.len() < M5_COMPUTE_DESCRIPTOR_BYTES { + return Err(M5ComputeError::BufferTooSmall); + } + if p.cdm_terminator < p.registers.cdm_base { + return Err(M5ComputeError::CdmTerminatorBeforeStream); + } + let registers = build_m5_compute_registers(p.registers)?; + let primary_alias = add(p.descriptor_low_alias, 0x40)?; + let secondary_alias = add(p.descriptor_low_alias, 0x760)?; + let secondary = [ + (0x10099u32, add(p.registers.operand_state_base, 5)?), + (0x10091, add(p.registers.operand_state_base, 0xf000)?), + (0x0a5c1, add(p.registers.operand_state_base, 0xf405)?), + (0x0a5c9, add(p.registers.operand_state_base, 0x1e400)?), + ]; + let raw = &mut raw[..M5_COMPUTE_DESCRIPTOR_BYTES]; + raw.fill(0); + for (offset, value) in [ + (0x00, 3u32), (0x0c, u32::from(p.registers.context_id)), + (0x748, 0x01e0_0028), (0xe68, 0x0030_0004), + (0xf20, (p.registers.dispatch_identity >> 32) as u32), + (0xf28, u32::MAX), (0xf50, p.event_machine_stamp), + (0xf54, p.queue_id), (0xf58, p.queue_ordinal), + (0xf5c, u32::from(p.has_event)), (0xf60, p.submission_index), + (0xf70, p.descriptor_work_ordinal), + ] { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [ + (4, p.submit_sequence), (0x10, p.scheduler_record), + (0x740, primary_alias), (0xe60, secondary_alias), + (0xed8, p.registers.preempt_base), (0xee0, p.cdm_terminator), + (0xf08, 0x0000_0001_5402_4201), + (0xf40, p.dispatch_stamps[0]), (0xf48, p.dispatch_stamps[1]), + (0xf68, p.registers.dispatch_identity & 0xffff_ffff), + (0xf7c, p.status[0]), (0xf84, p.status[1]), + (0xfb2, p.shared_control), (0xfcb, p.zero_page), + ] { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + for (index, count) in [37u16, 39, 39, 40].into_iter().enumerate() { + raw[0x18 + index * 2..0x1a + index * 2].copy_from_slice(&count.to_le_bytes()); + } + for (index, register) in registers.into_iter().enumerate() { + let offset = 0x40 + index * 12; + raw[offset..offset + 4].copy_from_slice(®ister.number.to_le_bytes()); + raw[offset + 4..offset + 12].copy_from_slice(®ister.value.to_le_bytes()); + } + for (index, (number, value)) in secondary.into_iter().enumerate() { + let offset = 0x760 + index * 12; + raw[offset..offset + 4].copy_from_slice(&number.to_le_bytes()); + raw[offset + 4..offset + 12].copy_from_slice(&value.to_le_bytes()); + } + raw[0xfba] = u8::from(p.shared_control_enabled); + raw[0xfbb..0xfcb].copy_from_slice(&p.event_metadata); + raw[0xfd3] = p.owner_id; + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ComputeRegister { + pub(crate) number: u32, + pub(crate) value: u64, +} + +impl M5ComputeRegister { + const fn new(number: u32, value: u64) -> Self { + Self { number, value } + } +} + +/// Command-owned addresses and identities; the runtime must retain their +/// backing and validate its mappings before publishing the complete command. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ComputeParameters { + pub(crate) preempt_base: u64, + pub(crate) cdm_base: u64, + pub(crate) dispatch_identity: u64, + pub(crate) context_id: u8, + pub(crate) work_ordinal: u8, + pub(crate) robustness: u64, + pub(crate) operand_state_base: u64, + /// Owned UMA pool slot, copied from native host descriptor +0x788 by + /// 25G76 generateRegisterList at 0x89e38dc..0x89e391c. This must match + /// shared-control +8 and the populated hardware UMA descriptor slot. + pub(crate) private_pool_hw_id: u64, + /// Native command descriptor +0x530, copied to register 0xd4a1. + pub(crate) descriptor_control_530: u64, + /// Descriptor +0x538; zero selects the native 0x60 default. The native + /// encoder shifts the selected value left 32 bits, discarding high bits. + pub(crate) descriptor_control_538: u64, + /// Native descriptor +0x60a/+0x60b bit 0. Their broader semantics are + /// still under investigation; neither is a fixed M5 device property. + pub(crate) descriptor_flag_60a: bool, + pub(crate) descriptor_flag_60b: bool, +} + +fn add(base: u64, offset: u64) -> Result { + base.checked_add(offset) + .ok_or(M5ComputeError::AddressOverflow) +} + +/// Encode the measured J813 profile, including its M5 control words and +/// enlarged operand-state offsets. Other G17G configurations are not implied. +pub(crate) fn build_m5_compute_registers( + p: M5ComputeParameters, +) -> Result<[M5ComputeRegister; M5_COMPUTE_REGISTER_COUNT], M5ComputeError> { + let context = ((p.context_id as u64) << 8) | p.work_ordinal as u64; + // 25G76 generateRegisterList +0x5d4..0x670. The final 0xa599 write + // restores 0x60 irrespective of the command-specific initial value. + let control_538 = if p.descriptor_control_538 == 0 { + 0x60 + } else { + p.descriptor_control_538 + }; + // 25G76 +0x644..0x66c; additional device-dependent bits used by other + // profiles are outside this measured J813 configuration. + let mode = if p.descriptor_flag_60a { + 0x14c0_8a60 + } else { + 0x10c0_8860 + } | if p.descriptor_flag_60b { 0x80 } else { 0 }; + Ok([ + M5ComputeRegister::new(0x1a510, p.preempt_base), + M5ComputeRegister::new(0x1a420, p.cdm_base), + M5ComputeRegister::new(0x1a4d0, add(p.preempt_base, 0x1480)?), + M5ComputeRegister::new(0x1a4d8, add(p.preempt_base, 0x1488)?), + M5ComputeRegister::new(0x1a4e0, add(p.preempt_base, 0x1490)?), + M5ComputeRegister::new(0x1a4e8, add(p.preempt_base, 0x1498)?), + M5ComputeRegister::new(0x1a440, 0x0000_0001_5402_4201), + M5ComputeRegister::new(0x0d4a1, p.descriptor_control_530), + M5ComputeRegister::new(0x0d411, 0x0000_0002_0000_0001), + M5ComputeRegister::new(0x0a599, (control_538 << 32) | 0x0040_0020), + M5ComputeRegister::new(0x1a458, mode), + M5ComputeRegister::new(0x101d9, 0x1c), + M5ComputeRegister::new(0x1a089, 0), + M5ComputeRegister::new(0x1a091, 0), + M5ComputeRegister::new(0x1a059, 0), + M5ComputeRegister::new(0x1a061, 0), + M5ComputeRegister::new(0x1a0b9, 0), + M5ComputeRegister::new(0x1a0c1, 0), + M5ComputeRegister::new(0x101d1, 0), + M5ComputeRegister::new(0x0d479, 0), + M5ComputeRegister::new(0x1a0e9, 8), + M5ComputeRegister::new(0x1a141, 0), + M5ComputeRegister::new(0x107a1, 0xff_0000), + M5ComputeRegister::new(0x1a540, p.dispatch_identity), + M5ComputeRegister::new(0x014a9, p.dispatch_identity), + M5ComputeRegister::new(0x0a351, p.dispatch_identity), + M5ComputeRegister::new(0x0a731, 0), + M5ComputeRegister::new(0x10201, context), + M5ComputeRegister::new(0x10428, context), + M5ComputeRegister::new(0x14028, p.private_pool_hw_id), + M5ComputeRegister::new(0x14070, p.robustness | 1), + M5ComputeRegister::new(0x10229, add(p.operand_state_base, 0x1e800)?), + M5ComputeRegister::new(0x140a8, add(p.operand_state_base, 0x1f000)?), + M5ComputeRegister::new(0x10099, add(p.operand_state_base, 0x0f405)?), + M5ComputeRegister::new(0x10091, add(p.operand_state_base, 0x1e400)?), + M5ComputeRegister::new(0x0a5c1, add(p.operand_state_base, 0x00005)?), + M5ComputeRegister::new(0x0a5c9, add(p.operand_state_base, 0x0f000)?), + M5ComputeRegister::new(0x0a731, 0), + M5ComputeRegister::new(0x1a440, 0x0000_0001_5402_4209), + M5ComputeRegister::new(0x0a599, 0x0000_0060_0040_0020), + ]) +} + +/// Write only the register array. No descriptor header or publication state +/// is manufactured here, and an invalid request leaves the destination alone. +pub(crate) fn encode_m5_compute_registers( + p: M5ComputeParameters, + bytes: &mut [u8], +) -> Result<(), M5ComputeError> { + if bytes.len() < M5_COMPUTE_REGISTER_BYTES { + return Err(M5ComputeError::BufferTooSmall); + } + let registers = build_m5_compute_registers(p)?; + for (index, register) in registers.iter().enumerate() { + let offset = index * 12; + bytes[offset..offset + 4].copy_from_slice(®ister.number.to_le_bytes()); + bytes[offset + 4..offset + 12].copy_from_slice(®ister.value.to_le_bytes()); + } + Ok(()) +} + diff --git a/drivers/gpu/drm/asahi/m5/drm.rs b/drivers/gpu/drm/asahi/m5/drm.rs new file mode 100644 index 0000000000000..7252236924f91 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/drm.rs @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Experimental M5 DRM adapter using the retained M5 firmware session. +//! First-job admission is deliberately separate from general Mesa support. + +use core::{any::Any, ops::Range}; +use kernel::{bindings, c_str, device, macros::vtable, new_mutex, prelude::*, static_lock_class, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + sync::{Arc, Mutex}, uapi, xarray}; +use crate::{alloc, driver::AsahiRegData, file, m5::live_boot::M5Runtime, + m5::resources::M5FirmwareMemory, m5::command, gem, gpu, hw, m5::vm_layout, mmu, queue}; + +pub(crate) type SharedM5Runtime = Arc>; +const USER_END: u64 = (1 << 42) - 2 * mmu::UAT_PGSZ as u64; + +/// Bytes of CDM control stream the witnessed M5 compute profile covers. The +/// captured stream (compute_program::STREAM) is twelve words: one dispatch +/// and the terminator at +44. Nothing longer has native evidence, and a +/// longer one is what a geometry-shader prepass submits. +const M5_COMPUTE_STREAM_BYTES: u64 = 48; +static QUEUE_NAME: &CStr = c_str!("m5_queue"); +static QUEUE_CLASS_KEY: Pin<&'static kernel::sync::LockClassKey> = static_lock_class!(); + +#[cfg(CONFIG_APPLE_CLPC)] +extern "C" { + fn apple_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) + -> core::ffi::c_int; + fn apple_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); + fn apple_clpc_gpu_put(gpu: *mut core::ffi::c_void); +} + +/// The CLPC's GPU input (`apple,clpc` on the GPU node). While the PMP runs, +/// its vote leaves the DRAM controller and the fabric at their lowest states; +/// the CLPC holds them at the top of its maps while M5 work is in flight, +/// and releases them its hold time after the last of it retires. Without a +/// CLPC the GPU runs as before, only slower under the PMP. +#[pin_data(PinnedDrop)] +pub(crate) struct M5ClpcInput { + gpu: *mut core::ffi::c_void, + /// Submissions in flight; the input is active while it is not zero. + #[pin] + in_flight: Mutex, +} + +// SAFETY: the CLPC serializes its input under its own lock, and the counter +// is only changed under `in_flight`. +unsafe impl Send for M5ClpcInput {} +// SAFETY: as above; shared use never changes the reference itself. +unsafe impl Sync for M5ClpcInput {} + +impl M5ClpcInput { + /// Takes the input before the firmware session starts, so that a CLPC + /// which has not probed yet defers the GPU without stranding a session. + pub(crate) fn new(dev: &device::Device) -> Result> { + #[allow(unused_mut)] + let mut gpu: *mut core::ffi::c_void = core::ptr::null_mut(); + #[cfg(CONFIG_APPLE_CLPC)] + { + // SAFETY: a live device and a writable result; C returns a + // reference, or NULL when the node names no CLPC. + let ret = unsafe { apple_clpc_gpu_get(dev.as_raw().cast(), &mut gpu) }; + if ret == EPROBE_DEFER.to_errno() { + return Err(EPROBE_DEFER); + } + if ret < 0 { + dev_warn!(dev, "no CLPC input ({}): GPU work gets no memory floor\n", ret); + gpu = core::ptr::null_mut(); + } + } + #[cfg(not(CONFIG_APPLE_CLPC))] + let _ = dev; + Arc::pin_init(pin_init!(Self { gpu, in_flight <- new_mutex!(0) }), GFP_KERNEL) + } + + fn set_active(&self, _active: bool) { + #[cfg(CONFIG_APPLE_CLPC)] + // SAFETY: NULL is accepted; a non-NULL reference is held until drop. + unsafe { apple_clpc_gpu_set_active(self.gpu, _active) }; + } + + /// Marks one submission in flight until the returned guard drops. + fn busy(&self) -> M5ClpcBusy<'_> { + let mut count = self.in_flight.lock(); + *count += 1; + if *count == 1 { + self.set_active(true); + } + M5ClpcBusy(self) + } +} + +#[pinned_drop] +impl PinnedDrop for M5ClpcInput { + fn drop(self: Pin<&mut Self>) { + #[cfg(CONFIG_APPLE_CLPC)] + // SAFETY: releases exactly the reference taken in new(), clearing the + // input first; NULL is accepted. + unsafe { apple_clpc_gpu_put(self.gpu) }; + } +} + +struct M5ClpcBusy<'a>(&'a M5ClpcInput); + +impl Drop for M5ClpcBusy<'_> { + fn drop(&mut self) { + let mut count = self.0.in_flight.lock(); + *count -= 1; + if *count == 0 { + self.0.set_active(false); + } + } +} + +pub(crate) struct M5DrmGpu { + // A strong reference prevents open DRM files from outliving the session. + runtime: SharedM5Runtime, + id: crate::regs::G17Id, + ids: gpu::SequenceIDs, + clpc: Arc, +} + +impl M5DrmGpu { + pub(crate) fn new(runtime: SharedM5Runtime, id: crate::regs::G17Id, + clpc: Arc) -> Self { + Self { runtime, id, ids: Default::default(), clpc } + } +} + +impl gpu::Gpu for M5DrmGpu { + fn stats_snapshot(&self) -> Option> { None } + fn as_any(&self) -> &dyn Any { self } + fn arc_as_any(self: Arc) -> Arc { self as Arc } + fn manager(&self) -> Option<&dyn gpu::GpuManager> { None } + fn init(&self) -> Result { + if self.is_crashed() { Err(ENODEV) } else { Ok(()) } + } + fn ids(&self) -> &gpu::SequenceIDs { &self.ids } + fn is_crashed(&self) -> bool { self.runtime.lock().is_crashed() } + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.gpu_generation = hw::GpuGen::G17 as u32; + params.gpu_variant = hw::GpuVariant::G as u32; + params.gpu_revision = self.id.gpu_rev as u32; + params.chip_id = 0x8142; + params.num_dies = self.id.num_dies; + params.num_clusters_total = self.id.num_clusters; + params.num_cores_per_cluster = self.id.num_cores; + for (i, mask) in self.id.core_masks.iter().enumerate() { + *params.core_masks.get_mut(i).ok_or(EIO)? = (*mask).into(); + } + // Dynamic M5 clock policy is not implemented. + params.max_frequency_khz = 0; + params.usc_generation = 3; + params.gpu_hal_generation = 300; + // One hardware command per submission. + params.max_commands_per_submission = 1; + Ok(()) + } + fn supports_vm_status(&self) -> bool { true } + /// VM_BIND defers unmaps while accepted work runs, and a submission installs its fence + /// in every output sync, also on failure. + fn tracked_submission(&self) -> bool { true } + fn user_range(&self) -> Range { mmu::UAT_PGSZ as u64..USER_END } + fn base_clock_hz(&self) -> u32 { 24_000_000 } + fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } + fn new_vm(&self, kernel_range: Range) -> Result { + self.runtime.lock().new_user_vm(self.ids.vm.next(), kernel_range) + } + fn new_queue(&self, vm: mmu::Vm, + _ualloc: Arc>, + _ualloc_priv: Arc>, + priority: u32, usc_exec_base: u64, + _usage: Arc) -> Result> { + if priority > 3 { return Err(EINVAL); } + let window = command::QueueUscWindow { + base: usc_exec_base, user_start: vm_layout::DRIVER_BASE, user_end: USER_END, + }; + window.validate().map_err(|_| EINVAL)?; + let queue = M5Queue { + runtime: self.runtime.clone(), vm, window, + fences: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, + clpc: self.clpc.clone(), + }; + Ok(KBox::new(queue, GFP_KERNEL)?) + } + fn map_timestamp_buffer(&self, _bo: gem::ObjectRef, _range: Range) + -> Result { Err(ENOTSUPP) } +} + +struct M5SubmissionFence; +#[vtable] +impl FenceOps for M5SubmissionFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("m5-submit") } +} + +/// Keep the original sync vector: creating a second output list would add an +/// allocation failure after userspace has already supplied its wait targets. +struct OutputPublication { + fence: UserFence, + syncs: KVec, + input_count: usize, + result: Result, +} + +impl Drop for OutputPublication { + fn drop(&mut self) { + if let Err(error) = self.result { self.fence.set_error(error); } + let fence = Fence::from_fence(&self.fence); + for mut sync in self.syncs.drain(self.input_count..) { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj.add_point(chain, &fence, sync.timeline_value); + } else { sync.syncobj.replace_fence(Some(&fence)); } + } + self.fence.signal(); + } +} + +struct UserAddressSpace<'a> { vm: &'a mmu::Vm } +impl command::GpuAddressSpace for UserAddressSpace<'_> { + fn covers(&self, address: u64, size: u64, access: command::GpuAccess) -> bool { + let (read, write) = match access { + command::GpuAccess::Read => (true, false), + command::GpuAccess::Write => (false, true), + command::GpuAccess::ReadWrite => (true, true), + }; + vm_layout::user_range_contains(&self.vm.kernel_range(), address, size, USER_END) + && self.vm.covers_range(address, size, read, write) + } +} + +struct M5Queue { + runtime: SharedM5Runtime, + vm: mmu::Vm, + window: command::QueueUscWindow, + fences: FenceContexts, + clpc: Arc, +} + +impl queue::Queue for M5Queue { + fn submit(&mut self, _reg_data: &AsahiRegData<'_>, _submission_id: u64, + syncs: KVec, + in_sync_count: usize, cmdbuf_raw: &[u8], + _objects: Pin<&xarray::XArray>>) -> Result { + if in_sync_count > syncs.len() { return Err(EINVAL); } + let fence = self.fences.new_fence(0, M5SubmissionFence)?.into(); + let mut publication = OutputPublication { + fence, syncs, input_count: in_sync_count, result: Err(ECANCELED), + }; + let result = (|| { + if self.vm.status().is_some_and(|status| status.get() != 0) { return Err(EIO); } + // Wait outside the runtime lock; another client may own the job + // whose fence satisfies this submission's dependency. + for sync in &publication.syncs[..in_sync_count] { + if let Some(fence) = &sync.fence { + // SAFETY: both calls borrow the strongly held input fence. + let waited = unsafe { bindings::dma_fence_wait_timeout( + fence.raw(), true, kernel::task::MAX_SCHEDULE_TIMEOUT) }; + if waited < 0 { return Err(Error::from_errno(waited as i32)); } + if waited == 0 { return Err(ETIMEDOUT); } + let status = unsafe { bindings::dma_fence_get_status(fence.raw()) }; + if status < 0 { return Err(Error::from_errno(status)); } + } + } + let _vm_job = self.vm.retain_job()?; + let address_space = UserAddressSpace { vm: &self.vm }; + let mut parser = command::UapiCommandParser::new(cmdbuf_raw); + let command = parser.next_hardware().map_err(|_| EINVAL)?.ok_or(EINVAL)?; + if parser.next_hardware().map_err(|_| EINVAL)?.is_some() { + pr_err!("M5 submit admission rejected: more than one hardware command\n"); + return Err(ENOTSUPP); + } + parser.finish().map_err(|_| EINVAL)?; + if let command::ParsedHardwareCommand::Compute { payload, attachments, .. } = command { + let timestamps = [payload.timestamps.start, payload.timestamps.end]; + if payload.flags != 0 || payload.sampler_heap != 0 || payload.sampler_count != 0 + || timestamps.iter().any(|t| t.handle != 0 || t.offset != 0) { + pr_err!("M5 compute admission rejected flags={:#x} sampler_heap={:#x} samplers={} timestamps={}\n", + payload.flags, payload.sampler_heap, payload.sampler_count, + timestamps.iter().any(|t| t.handle != 0 || t.offset != 0)); + return Err(ENOTSUPP); + } + // The witnessed M5 compute profile is one dispatch: the + // captured control stream is twelve words, a single dispatch + // followed by the terminator at +44, and the descriptor + // carries exactly one dispatch identity, one submission index + // and one pair of dispatch stamps. A longer stream -- every + // geometry-shader and transform-feedback prepass emits four or + // five dispatches -- is accepted by firmware, linked into the + // job list and never retired, so the submitting ioctl times + // out after forty seconds and quarantines the VM. Refuse it + // here instead: the client drops the batch and the session + // survives, exactly as it does for an out-of-profile render. + let stream_bytes = payload.control_stream_end + .saturating_sub(payload.control_stream_base); + if stream_bytes > M5_COMPUTE_STREAM_BYTES { + pr_err!("M5 compute admission rejected stream_bytes={} (profile admits one dispatch, {})\n", + stream_bytes, M5_COMPUTE_STREAM_BYTES); + return Err(ENOTSUPP); + } + let translated = command::translate_compute_command(payload, attachments, + self.window, &address_space).map_err(|_| EINVAL)?; + let _busy = self.clpc.busy(); + return self.runtime.lock().submit_application_compute(&self.vm, &translated); + } + let command::ParsedHardwareCommand::Render { + payload, vertex_attachments, fragment_attachments, .. + } = command else { + pr_err!("M5 submit admission rejected: neither a render nor a compute command\n"); + return Err(ENOTSUPP); + }; + admit_reference_render(&payload).map_err(|error| { + pr_err!("M5 render admission rejected size={}x{} layers={} utiles={}x{} samples={} sample_bytes={} positions={:#x}\n", + payload.width, payload.height, payload.layers, + payload.utile_width, payload.utile_height, payload.samples, + payload.sample_size, payload.multisample_control); + error + })?; + command::validate_m5_reference_depth(&payload, &address_space) + .map_err(|error| { + pr_err!("M5 depth admission rejected {:?}: size={}x{} dimensions={:#x} control={:#x} stride={:#x} meta_stride={:#x} base={:#x} meta={:#x} clear={:#x}\n", + error, payload.width, payload.height, payload.depth_dimensions, + payload.zls_control, payload.depth.stride, payload.depth.compression_stride, + payload.depth.base, payload.depth.compression_base, payload.depth_clear); + ENOTSUPP + })?; + let (parameters, mut controls) = M5FirmwareMemory::reference_render_profile(); + let mut translated = command::translate_render_command(payload, + vertex_attachments, fragment_attachments, self.window, &address_space, + command::RenderInternalState { parameters }).map_err(|_| EINVAL)?; + // Native single-layer M5 ZLS does not encode an array stride. + // The full data/metadata ranges were validated above. + translated.parameters.depth_stride = 0; + translated.parameters.depth_aux_stride = 0; + translated.parameters.stencil_stride = 0; + translated.parameters.stencil_aux_stride = 0; + if payload.samples > 1 { + // 25G76 2x/4x captures: 10051 carries samples << 16, + // d491 enables multisampling, and PTC gains eight entries. + // The UAPI already supplies sample positions and utile size. + translated.parameters.tib_blocks = u64::from(payload.samples) << 16; + controls.fragment.resource_flags |= 1 << 32; + controls.fragment.ptc_max = 0xd6; + } + // Native 25G76 indexed draws use 15021=0x10c000. The reference + // triangle's low bit causes fragment recovery with larger + // indexed overdraw, even after the tiling stage completes. + // Bit18 selects the narrow depth plane: the same controlled + // render against a Depth16Unorm attachment publishes 0x14c001 + // where the 32-bit one publishes 0x10c001, and that bit is the + // only difference in the whole register program beyond the ZLS + // format field and the integer clear value. + let z16 = payload.zls_control & (3 << 25) == (2 << 25); + translated.parameters.aux_fb_flags = + if z16 { 0x14c000 } else { 0x10c000 }; + // M5's second store-program pair refers to the same EOT program + // in the admitted native profile, with caller-owned program VAs. + controls.store_pipeline_2 = translated.parameters.store_pipeline; + controls.store_pipeline_2_bind = translated.parameters.store_pipeline_bind; + controls.partial_store_pipeline_2 = translated.parameters.partial_store_pipeline; + controls.partial_store_pipeline_2_bind = translated.parameters.partial_store_pipeline_bind; + let _busy = self.clpc.busy(); + self.runtime.lock().submit_application(&self.vm, translated.parameters, + controls, 1, [0x7d4000, 0x934000])?; + Ok(()) + })(); + // The runtime records and quarantines a failed published VM before + // this output fence is installed/signalled by the publication guard. + publication.result = result; + result + } +} + +/// Admit the captured M5 single-layer render-state envelope. This is an experimental first-job ABI +/// path, not an assertion that arbitrary Vulkan programs are supported. +fn admit_reference_render(command: &command::UapiRenderCommand) -> Result { + // A single 32-bit color target has an eight-byte aligned sample stride + // in Mesa's tilebuffer layout; sample_size includes that padding. + let sample_layout = match command.samples { + 1 => command.utile_width == 32 && command.utile_height == 32, + 2 => command.utile_width == 32 && command.utile_height == 32 + && command.sample_size == 8 && command.multisample_control == 0x44cc, + 4 => command.utile_width == 32 && command.utile_height == 16 + && command.sample_size == 8 && command.multisample_control == 0xeaa2_6e26, + _ => false, + }; + let timestamps = [command.vertex_timestamps.start, command.vertex_timestamps.end, + command.fragment_timestamps.start, command.fragment_timestamps.end]; + if !(1..=command::M5_MAX_RENDER_DIMENSION).contains(&u32::from(command.width)) + || !(1..=command::M5_MAX_RENDER_DIMENSION).contains(&u32::from(command.height)) + || command.layers != 1 + || !sample_layout + || command.sampler_heap != 0 || command.sampler_count != 0 + || command.occlusion_query_base != 0 + || timestamps.iter().any(|timestamp| timestamp.handle != 0 || timestamp.offset != 0) { + return Err(ENOTSUPP); + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/m5/fw_msg.rs b/drivers/gpu/drm/asahi/m5/fw_msg.rs new file mode 100644 index 0000000000000..368ed4cc36f34 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/fw_msg.rs @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +const INITDATA_TAG: u64 = 0x0081_0000_0000_0000; + +const INITDATA_ADDRESS_MASK: u64 = 0x0000_0fff_ffff_ffff; + +const INITDATA_ADDRESS_PREFIX: u64 = 0xffff_f000_0000_0000; + +/// `AGXArmFirmware::powerTransitionEnd(AGFIFirmwareRole, bool)` closes the GPU +/// power-on transition by sending this note to EACH firmware role. +/// +/// KEXT_PROVEN, 25G83 @0xfffffe00088936a4. Both send sites are identical: +/// +/// ldr x0, [x19, #6616] / [x19, #6672] ; the two role objects +/// ldr x8, [obj + 0x8a8] ; sendAsyncNoteToFirmware +/// mov x1, #0x89000000000000 ; tag 0x89, payload 0 +/// mov w2, #0x1 ; gated=true +/// blraa x8, x16 +/// +/// MEASURED on hardware (m1n1 hv capture 20260904-001852Z): it is the LAST +/// EP-0x21 message of bring-up, sent once per role right after the initdata +/// handshake completes and 11.4 seconds before the first work doorbell: +/// +/// 57.95ms role0 -> ep=0x21 0x0089000000000000 +/// 59.31ms role1 -> ep=0x21 0x0089000000000000 +/// ... +/// 11425.12ms role0 -> ep=0x21 0x0083000000000000 <- first doorbell +/// +/// This driver has never sent it. Its EP-0x21 vocabulary was 0x81 initdata, +/// 0x83 doorbell, 0x84 device control and 0x86 recovery inspection only. +const POWER_TRANSITION_END_TAG: u64 = 0x0089_0000_0000_0000; + +/// Encode the role's power-transition-end note. The payload is zero in every +/// captured instance and the kext materialises the bare tag with a single +/// `movz x1, #0x89, lsl #48`. +pub(crate) const fn encode_power_transition_end() -> u64 { + POWER_TRANSITION_END_TAG +} + +/// Refusals made by the evidence-backed codec. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum WireError { + AddressOutOfRange, +} + +/// Encode the G17 init-data handoff exactly as the recovered host does. +/// +/// The host instruction truncates to 44 bits. Linux instead rejects an +/// address unless it is already in low-44-bit or firmware-canonical form. +pub(crate) const fn encode_initdata_doorbell(firmware_va: u64) -> Result { + let high = firmware_va & !INITDATA_ADDRESS_MASK; + if high != 0 && high != INITDATA_ADDRESS_PREFIX { + return Err(WireError::AddressOutOfRange); + } + + Ok(INITDATA_TAG | (firmware_va & INITDATA_ADDRESS_MASK)) +} diff --git a/drivers/gpu/drm/asahi/m5/initdata.rs b/drivers/gpu/drm/asahi/m5/initdata.rs new file mode 100644 index 0000000000000..f2ad6afea017f --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/initdata.rs @@ -0,0 +1,1630 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +use core::result::Result; + +/// Root size for the primary (`gfx-asc`) instance. +pub(crate) const ROOT_SIZE_PRIMARY: usize = 0x0b8; + +/// Root size for the secondary (`gfx1-asc`) instance; the extra 0x10 bytes are +/// the two secondary-only pointers at `+0xb8`/`+0xc0`. +pub(crate) const ROOT_SIZE_SECONDARY: usize = 0x0c8; + +pub(crate) const MAIN_CONFIG_SIZE: usize = 0x600; + +pub(crate) const REGION_A_SIZE: usize = 0x4000; + +pub(crate) const REGION_C_SIZE: usize = 0x1000; + +pub(crate) const STATUS_BLOCK_SIZE: usize = 0x80; + +pub(crate) const HW_DATA_BUNDLE_MIN_ALLOC: usize = 0x10000; + +/// The native placement additionally hosts the twelve work rings and both main +/// configuration objects inside the same cluster, which then spans this much +/// (see [`WORK_RING_BUNDLE_OFFSETS`], [`NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET`]). +pub(crate) const NATIVE_SHARED_CLUSTER_SIZE: usize = 0x28000; + +/// The same tuple as the little-endian u64 the firmware compares in one go. +pub(crate) const INITDATA_VERSION_WORD: u64 = 0x0c8a_a322_0396_04c0; + +/// Target/build-specific root encoding. This is a serialization choice, not +/// permission to start firmware or reuse the other target's hardware tables. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RootProfile { + J700G17P, + /// J813 Mac17,3 / 25G76, RTKit-3255.160.4.release. Both roots were captured + /// before INBOX1 publication. Independently confirmed by the loaded GFX + /// version comparison at +0x1fa5c and GFX1 at +0x21140. + J813G17G25G76, + /// J873 Mac18,5 / T8152 / G18G / 26A428, RTKit_release-3514.0.15 + /// (armfw_g18g-b000 on GFX, armfw1_g18g-b010 on GFX1). No capture of a + /// native root exists; the encoding is source-constructed from the 26A428 + /// kext and firmware (see the "J873 / G18G" section below). Selecting it + /// changes nothing for the other two profiles. + J873G18G26A428, +} + +impl RootProfile { + pub(crate) const fn version_word(self) -> u64 { + match self { + Self::J700G17P => INITDATA_VERSION_WORD, + Self::J813G17G25G76 => 0x0c8a_a322_039e_04c0, + Self::J873G18G26A428 => G18G_26A428_INITDATA_VERSION_WORD, + } + } + + /// Bytes of the root object for `role`. J700/J813 keep the historical + /// 0xb8/0xc8 split. G18G sizes both roots to 0xc8 because both firmware + /// images copy 0xc8 bytes from each root (FW_PROVEN b000 0x21664-0x216a8). + pub(crate) const fn root_size(self, role: InstanceRole) -> usize { + match self { + Self::J873G18G26A428 => G18G_26A428_ROOT_SIZE, + Self::J700G17P | Self::J813G17G25G76 => root_size(role), + } + } + + /// Opcode of the one opening device-control record staged at main + /// `+0x4c0`. G18G renumbered the opening handler: 0x16 is now an + /// unrelated handler and 0x15 reaches the opening (FW_PROVEN jump table + /// 0xfffffc0000006c60, handler b000 0x3ed0). The secondary keeps 0x2a. + pub(crate) const fn control_opening_opcode(self, role: InstanceRole) -> u32 { + match (self, role) { + (Self::J873G18G26A428, InstanceRole::Primary) => { + G18G_26A428_CONTROL_OPENING_OPCODE_PRIMARY + } + (Self::J873G18G26A428, InstanceRole::Secondary) => { + G18G_26A428_CONTROL_OPENING_OPCODE_SECONDARY + } + (_, InstanceRole::Primary) => CONTROL_OPENING_OPCODE_PRIMARY, + (_, InstanceRole::Secondary) => CONTROL_OPENING_OPCODE_SECONDARY, + } + } + + /// Bundle offsets of the five views published at main `+0x254`. + pub(crate) const fn bundle_view_offsets(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => G18G_26A428_BUNDLE_VIEW_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_OFFSETS, + } + } + + /// Extents each view must fit inside the bundle allocation. + pub(crate) const fn bundle_view_extents(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => G18G_26A428_BUNDLE_VIEW_EXTENTS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_EXTENTS, + } + } + + /// Bundle offset named twice by main `+0x08`/`+0x10` (the empty BRN + /// table and the 0x88 block, which the allocator places together). + pub(crate) const fn repeated_region_bundle_offset(self) -> usize { + match self { + Self::J873G18G26A428 => G18G_26A428_SHARED_LAYOUT.block_88, + Self::J700G17P | Self::J813G17G25G76 => MAIN_REPEATED_REGION_BUNDLE_OFFSET, + } + } + +} + +const INITDATA_ADDR_MASK: u64 = 0x0000_0fff_ffff_ffff; + +const INITDATA_ADDR_PREFIX: u64 = 0xffff_f000_0000_0000; + +const ROOT_VERSION: usize = 0x00; + +const ROOT_REGION_A: usize = 0x08; + +const ROOT_MAIN_CONFIG: usize = 0x18; + +const ROOT_REGION_C: usize = 0x20; + +const ROOT_INSTANCE_KIND: usize = 0x28; + +const ROOT_CONSTANT_2C: usize = 0x2c; + +const ROOT_PAGE_SIZE: usize = 0x30; + +const ROOT_PAGE_BITS: usize = 0x32; + +const ROOT_LEVEL_COUNT: usize = 0x33; + +const ROOT_LEVELS: usize = 0x34; + +/// Root `+0xa8`: Status A pointer. (The M5-derived model misused this slot as +/// a role pointer.) +const ROOT_STATUS_A: usize = 0xa8; + +/// Root `+0xb0`: Status B pointer. Real for the primary, zero for the +/// secondary. (The M5-derived model misused it as a cross-role pointer.) +const ROOT_STATUS_B: usize = 0xb0; + +const ROOT_SECONDARY_EXTRA_0: usize = 0xb8; + +const ROOT_SECONDARY_EXTRA_1: usize = 0xc0; + +pub(crate) const UAT_PAGE_SIZE: u16 = 0x4000; + +pub(crate) const UAT_PAGE_BITS: u8 = 14; + +pub(crate) const UAT_LEVEL_COUNT: usize = 3; + +/// `(index_shift, entry_count)` for the three levels below the root, +/// confirmed against a live table walk. +pub(crate) const UAT_LEVELS: [(u8, u16); UAT_LEVEL_COUNT] = [(36, 64), (25, 2048), (14, 2048)]; + +pub(crate) const UAT_LEVEL_DESC_SIZE: usize = 0x20; + +/// Level descriptor `+0x10`: physical mask, identical in all three. +pub(crate) const UAT_LEVEL_PHYS_MASK: u64 = 0x0000_03ff_ffff_c000; + +const MAIN_HW_DATA: usize = 0x00; + +/// The same repeated-region address appears at `+0x08` and `+0x10`; it is the +/// bundle base plus [`MAIN_REPEATED_REGION_BUNDLE_OFFSET`], not the bundle +/// base itself. It reads zero at handoff (firmware writes into it), so the +/// host only has to provide it mapped. +const MAIN_REPEATED_0: usize = 0x08; + +const MAIN_REPEATED_1: usize = 0x10; + +pub(crate) const MAIN_REPEATED_REGION_BUNDLE_OFFSET: usize = 0xc500; + +const MAIN_CHANNEL_TABLE: usize = 0x20; + +/// Main `+0x254`: five unaligned u64 views into the hardware-data bundle. +const MAIN_BUNDLE_VIEWS: usize = 0x254; + +/// Main `+0x2d0`: six qwords of region views. +const MAIN_REGION_VIEWS: usize = 0x2d0; + +/// Main `+0x300`: u32 4, secondary only. +const MAIN_SECONDARY_KIND: usize = 0x300; + +const MAIN_SECONDARY_KIND_VALUE: u32 = 4; + +/// Main `+0x3e0`: u32 0xff, primary only. +const MAIN_PRIMARY_MASK: usize = 0x3e0; + +const MAIN_PRIMARY_MASK_VALUE: u32 = 0xff; + +const MAIN_PRIMARY_SCALAR_32C: usize = 0x32c; + +const MAIN_PRIMARY_SCALAR_344: usize = 0x344; + +/// Main `+0x471`: unaligned u64, secondary only, pointing *into* bundle view 2 +/// at [`SECONDARY_EXTRA_VIEW_OFFSET`] (not a fresh allocation — the M5-derived +/// model had this at `+0x481` backed by an invented 0x12688-byte buffer; both +/// were wrong). +const MAIN_SECONDARY_EXTRA_PTR: usize = 0x471; + +pub(crate) const SECONDARY_EXTRA_VIEW_INDEX: usize = 2; + +pub(crate) const SECONDARY_EXTRA_VIEW_OFFSET: usize = 0xe40; + +/// Main `+0x4c0` is both a scalar and a place: the leading u32 of the +/// device-control ring, which is embedded in the main configuration object at +/// exactly this offset on both instances. Writing the opening opcode here *is* +/// staging the opening device-control record (records are 0x40 bytes; fields +/// past the opcode are zero). The control channel's producer counter must then +/// be set to the role-specific value returned by +/// [`control_opening_producer`] before the descriptor is handed over. +const MAIN_CONTROL_RING: usize = 0x4c0; + +pub(crate) const CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x16; + +pub(crate) const CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; + +/// Bundle offsets of the five views named at main `+0x254`. They are views of +/// the common bundle, not independent objects: view 0 overlaps the tail of +/// the hardware-data object, and views 3 and 4 overlap each other by all but +/// 0x80 bytes. +pub(crate) const BUNDLE_VIEW_OFFSETS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; + +/// Logical extents of the five views. +pub(crate) const BUNDLE_VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; + +/// The context-0 mappings of the PB descriptor and UMA page-pool descriptor +/// tables published at main `+0x2d8` and `+0x2e8`, respectively. The matching +/// firmware-high mappings are published at `+0x2e0` and `+0x2f0`. +pub(crate) const REGION_VIEW_LOW_ADDRS: [u64; 2] = [0x70_0183_8000, 0x70_0184_0000]; + +pub(crate) const CHANNEL_TABLE_ENTRIES: usize = 17; + +pub(crate) const CHANNEL_ENTRY_SIZE: usize = 0x20; + +/// Channel entry layout: three *separate* 32-bit state-counter addresses at +/// `+0x00`/`+0x08`/`+0x10` (the counters themselves sit at 0x10 spacing in +/// their state block — do not collapse them into one state object), then the +/// ring base at `+0x18`. +pub(crate) const CHANNEL_ENTRY_STATE_COUNT: usize = 3; + +pub(crate) const CHANNEL_STATE_SPACING: usize = 0x10; + +/// Entries 0..11 are ordered TA_0, 3D_0, CL_0, TA_1, ... CL_3. +pub(crate) const WORK_CHANNEL_COUNT: usize = 12; + +/// Entry 12 is device control. It continues the compact control-state grid +/// ([`WORK_STATE_GRID_OFFSETS`] has 13 entries, one more than there are work +/// channels), and its ring is embedded at main `+0x4c0`. The M5-derived model +/// refused this index, claiming a "cursor pair at +0x1a0/+0x1a8"; that was a +/// misread — those offsets are exactly this entry's state-counter addresses 0 +/// and 1. +pub(crate) const CONTROL_CHANNEL_INDEX: usize = 12; + +/// Entries 13 and 14 are report channels with split state objects: their +/// state addresses alias off status A rather than the compact grid, and the +/// firmware peer of each host-owned counter is at `+0x20` within its object. +pub(crate) const REPORT_CHANNEL_INDICES: [usize; 2] = [13, 14]; + +/// Entry 15 is partial: only its first state address, no ring. Entry 16 is +/// empty. Filling them like the rest declares channels that do not exist. +pub(crate) const PARTIAL_CHANNEL_INDEX: usize = 15; + +/// Offsets of the channel state blocks (channels 0..=12) inside the compact +/// state grid. Native state is *not* in channel-table order: within each +/// workload group the work queues sit at descending offsets, and the control +/// channel's block follows at `+0x300`. Preserving these address identities is +/// part of the descriptor ABI. +pub(crate) const WORK_STATE_GRID_OFFSETS: [usize; 13] = [ + 0x0c0, 0x1c0, 0x2c0, // TA_0, 3D_0, CL_0 + 0x080, 0x180, 0x280, // TA_1, 3D_1, CL_1 + 0x040, 0x140, 0x240, // TA_2, 3D_2, CL_2 + 0x000, 0x100, 0x200, // TA_3, 3D_3, CL_3 + 0x300, // device control +]; + +/// The primary status-B object runs from there to status A: `0xeb00` bytes. +pub(crate) const PRIMARY_STATUS_B_OBJECT_SIZE: usize = 0xeb00; + +/// State-counter addresses of report channels 13/14, as offsets from status A. +pub(crate) const REPORT_STATE_STATUS_A_OFFSETS: [[usize; 3]; 2] = + [[0x00040, 0x002c0, 0x00080], [0x00240, 0xa6ac0, 0x00280]]; + +/// Ring bases of report channels 13/14, as offsets from status A. +pub(crate) const REPORT_RING_STATUS_A_OFFSETS: [usize; 2] = [0x04ac0, 0xafac0]; + +/// Entry 15's sole state address, as an offset from status A. +pub(crate) const PARTIAL_STATE_STATUS_A_OFFSET: usize = 0x2d2c0; + +/// Native bundle offsets of the twelve work rings (0x1800 each), in +/// channel-table order. These extend past [`HW_DATA_BUNDLE_MIN_ALLOC`]; a host +/// using this placement sizes the cluster [`NATIVE_SHARED_CLUSTER_SIZE`]. +pub(crate) const WORK_RING_BUNDLE_OFFSETS: [usize; 12] = [ + 0x10dc0, 0x16dc0, 0x1cdc0, 0x0f5c0, 0x155c0, 0x1b5c0, 0x0ddc0, 0x13dc0, 0x19dc0, 0x0c5c0, + 0x125c0, 0x185c0, +]; + +/// Native bundle offsets of the two main configuration objects. +pub(crate) const NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET: usize = 0x1e5c0; + +pub(crate) const NATIVE_SECONDARY_MAIN_BUNDLE_OFFSET: usize = 0x22a80; + +pub(crate) const HW_REGISTER_MAP_ENTRY_SIZE: usize = 0x28; + +pub(crate) const HW_REGISTER_MAP_SLOTS: usize = 53; + +/// Two GPU mappings of one present completion-queue backing. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmCompletionAliases { + pub(crate) low: u64, + pub(crate) high: u64, +} + +pub(crate) const M5_25G76_COMPLETION_QUEUE_COUNT: usize = 4; + +pub(crate) const M5_25G76_COMPLETION_BACKING_SIZE: usize = 0x800 * 0x40; + +/// M5 publishes all four descriptors at hardware-data +0x2614, with ordinals +/// 0,1,2,3. Each names low/high mappings of its own retained 128-KiB backing. +/// The two native pre-handoff captures agree on every descriptor byte. +/// Construction only: this does not allocate, map, publish, or poll a queue. +pub(crate) fn encode_m5_25g76_completion_table( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_COMPLETION_QUEUE_COUNT * 0x20 { + return Err(InitdataError::BufferSize); + } + let extent = M5_25G76_COMPLETION_BACKING_SIZE as u64; + // Validate every range before mutating the output. Different queues must + // not alias each other in either address space. + for (i, alias) in aliases.iter().enumerate() { + for address in [alias.low, alias.high] { + required_address(address)?; + let end = address.checked_add(extent).ok_or(InitdataError::AddressOutOfRange)?; + required_address(end - 1)?; + } + for prior in &aliases[..i] { + if (alias.low < prior.low + extent && prior.low < alias.low + extent) + || (alias.high < prior.high + extent && prior.high < alias.high + extent) + { + return Err(InitdataError::PairAliasedObject); + } + } + } + out.fill(0); + for (ordinal, alias) in aliases.iter().enumerate() { + let offset = ordinal * 0x20; + put_u64(out, offset, alias.low); + put_u64(out, offset + 8, alias.high); + put_u32(out, offset + 0x10, 0x800); + put_u32(out, offset + 0x14, 0x40); + put_u32(out, offset + 0x18, ordinal as u32); + } + Ok(()) +} + +/// Status block `+0x04` holds u32 1 pre-init. `+0x10`/`+0x14` are *post-ack +/// firmware state* and must be zero at handoff. +pub(crate) const STATUS_BLOCK_ONE_OFFSET: usize = 0x04; + +pub(crate) const PRIMARY_STATUS_B_FWCTL_STATE: usize = 0x48e0; + +pub(crate) const PRIMARY_STATUS_B_FWCTL_RING: usize = 0x48e8; + +pub(crate) const PRIMARY_STATUS_B_CONFIG_HEADER: usize = 0xe434; + +pub(crate) const PRIMARY_STATUS_B_CONFIG_OFFSET: usize = 0xe440; + +/// The secondary root sits exactly this far above the primary root; the +/// secondary's initialisation message carries the primary's address plus this +/// delta in the same field, so the pair is not placed freely. +pub(crate) const SECONDARY_ROOT_DELTA: u64 = 0x8000; + +/// Physical firmware instance encoded at root `+0x28`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum InstanceRole { + /// `gfx-asc`: scheduler, device control, and all work channels. + Primary = 0, + /// `gfx1-asc`: control-only peer. Linux starts it after primary RTKit is ready. + Secondary = 1, +} + +pub(crate) const fn root_size(role: InstanceRole) -> usize { + match role { + InstanceRole::Primary => ROOT_SIZE_PRIMARY, + InstanceRole::Secondary => ROOT_SIZE_SECONDARY, + } +} + +/// GPU virtual addresses of the objects both instances share. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedAddresses { + /// Shared 16 KiB region A (root `+0x08`). + pub(crate) region_a: u64, + /// Shared region C (root `+0x20`), 0x1000 bytes. + pub(crate) region_c: u64, + /// Base of the contiguous hardware-data bundle. The hardware-data object + /// sits at the base; the five main-config views, the repeated region, and + /// the secondary's `+0x471` pointer are all derived offsets into it. + pub(crate) hw_data_bundle: u64, + /// Size of that one allocation; must be at least + /// [`HW_DATA_BUNDLE_MIN_ALLOC`]. + pub(crate) hw_data_bundle_alloc: usize, +} + +/// GPU virtual addresses of one instance's private objects. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct InstanceAddresses { + /// This instance's root; its address rides the endpoint-0x20 doorbell. + pub(crate) root: u64, + pub(crate) main_config: u64, + pub(crate) status_a: u64, + /// Required for the primary; must be zero for the secondary (whose root + /// still has the field, written zero). + pub(crate) status_b: u64, + /// Root `+0xb8`/`+0xc0`. Required for the secondary; the primary root is + /// only 0xb8 bytes and cannot even hold them, so they must be zero there. + pub(crate) secondary_extras: [u64; 2], +} + +/// One channel-table entry: three state-counter addresses, then the ring base. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ChannelEntry { + pub(crate) states: [u64; 3], + pub(crate) ring: u64, +} + +impl ChannelEntry { + pub(crate) const EMPTY: ChannelEntry = ChannelEntry { + states: [0; 3], + ring: 0, + }; +} + +/// The primary's global region views at main `+0x2d0`: a globally retained, +/// initially zero HWDSID accounting table (256 pairs of u32), followed by +/// the high halves of two low/high alias pairs (the low halves +/// are the fixed context-0 addresses in +/// [`REGION_VIEW_LOW_ADDRS`]). Each pair must map one physical page from both +/// address spaces — an invariant of the UAT mapping layer that cannot be +/// checked from the addresses alone. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrimaryRegionViews { + pub(crate) sentinel_high: u64, + pub(crate) alias_high: [u64; 2], +} + +/// M5 25G76 descriptor fields +0x10 and +0x14 are independent. In slot 39 +/// native publishes 0x10000 and 0x8000 respectively. The older RegisterWindow +/// encoder duplicates one size and must not serialize this target's table. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RegisterWindow { + pub(crate) slot: usize, + pub(crate) phys: u64, + pub(crate) device_va: u64, + pub(crate) size_10: u32, + pub(crate) size_14: u32, + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +impl M5RegisterWindow { + /// Actual M5 PTE walks cover size_10 rounded to 16 KiB, including the + /// four pages of slot 39. size_14 is independently serialized firmware + /// metadata and must not truncate that mapping to two pages. + pub(crate) fn mapped_pages(&self) -> Result<(u64, u64, u64), InitdataError> { + let mask = UAT_PAGE_SIZE as u64 - 1; + if self.size_10 == 0 || self.phys & mask != self.device_va & mask { + return Err(InitdataError::RegisterWindow); + } + let physical = self.phys & !mask; + let virtual_address = self.device_va & !mask; + let end = self.device_va.checked_add(self.size_10 as u64) + .and_then(|value| value.checked_add(mask)) + .ok_or(InitdataError::AddressOutOfRange)? & !mask; + let span = end.checked_sub(virtual_address).ok_or(InitdataError::AddressOutOfRange)?; + physical.checked_add(span).ok_or(InitdataError::AddressOutOfRange)?; + Ok((physical, virtual_address, span)) + } +} + +/// Source-constructed register descriptors from both identical pre-handoff +/// J813/25G76 captures. These are firmware virtual mappings, not a request to +/// access or power the devices. No runtime currently consumes this profile. +pub(crate) const M5_25G76_REGISTER_WINDOWS: [M5RegisterWindow; 22] = { + const fn w(slot: usize, phys: u64, va: u64, size_10: u32, size_14: u32, + unk_18: u64, flag: u32) -> M5RegisterWindow { + M5RegisterWindow { slot, phys, device_va: 0xffff_fc21_8000_0000 + va, + size_10, size_14, unk_18, flag } + } + [ + w(0, 0x381014000, 0x000000, 0x04000, 0x04000, 0, 2), + w(3, 0x220104000, 0x008000, 0x18000, 0x18000, 0, 2), + w(8, 0x3a1350000, 0x028000, 0x04000, 0x04000, 0, 2), + w(9, 0x3803d0000, 0x030000, 0x04000, 0x04000, 0, 2), + w(10, 0x3803c0000, 0x038000, 0x04000, 0x04000, 0, 0), + w(12, 0x42165c000, 0x040000, 0x04000, 0x04000, 0, 2), + w(14, 0x380280000, 0x048000, 0x08000, 0x08000, 0, 0), + w(15, 0x38c840000, 0x058000, 0x24000, 0x24000, 0, 2), + w(17, 0x580000000, 0x080000, 0x21400, 0x21400, 0, 2), + w(22, 0x581000000, 0x0a8000, 0x08000, 0x08000, 0, 2), + w(24, 0x580ec0000, 0x0b8000, 0x10000, 0x10000, 0, 2), + w(26, 0x580d04000, 0x0d0000, 0x08000, 0x08000, 0xd04000, 2), + w(27, 0x580d0d000, 0x0e1000, 0x01000, 0x01000, 0xd0d000, 2), + w(28, 0x580d50000, 0x0e8000, 0x10000, 0x10000, 0xd50000, 2), + w(29, 0x580d10000, 0x100000, 0x04000, 0x04000, 0xd10000, 2), + w(31, 0x580d40000, 0x108000, 0x04000, 0x04000, 0xd40000, 2), + w(32, 0x580d60000, 0x110000, 0x08000, 0x08000, 0xd60000, 2), + w(35, 0x580e00000, 0x120000, 0x04000, 0x04000, 0xe00000, 2), + w(39, 0x580e08000, 0x128000, 0x10000, 0x08000, 0, 2), + w(41, 0x580e33800, 0x143800, 0x04000, 0x04000, 0, 2), + w(42, 0x580e35000, 0x151000, 0x04000, 0x04000, 0, 2), + w(50, 0x580014400, 0x160400, 0x00400, 0x00400, 0x14400, 0), + ] +}; + +pub(crate) fn encode_m5_25g76_register_map(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for window in M5_25G76_REGISTER_WINDOWS { + let base = window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, window.phys); + put_u64(out, base + 8, window.device_va); + put_u32(out, base + 0x10, window.size_10); + put_u32(out, base + 0x14, window.size_14); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + for slot in [2, 5, 6, 7, 30, 33, 34, 37, 38, 40, 43, 46, 47, 48, 49, 52] { + put_u32(out, slot * HW_REGISTER_MAP_ENTRY_SIZE + 0x20, 2); + } + Ok(()) +} + +pub(crate) const M5_25G76_HW_DATA_SIZE: usize = 0x2708; + +pub(crate) const M5_25G76_SHARED_PREFIX_SIZE: usize = 0xc500; + +pub(crate) const M5_25G76_AON_PTD_SIZE: usize = 0xe4; + +/// J813/25G76 AON PTD backing, captured twice before each role's handoff and +/// unchanged after a passing Metal workload. The three recorded words are +/// scoped to this platform's telemetry description; they are not generic +/// M5 constants. The allocation and GPU binding must remain caller-owned. +pub(crate) fn encode_m5_25g76_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_25G76_AON_PTD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, 0x6730_5452); + put_u32(out, 4, 0x001d_0001); + put_u32(out, 8, 0x0000_1401); + Ok(()) +} + +const M5_25G76_STATUS_CONFIG_WORDS: &[(usize, u32)] = &[ + (0x010, 0x0000001f), (0x014, 0x3e0a3d71), (0x018, 0x401ae148), + (0x01c, 0x00000028), (0x020, 0x00000001), (0x0c8, 0x00002af8), + (0x0cc, 0x00001f40), (0x0d8, 0x41980000), (0x0dc, 0x3d0f5c28), + (0x0f0, 0x000026ac), (0x0f4, 0x000000c8), (0x108, 0x42c80000), + (0x10c, 0x43c80000), (0x110, 0x000000c8), (0x128, 0x00000001), + (0x12c, 0x00005eb0), (0x130, 0x3e800000), (0x134, 0x3f99999a), + (0x158, 0x0000000e), (0x1c0, 0x00000001), (0x1c4, 0x00000001), + (0x1c8, 0x00000002), (0x1cc, 0x00000001), (0x1d0, 0x00000001), + (0x1d4, 0x00000001), (0x1d8, 0x00000001), (0x588, 0x00000001), + (0x58c, 0x0000000a), (0x590, 0x00000dac), (0x594, 0x000003e8), + (0x598, 0x00000bb8), (0x59c, 0x000002bc), (0x5cc, 0x00000001), + (0x5d0, 0x00000004), (0x5dc, 0x00001f40), (0x5e0, 0x000000c8), + (0x5e4, 0x00000fa0), (0x5e8, 0x0000012c), (0x5ec, 0x000007d0), + (0x5f0, 0x0000012c), (0x5f4, 0x000003e8), (0x5f8, 0x0000012c), + (0x600, 0x00000001), (0x604, 0x41700000), (0x608, 0x40a00000), + (0x60c, 0x00000020), (0x610, 0x000004b0), (0x620, 0x0000044c), +]; + +/// Build the primary status object before either firmware role is started. +/// Secondary root+0xb8 aliases status+0xe440. GFX1 copies that pointer to +/// firmware global 0x176248 at text+0x211d4 and later writes config+0x670, +/// +0x678 and +0x680 at text+0x1af94..0x1afc4. Those are firmware-owned +/// runtime state, not platform constants, and must start at zero even when +/// a primary pre-handoff capture (taken after secondary startup) has values. +pub(crate) fn encode_m5_25g76_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != PRIMARY_STATUS_B_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, PRIMARY_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, PRIMARY_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, PRIMARY_STATUS_B_CONFIG_HEADER, 1); + for &(offset, value) in M5_25G76_STATUS_CONFIG_WORDS { + put_u32(out, PRIMARY_STATUS_B_CONFIG_OFFSET + offset, value); + } + Ok(()) +} + +/// Shared private arena observed before primary initialization on J813. +/// The status and secondary-grid pointers name overlapping views of this +/// one backing. This extent is a measured mapped arena, not a claim about +/// the sizes of the native host's individual suballocations. +pub(crate) const M5_25G76_PRIVATE_CLUSTER_SIZE: usize = 0x178000; + +pub(crate) fn encode_m5_25g76_private_cluster( + fwctl_state: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_PRIVATE_CLUSTER_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + let fwctl_ring = fwctl_state.checked_add(0x40).ok_or(InitdataError::AddressOutOfRange)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_m5_25g76_primary_status_b(fwctl_state, fwctl_ring, &mut out[0x340..0xee40])?; + for offset in [0xee40, 0xc3100] { + encode_status_block(&mut out[offset..offset + STATUS_BLOCK_SIZE])?; + } + encode_secondary_root_extra_1_for_profile(RootProfile::J813G17G25G76, + &mut out[0x177400..0x177480])?; + // The main-config encoders carry one measured opening record (0x16 / + // 0x2a). Publish exactly one producer entry for each control channel. + // All work-channel and report counters remain zero. + put_u32(out, 0x320, 1); + put_u32(out, 0x1773e0, 1); + Ok(()) +} + +/// Caller-owned bindings in the power/performance policy allocation. +/// The host stores mach_absolute_time() at policy+0x651c. Supply the live +/// platform counter in its native timebase, never a captured timestamp. +#[derive(Clone, Copy)] +pub(crate) struct M5PolicyBindings { + pub state: u64, + pub aon_ptd: u64, + pub boot_timestamp: u64, +} + +/// Pre-handoff J813/25G76 scalars, identical across two cold boots after +/// removing owned pointers and the native boot timestamp. The repeated +/// 512-entry u16 table is encoded separately. Calibration fields whose +/// meanings remain unknown are scoped to this measured ten-core target. +const M5_25G76_POLICY_WORDS: &[(usize, u32)] = &[ + (0x3d98, 0xffffffff), (0x3db0, 0xffffffff), (0x5244, 0x00030d40), + (0x5248, 0x00030d40), (0x5250, 0x00000004), (0x5254, 0x3f800000), + (0x526c, 0x00000001), (0x5270, 0x00000001), (0x5280, 0x00000064), + (0x5284, 0x00000001), (0x5288, 0x000004b0), (0x5290, 0x00000064), + (0x52a0, 0x00000004), (0x52a4, 0x3f800000), (0x52ac, 0x00000001), + (0x52b0, 0x00000001), (0x52c0, 0x00000064), (0x52c4, 0x00000001), + (0x52c8, 0x000004b0), (0x52d0, 0x00000064), (0x5300, 0x3f828f5c), + (0x5304, 0x3f828f5c), (0x5308, 0x3f828f5c), (0x530c, 0x3f828f5c), + (0x5310, 0x3f828f5c), (0x5314, 0x3f828f5c), (0x5318, 0x3f828f5c), + (0x531c, 0x3f828f5c), (0x5320, 0x3f828f5c), (0x5324, 0x3f828f5c), + (0x5328, 0x3f828f5c), (0x532c, 0x3f828f5c), (0x5330, 0x3f828f5c), + (0x5bf8, 0x00000271), (0x5c04, 0x3f7f2e9f), (0x5c0c, 0x3b516154), + (0x5c14, 0x3ca59586), (0x5c20, 0x46bd6000), (0x5c24, 0x40a90fe4), + (0x5c2c, 0xbd4ac5b4), (0x5c30, 0x00000028), (0x5c34, 0x000004b0), + (0x5c38, 0x000004b0), (0x5c44, 0x00005eb0), (0x5c78, 0x000004b0), + (0x5cb0, 0x00000008), (0x5cbc, 0x3f000000), (0x5cc4, 0x3f000000), + (0x5ccc, 0x3f99999a), (0x5cd8, 0x47800000), (0x5cdc, 0x3e800000), + (0x5ce8, 0x00000028), (0x5cec, 0x000004b0), (0x5cfc, 0x00005eb0), + (0x5d08, 0x00000010), (0x5d10, 0x0005dc00), (0x5d68, 0x0000004a), + (0x5d70, 0x0000005a), (0x5d74, 0x00000032), (0x5d78, 0x00000006), + (0x5d80, 0x00000006), (0x5d84, 0x00000001), (0x5d8c, 0x3f4ccccd), + (0x5d90, 0x3f7eb852), (0x5d94, 0x3e4ccccd), (0x5d98, 0x3ba3d70a), + (0x5d9c, 0x3f144b3c), (0x5da0, 0x3f144b3c), (0x5da8, 0x42be0000), + (0x5dac, 0x4029eb85), (0x5db0, 0x4029eb85), (0x5db4, 0x4139435e), + (0x5db8, 0x00000064), (0x5dbc, 0x000004b0), (0x5dc0, 0x00000064), + (0x5dcc, 0x0000004a), (0x5dd4, 0x00000001), (0x5e00, 0x00000064), + (0x5e20, 0x00000401), (0x5e24, 0x00006400), (0x5e2c, 0x001f4000), + (0x5e30, 0x0000c800), (0x5e34, 0x000fa000), (0x5e38, 0x00012c00), + (0x5e3c, 0x0007d000), (0x5e40, 0x00012c00), (0x5e44, 0x0003e800), + (0x5e48, 0x00012c00), (0x5e4c, 0x00000100), (0x5ea4, 0x00000001), + (0x5ed0, 0x00000001), (0x74d8, 0x47800000), (0x74dc, 0x42200000), + (0x74e0, 0x44960000), (0x74e4, 0x0000062a), (0x74f8, 0x00000028), + (0x7500, 0x42c80000), (0x7504, 0x000004b0), (0x750c, 0x3f4ccccd), + (0x7510, 0x3e4ccccd), (0x7550, 0xdcdcfbfa), (0x7558, 0x00001f40), + (0x7560, 0x00000004), (0x7574, 0x3f800000), (0x757c, 0x3d0f5c28), + (0x7588, 0x47800000), (0x758c, 0x41980000), (0x7598, 0x00000028), + (0x759c, 0x000004b0), (0x75a0, 0x000004b0), (0x75ac, 0x00002af8), + (0x75e0, 0x000004b0), (0x77f8, 0x00000002), (0x77fc, 0x42000006), + (0x7a10, 0xb8520000), (0x7a14, 0x00003f7e), (0x7a18, 0xd70a0000), + (0x7a1c, 0x00003ba3), (0x7a24, 0x000043c8), (0x7a30, 0x00004780), + (0x7a34, 0x000042c8), (0x7a3c, 0x0384bac8), (0x7a40, 0x03e80000), + (0x7a44, 0x03e80000), (0x7a4c, 0x80000000), (0x7a50, 0x26ac453b), + (0x7a84, 0x03e80000), (0x7aa0, 0x00c80000), (0x7b58, 0x000004b0), + (0x7b6c, 0x00000001), (0x7ba0, 0x00000101), (0x7ba4, 0x00000200), + (0x7bbc, 0x00023f00), (0x7bc0, 0x00000100), (0x7bc4, 0x00000100), + (0x7bdc, 0x00024000), (0x7be0, 0x00000100), (0x7be4, 0x00000100), + (0x7bfc, 0x00024100), (0x7c48, 0x00000001), (0x7c50, 0x00005eb0), + (0x7c54, 0x00005eb0), (0x7c58, 0x00005eb0), (0x7c68, 0x3f800000), + (0x7c6c, 0x00000004), (0x7c70, 0x00000008), (0x7c74, 0x0002ee00), + (0x7d44, 0x0000007c), (0x7d50, 0x3f77bdef), (0x7d58, 0x3d042108), + (0x7d60, 0x3e0a3d71), (0x7d6c, 0x47800000), (0x7d70, 0x401ae148), + (0x7d7c, 0x00000028), (0x7d80, 0x000004b0), (0x7d84, 0x000004b0), + (0x7d8c, 0x46bd6000), (0x7d90, 0x00005eb0), (0x7d9c, 0x000000f8), + (0x7da4, 0x005ad200), (0x7dc4, 0x000004b0), (0x7e04, 0x00000001), + (0x7ed4, 0x00000032), (0x7ed8, 0x00000001), (0x7ee0, 0x3f638e39), + (0x7ee4, 0x3f2aaaab), (0x7ee8, 0x3de38e39), (0x7eec, 0x3eaaaaab), + (0x7ef0, 0xbecccccd), (0x7ef4, 0xbf4ccccd), (0x7efc, 0x47800000), + (0x7f00, 0xc0a00000), (0x7f04, 0xc0a00000), (0x7f0c, 0x00000064), + (0x7f10, 0x000004b0), (0x7f14, 0x00000064), (0x7f1c, 0x45fa0000), + (0x7f20, 0x00000578), (0x7f28, 0x0000000e), (0x7f2c, 0x00000048), + (0x7f30, 0x00000018), (0x7f34, 0x001a5e00), (0x7f3c, 0x0008ca00), + (0x7f48, 0x47800000), (0x7f70, 0x00001f40), (0x7f74, 0x00000004), + (0x7f78, 0x000009c4), (0x7f7c, 0x00000258), (0x7f80, 0x00000002), + (0x7f84, 0x00000004), (0x7f88, 0x00000032), (0x7fb8, 0x47800000), + (0x7fc8, 0x00000028), (0x7fcc, 0x000004b0), (0x8098, 0x00000001), + (0x80ac, 0x0000000a), (0x80b8, 0x00000001), (0x8174, 0xac000000), + (0x8178, 0xe800000d), (0x817c, 0xb8000003), (0x8180, 0xb000000b), + (0x8184, 0xb0000004), (0x8188, 0xbc000004), (0x818c, 0x20000002), + (0x8198, 0x00000001), (0x81b4, 0x00000004), (0x81c0, 0x3f780000), + (0x81c8, 0x3d000000), (0x81d0, 0x40a00000), (0x81dc, 0x47800000), + (0x81e0, 0x41700000), (0x81ec, 0x0000044c), (0x81f0, 0x000004b0), + (0x81f4, 0x000004b0), (0x8200, 0x000004b0), (0x8234, 0x000004b0), + (0x827c, 0x00000064), (0x8290, 0xdcdcfbfa), (0x8298, 0x0000007d), + (0x829c, 0x00000001), (0x8ee0, 0xdcdcfbfb), (0x8ee8, 0x00000001), + (0xaeec, 0x41c80000), (0xaef0, 0x41c80000), (0xb6a8, 0x67305452), + (0xb6e8, 0x00000001), (0xb728, 0x5243464c), (0xb72c, 0x4758464c), + (0xb738, 0x47584c50), (0xb748, 0x00000001), (0xb7a8, 0x00000001), + (0xb7b4, 0x42b73333), (0xb7f4, 0x449b8000), (0xb834, 0x42bc0000), + (0xb8e0, 0x0000186a), +]; + +/// Initialize the six allocations following M5 hardware-data. Preserve the +/// hardware block and subsequent channel resources. No firmware is started. +pub(crate) fn encode_m5_25g76_bundle_policy( + bindings: M5PolicyBindings, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() < M5_25G76_SHARED_PREFIX_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(bindings.state)?; + required_address(bindings.aon_ptd)?; + let state_end = bindings.state.checked_add(STATUS_BLOCK_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + let aon_end = bindings.aon_ptd.checked_add(M5_25G76_AON_PTD_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(state_end - 1)?; + required_address(aon_end - 1)?; + if bindings.state < aon_end && bindings.aon_ptd < state_end { + return Err(InitdataError::PairAliasedObject); + } + out[0x2740..M5_25G76_SHARED_PREFIX_SIZE].fill(0); + for &(offset, value) in M5_25G76_POLICY_WORDS { + put_u32(out, offset, value); + } + for index in 0..512 { + put_u16(out, 0x6ed7 + index * 2, 0x7fff); + } + put_u64(out, 0x8ed8, bindings.state); + put_u64(out, 0xb6a0, bindings.aon_ptd); + put_u64(out, 0xb75c, bindings.boot_timestamp); + Ok(()) +} + +/// Recorded platform words outside the explicit register, performance and +/// owned-pointer fields. Meanings not yet derived are deliberately kept as +/// target-scoped measured values. Never insert runtime completion counters +/// or captured allocation addresses here. Reference hardware-data SHA-256: +/// 6f9ac0c4664f898a0ddb4e5df9ad160a23c1b52d3fd750750779cbdebe446be9. +const M5_25G76_HW_PLATFORM_WORDS: &[(usize, u32)] = &[ + (0x0004, 0x0000006f), (0x0008, 0xffc00000), (0x0014, 0x00000010), + (0x001c, 0x00000010), (0x0020, 0xffff8000), (0x0024, 0x000002ff), + (0x0028, 0x81400000), (0x002c, 0xfffffc21), (0x00e0, 0x00002008), + (0x00e8, 0x20080000), (0x00f4, 0x00002008), (0x00f8, 0x000024cb), + (0x00fc, 0xc98a2cfa), (0x0100, 0xf4f624cb), (0x0104, 0x1877e917), + (0x0108, 0x38d924cb), (0x010c, 0xbdab0000), (0x02d8, 0xfff82000), + (0x02dc, 0xd32d2cdb), (0x02e0, 0xf5002000), (0x02e4, 0x21dae926), + (0x02e8, 0x38b62000), (0x02ec, 0xc7420008), (0x03e0, 0x00007fe0), + (0x03e8, 0x7fe00000), (0x03f4, 0x00007fe0), (0x03f8, 0x00008000), + (0x0400, 0x80000000), (0x040c, 0x00008000), (0x05d8, 0x4b232645), + (0x05dc, 0x00000e98), (0x05e0, 0xd5a2ea5f), (0x05e4, 0x40003fff), + (0x05e8, 0xca5e4000), (0x05ec, 0x4000f5a2), (0x0e90, 0x00008142), + (0x0e94, 0x00000002), (0x0ea0, 0x00000001), (0x0ea4, 0x00000001), + (0x0eb8, 0x00000001), (0x0ec0, 0x00000001), (0x0ec8, 0x00000001), + (0x0ed0, 0x00005dc0), (0x0ed4, 0x00000001), (0x0ed8, 0x00000008), + (0x0ee0, 0x00000001), (0x0ee4, 0x00000001), (0x0ee8, 0x00000001), + (0x0f04, 0x0000001f), (0x0f24, 0x07800004), (0x0f28, 0x00000006), + (0x0f34, 0x00000001), (0x0f38, 0x00000001), (0x0f4c, 0x00000031), + (0x0f6c, 0x00000001), (0x0f88, 0x0000000a), (0x0f8c, 0x00000001), + (0x0fac, 0x00000001), (0x0fb8, 0x00000020), (0x0fbc, 0x00000005), + (0x0fc0, 0x0000000a), (0x0fc4, 0x0000000c), (0x2544, 0x00000004), + (0x2548, 0x00000006), (0x254c, 0x00000003), (0x2550, 0x00000007), + (0x2554, 0x00000007), (0x2568, 0x0000000a), (0x2570, 0x00000001), + (0x258c, 0x00180000), (0x2594, 0x00000001), (0x2598, 0x00000001), + (0x259c, 0x00000001), (0x25a4, 0x00000001), (0x25a8, 0x00000001), + (0x25ac, 0x00000001), (0x25b4, 0xffffffff), (0x25b8, 0xffffffff), + (0x25bc, 0xffffffff), (0x25c0, 0xffffffff), (0x25c4, 0xffffffff), + (0x25c8, 0xffffffff), (0x25cc, 0xffffffff), (0x25d0, 0xffffffff), + (0x25d4, 0xffffffff), (0x25d8, 0xffffffff), (0x25dc, 0xffffffff), + (0x25e0, 0xffffffff), (0x25f4, 0x00000001), (0x2600, 0x00000001), + (0x2608, 0x00000001), (0x2610, 0x00000100), (0x26b4, 0x00000001), + (0x26b8, 0x00000001), (0x26e0, 0x000003e8), (0x26e8, 0x00000001), +]; + +/// Reconstruct the M5 reference hardware-data block with caller-owned queue +/// and QoS bindings. Performance values are measured on the ten-core J813 +/// reference, not derived from ADT and not a generic M5 power configuration. +/// Both pre-handoff firmware roles supplied identical reference bytes. +/// This encoder grants no runtime admission and performs no MMIO accesses. +pub(crate) fn encode_m5_25g76_hw_data( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + qos_high: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(qos_high)?; + let qos_end = qos_high.checked_add(0xe40).ok_or(InitdataError::AddressOutOfRange)?; + required_address(qos_end - 1)?; + let mut completion = [0; 0x80]; + encode_m5_25g76_completion_table(aliases, &mut completion)?; + for alias in aliases { + if qos_high < alias.high + M5_25G76_COMPLETION_BACKING_SIZE as u64 + && alias.high < qos_end { + return Err(InitdataError::PairAliasedObject); + } + } + + out.fill(0); + for &(offset, value) in M5_25G76_HW_PLATFORM_WORDS { + put_u32(out, offset, value); + } + encode_m5_25g76_register_map(&mut out[0x640..0xe88])?; + out[0x2614..0x2694].copy_from_slice(&completion); + put_u64(out, 0x26ac, qos_high); + + // Native includes index zero plus twelve operating points. Each voltage + // occupies all sixteen lanes of its 0x40-byte state row. + const FREQUENCY: [u32; 13] = [0, 338, 486, 636, 796, 888, 988, 1084, 1182, 1278, 1374, 1470, 1578]; + const CORE_VOLTAGE: [u32; 13] = [125, 615, 630, 660, 705, 735, 755, 800, 830, 865, 905, 945, 1010]; + const MEMORY_VOLTAGE: [u32; 13] = [770, 770, 770, 770, 770, 770, 770, 800, 830, 865, 905, 945, 1010]; + const RELATIVE_A: [u32; 13] = [0, 17, 23, 30, 42, 50, 58, 71, 82, 85, 89, 93, 100]; + const RELATIVE_B: [u32; 13] = [0, 0, 11, 24, 36, 44, 52, 60, 68, 75, 83, 91, 100]; + for index in 0..13 { + put_u32(out, 0xfc8 + index * 4, FREQUENCY[index]); + put_u32(out, 0x1848 + index * 4, 0x3f828f5c); + put_u32(out, 0x18c8 + index * 4, RELATIVE_A[index]); + put_u32(out, 0x1908 + index * 4, RELATIVE_B[index]); + for lane in 0..16 { + put_u32(out, 0x1008 + index * 0x40 + lane * 4, CORE_VOLTAGE[index]); + put_u32(out, 0x1408 + index * 0x40 + lane * 4, MEMORY_VOLTAGE[index]); + } + } + for index in 0..16 { + put_u32(out, 0x19c8 + index * 4, index as u32); + } + Ok(()) +} + +/// Fail-closed validation errors for the grounded layout. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum InitdataError { + BufferSize, + AddressOutOfRange, + NullRequiredAddress, + RoleSpecificAddress, + BundleAllocationTooSmall, + ChannelTableShape, + RegisterWindow, + PairAliasedObject, +} + +fn address_representable(address: u64) -> bool { + let high = address & !INITDATA_ADDR_MASK; + high == 0 || high == INITDATA_ADDR_PREFIX +} + +fn required_address(address: u64) -> Result<(), InitdataError> { + if address == 0 { + Err(InitdataError::NullRequiredAddress) + } else if !address_representable(address) { + Err(InitdataError::AddressOutOfRange) + } else { + Ok(()) + } +} + +fn put_u16(bytes: &mut [u8], offset: usize, value: u16) { + bytes[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn write_runs(bytes: &mut [u8], runs: &[(usize, &[u8])]) { + for (offset, data) in runs { + bytes[*offset..*offset + data.len()].copy_from_slice(data); + } +} + +fn validate_shared(shared: &SharedAddresses) -> Result<(), InitdataError> { + required_address(shared.region_a)?; + required_address(shared.region_c)?; + required_address(shared.hw_data_bundle)?; + if shared.hw_data_bundle_alloc < HW_DATA_BUNDLE_MIN_ALLOC { + return Err(InitdataError::BundleAllocationTooSmall); + } + Ok(()) +} + +fn validate_instance(role: InstanceRole, inst: &InstanceAddresses) -> Result<(), InitdataError> { + required_address(inst.root)?; + required_address(inst.main_config)?; + required_address(inst.status_a)?; + match role { + InstanceRole::Primary => { + // The primary root has no +0xb8/+0xc0 fields at all; carrying + // secondary extras there is a role-confusion bug at the caller. + if inst.secondary_extras != [0, 0] { + return Err(InitdataError::RoleSpecificAddress); + } + required_address(inst.status_b)?; + } + InstanceRole::Secondary => { + // Every hardware-booting host hands the secondary a zero status-B; + // a nonzero value there is ungrounded, so fail closed on it. + if inst.status_b != 0 { + return Err(InitdataError::RoleSpecificAddress); + } + for extra in inst.secondary_extras { + required_address(extra)?; + } + } + } + Ok(()) +} + +/// Encode one 0x20-byte UAT level descriptor. Every field is derivable from +/// the level's index shift and entry count; the index mask +/// `(entries - 1) << shift` reproduces all three captured masks exactly. +fn encode_uat_level_descriptor(bytes: &mut [u8], offset: usize, shift: u8, entries: u16) { + bytes[offset] = 8; + bytes[offset + 1] = UAT_PAGE_BITS; + bytes[offset + 2] = UAT_PAGE_BITS; + bytes[offset + 3] = shift; + put_u16(bytes, offset + 0x04, entries); + put_u16(bytes, offset + 0x06, UAT_PAGE_SIZE); + put_u64(bytes, offset + 0x08, 1); + put_u64(bytes, offset + 0x10, UAT_LEVEL_PHYS_MASK); + put_u64(bytes, offset + 0x18, (entries as u64 - 1) << shift); +} + +/// Encode the common root layout with an explicitly selected firmware build. +/// The existing runtime continues to call the J700-only wrapper above. +pub(crate) fn encode_root_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + out: &mut [u8], +) -> Result<(), InitdataError> { + // Profile-sized: identical to root_size(role) for J700/J813. + if out.len() != profile.root_size(role) { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + + out.fill(0); + put_u64(out, ROOT_VERSION, profile.version_word()); + put_u64(out, ROOT_REGION_A, shared.region_a); + // ROOT_ZERO_10 deliberately left zero. + put_u64(out, ROOT_MAIN_CONFIG, inst.main_config); + put_u64(out, ROOT_REGION_C, shared.region_c); + put_u32(out, ROOT_INSTANCE_KIND, role as u32); + put_u32(out, ROOT_CONSTANT_2C, 1); + put_u16(out, ROOT_PAGE_SIZE, UAT_PAGE_SIZE); + out[ROOT_PAGE_BITS] = UAT_PAGE_BITS; + out[ROOT_LEVEL_COUNT] = UAT_LEVEL_COUNT as u8; + for (index, (shift, entries)) in UAT_LEVELS.iter().enumerate() { + encode_uat_level_descriptor( + out, + ROOT_LEVELS + index * UAT_LEVEL_DESC_SIZE, + *shift, + *entries, + ); + } + put_u64(out, ROOT_STATUS_A, inst.status_a); + put_u64(out, ROOT_STATUS_B, inst.status_b); + if role == InstanceRole::Secondary { + put_u64(out, ROOT_SECONDARY_EXTRA_0, inst.secondary_extras[0]); + put_u64(out, ROOT_SECONDARY_EXTRA_1, inst.secondary_extras[1]); + } + Ok(()) +} + +/// Derive the native channel table for one instance. +/// +/// `state_grid` is this instance's compact channel-state block (0x40-stride, +/// permuted per [`WORK_STATE_GRID_OFFSETS`]); `status_a` anchors the trailing +/// report/partial channels; the ring for entry 12 is `main_config + 0x4c0`; +/// work rings for the primary sit in the bundle at +/// [`WORK_RING_BUNDLE_OFFSETS`] (which requires the +/// [`NATIVE_SHARED_CLUSTER_SIZE`] cluster). The secondary leaves entries +/// 0..11 empty: it is a control-only peer with no work channels. +pub(crate) fn derive_channel_table( + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = [ChannelEntry::EMPTY; CHANNEL_TABLE_ENTRIES]; + let grid_states = |index: usize| { + let base = state_grid + WORK_STATE_GRID_OFFSETS[index] as u64; + [ + base, + base + CHANNEL_STATE_SPACING as u64, + base + 2 * CHANNEL_STATE_SPACING as u64, + ] + }; + if role == InstanceRole::Primary { + for index in 0..WORK_CHANNEL_COUNT { + table[index] = ChannelEntry { + states: grid_states(index), + ring: bundle + WORK_RING_BUNDLE_OFFSETS[index] as u64, + }; + } + } + table[CONTROL_CHANNEL_INDEX] = ChannelEntry { + states: grid_states(CONTROL_CHANNEL_INDEX), + ring: main_config + MAIN_CONTROL_RING as u64, + }; + for (slot, index) in REPORT_CHANNEL_INDICES.iter().enumerate() { + let mut states = [0u64; 3]; + for (word, offset) in states.iter_mut().zip(REPORT_STATE_STATUS_A_OFFSETS[slot]) { + *word = status_a + offset as u64; + } + table[*index] = ChannelEntry { + states, + ring: status_a + REPORT_RING_STATUS_A_OFFSETS[slot] as u64, + }; + } + table[PARTIAL_CHANNEL_INDEX] = ChannelEntry { + states: [status_a + PARTIAL_STATE_STATUS_A_OFFSET as u64, 0, 0], + ring: 0, + }; + table +} + +fn validate_populated_entry(entry: &ChannelEntry) -> Result<(), InitdataError> { + // A populated channel needs all three state-counter addresses — a null + // producer (state 2) on a populated entry makes firmware fault on address + // zero — and a real ring. + for state in entry.states { + if state == 0 { + return Err(InitdataError::ChannelTableShape); + } + if !address_representable(state) { + return Err(InitdataError::AddressOutOfRange); + } + } + required_address(entry.ring) +} + +fn validate_channel_table( + role: InstanceRole, + main_config: u64, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], +) -> Result<(), InitdataError> { + for (index, entry) in channels.iter().enumerate() { + match index { + 0..=11 => match role { + InstanceRole::Primary => validate_populated_entry(entry)?, + // The secondary has no work channels at all. + InstanceRole::Secondary => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + }, + 12 => { + validate_populated_entry(entry)?; + // Device control is embedded in the main object, at exactly + // main + 0x4c0 on both instances. + if entry.ring != main_config + MAIN_CONTROL_RING as u64 { + return Err(InitdataError::ChannelTableShape); + } + } + 13 | 14 => validate_populated_entry(entry)?, + 15 => { + // Partial: a first state address and nothing else. + required_address(entry.states[0])?; + if entry.states[1] != 0 || entry.states[2] != 0 || entry.ring != 0 { + return Err(InitdataError::ChannelTableShape); + } + } + _ => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + } + } + Ok(()) +} + +/// The two complete M5/25G76 pre-handoff main configurations match the common +/// layout except primary +0x32c/+0x344 and secondary +0x300. Keep these +/// measured scalars build-specific; their meanings are not inferred here. +pub(crate) fn encode_main_config_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], + region_views: Option<&PrimaryRegionViews>, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != MAIN_CONFIG_SIZE { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + validate_channel_table(role, inst.main_config, channels)?; + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + required_address(views.sentinel_high)?; + for high in views.alias_high { + required_address(high)?; + } + } + (InstanceRole::Secondary, None) => {} + _ => return Err(InitdataError::RoleSpecificAddress), + } + // Every view must fit inside the one bundle allocation. (The last two end + // exactly at 0xf000, inside the 0x10000 minimum.) The view table is the + // profile's: J700/J813 keep BUNDLE_VIEW_OFFSETS/EXTENTS unchanged. + let view_offsets = profile.bundle_view_offsets(); + for (offset, extent) in view_offsets.iter().zip(profile.bundle_view_extents()) { + if offset + extent > shared.hw_data_bundle_alloc { + return Err(InitdataError::BundleAllocationTooSmall); + } + } + + out.fill(0); + // Firmware log enable. The emitter at A000 0xfffffc0000021fa0 reads this + // word and returns immediately when it is zero; every buffer the facility + // needs is already allocated and pointed at by the addresses below. The + // records are plain ASCII. Shipping macOS provisions the whole facility + // and then throws the messages away (drainFirmwareLogRing is `bti c; ret`), + // which is why the captured value is zero. It stays zero here. + let bundle = shared.hw_data_bundle; + put_u64(out, MAIN_HW_DATA, bundle); + let repeated = bundle + profile.repeated_region_bundle_offset() as u64; + put_u64(out, MAIN_REPEATED_0, repeated); + put_u64(out, MAIN_REPEATED_1, repeated); + + for (index, entry) in channels.iter().enumerate() { + let base = MAIN_CHANNEL_TABLE + index * CHANNEL_ENTRY_SIZE; + for (word, state) in entry.states.iter().enumerate() { + put_u64(out, base + word * 8, *state); + } + put_u64(out, base + CHANNEL_ENTRY_STATE_COUNT * 8, entry.ring); + } + + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + // The five bare pointers are views into the one bundle + // allocation, at fixed offsets. Unaligned stores at +0x254. + for (index, offset) in view_offsets.iter().enumerate() { + put_u64(out, MAIN_BUNDLE_VIEWS + index * 8, bundle + *offset as u64); + } + // High-only sentinel followed by two low/high region-view pairs. + put_u64(out, MAIN_REGION_VIEWS, views.sentinel_high); + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x10, views.alias_high[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u64(out, MAIN_REGION_VIEWS + 0x20, views.alias_high[1]); + if profile == RootProfile::J700G17P { + put_u32(out, MAIN_PRIMARY_SCALAR_32C, 1); + put_u32(out, MAIN_PRIMARY_SCALAR_344, 0xabcd_abcd); + } + // KEXT_PROVEN for G18G too (0xfffffe0008c31780: main+0x3e0=0xff). + put_u32(out, MAIN_PRIMARY_MASK, MAIN_PRIMARY_MASK_VALUE); + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + (InstanceRole::Secondary, _) => { + // The secondary keeps the two low context-0 values but carries no + // view or region addresses; the values are therefore not + // properties of any region the host allocates. + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u32(out, MAIN_SECONDARY_KIND, match profile { + RootProfile::J700G17P => MAIN_SECONDARY_KIND_VALUE, + RootProfile::J813G17G25G76 => 0xc, + // INFERRED: 26A428 stores accel+0x10a74 here (25G76: + // accel+0xed30, measured 0xc on J813). Value not read on J873. + RootProfile::J873G18G26A428 => G18G_26A428_MAIN_SECONDARY_KIND, + }); + if profile == RootProfile::J873G18G26A428 { + // KEXT_PROVEN (26A428 initFirmwareSharedData, g18g_key.c:4464): + // role 1 main+0x481 = FW VA of the perf/policy allocation. + // 25G76 stored it at +0x471 and also wrote an accelerator + // pair at +0x479/+0x481, which 26A428 no longer writes. + put_u64(out, G18G_26A428_MAIN_SECONDARY_POLICY_PTR, + bundle + G18G_26A428_SHARED_LAYOUT.policy as u64); + } else { + // One additional unaligned pointer *into* primary view 2, landing + // five bytes before that view's first populated run. + let extra = bundle + + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] as u64 + + SECONDARY_EXTRA_VIEW_OFFSET as u64; + put_u64(out, MAIN_SECONDARY_EXTRA_PTR, extra); + } + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + _ => unreachable!(), + } + Ok(()) +} + +/// Encode target-specific pre-handoff platform policy. M5's native policy is +/// kept separate from the experimental T8140 watchdog and idle overrides. +pub(crate) fn encode_region_c_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + if profile == RootProfile::J873G18G26A428 { + // Derived from the J813 bytes below through the proven +4 shift. + return encode_g18g_26a428_region_c(out); + } + out.fill(0); + write_runs(out, REGION_C_RUNS); + if profile == RootProfile::J813G17G25G76 { + // Scalars measured before each ASC received its root on two cold + // J813 / 25G76 boots. These are a scoped reference configuration, + // not generic M5 calibration or a power-management qualification. + // The final pair moved by 0x20; leaving the T8140 pair in place would + // initialize unrelated M5 fields. + for (offset, value) in [ + (0x084, 0x0000_5eb0), + (0x088, 0x0000_04b0), + (0x08c, 0x0000_04b0), + (0x098, 0x0000_04b0), + (0x7c4, 0x0000_04e2), + (0x7e0, 0x3f33_3333), + (0x9a8, 0x0000_0333), + (0xe38, 0), + (0xe3c, 0), + (0xe58, 0x0000_0300), + (0xe5c, 0x0000_0100), + ] { + put_u32(out, offset, value); + } + return Ok(()); + } + // regionC+0x9cc is the KSM progress-watchdog threshold for data master 2 + // (compute); the firmware copies it to 0x1778d0 and its handler starts + // `cbz w20 -> ret`, so 0 disables the watchdog. We ship 0x32 (50) here and + // leave the other data master's slot at 0 -- i.e. we arm the watchdog on + // exactly the queue we submit to. When work cannot progress for any other + // reason this turns a stall into a self-sustaining recovery loop. + for (offset, value) in [ + (0x7c, 1), + (0x80, 1), + (0x9c, 0x3000_001d), + (0xa0, 0x3100_0002), + ] { + put_u32(out, offset, value); + } + Ok(()) +} + +/// Encode one 0x80-byte status block in its pre-init state: u32 1 at `+0x04`, +/// everything else zero. `+0x10`/`+0x14` are post-acknowledgement firmware +/// state and must be zero at handoff. +pub(crate) fn encode_status_block(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, STATUS_BLOCK_ONE_OFFSET, 1); + Ok(()) +} + +pub(crate) fn encode_secondary_root_extra_1_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + write_runs(out, SECONDARY_ROOT_EXTRA_1_RUNS); + // G18G: KEXT_PROVEN identical store geometry for this 0x60 object + // (26A428 vs 25G76 R_60 stores, all at the same offsets); the J813 values + // are carried unchanged (INFERRED, listed in G18G_26A428_UNPROVEN). + if matches!(profile, RootProfile::J813G17G25G76 | RootProfile::J873G18G26A428) { + put_u32(out, 0x10, 2); + } + Ok(()) +} + +/// Recorded T8140 region-C platform tuning constants (values known, meanings +/// not; generated from the hardware-validated `g17p_initdata.py` +/// `REGION_C_CONSTANTS`). 12 sparse runs, 180 bytes total. The final run +/// initializes the byte preceding the host power-assert counter and the +/// counter itself to their native cold value of zero. +pub(crate) const REGION_C_RUNS: &[(usize, &[u8])] = &[ + (0x0024, &[0xb8, 0x0b, 0x00, 0x00]), + ( + 0x0030, + &[ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x0054, + &[ + 0xff, 0xff, 0x28, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, + ], + ), + (0x0078, &[0x01, 0x00, 0x00, 0x00]), + ( + 0x0084, + &[ + 0x4e, 0x25, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, + ], + ), + (0x0098, &[0xe8, 0x03, 0x00, 0x00]), + ( + 0x07c4, + &[ + 0xd0, 0x07, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0xcd, 0xcc, 0x4c, 0x3f, 0xcd, 0xcc, + 0x4c, 0x3e, 0x66, 0x66, 0x66, 0x3f, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x80, 0x3e, + 0x9a, 0x99, 0x19, 0x3f, 0x66, 0x66, 0x66, 0x3f, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, + ], + ), + ( + 0x0998, + &[ + 0x28, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x09b8, + &[ + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + ], + ), + (0x0e1c, &[0x00, 0x01, 0x00, 0x00]), + (0x0e38, &[0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00]), + (0x0e50, &[0x00, 0x00, 0x00, 0x00]), +]; + +pub(crate) const SECONDARY_ROOT_EXTRA_1_RUNS: &[(usize, &[u8])] = &[ + (0x14, &[0x01]), + (0x2c, &[0x01]), + (0x30, &[0x01]), + (0x40, &[0x01]), + (0x4c, &[0x01]), + (0x50, &[0x6a, 0x18]), +]; + +/// Root `+0x00`: u16 {0x04c0, 0x83a0, 0x4339, 0x0c8b}. FW_PROVEN: b000 +/// 0x2164c-0x21660 and b010 0x294f0-0x29504 compare root+0 with this word and +/// abort with "Host and FW Interface is mismatched" otherwise. KEXT_PROVEN: +/// AGXArmFirmware::initFirmwareData (0xfffffe0008c31f6c) writes it to both +/// roots. +pub(crate) const G18G_26A428_INITDATA_VERSION_WORD: u64 = 0x0c8b_4339_83a0_04c0; + +/// Both roots are 0xc8 bytes. FW_PROVEN: each image copies 0xc8 bytes of its +/// root (b000 0x21664-0x216a8) and requires root+0x2c != 0 (b000 0x216ac). +/// Root field offsets are unchanged from 25G76 (KEXT_PROVEN g18g_key.c:3748). +pub(crate) const G18G_26A428_ROOT_SIZE: usize = 0xc8; + +/// `AGXFirmware::allocFirmwareData` allocation-size table, 24 u64. +/// KEXT_PROVEN: 25G76 __const 0xfffffe0007139820. Index names are the +/// `KX_*` constants below. +pub(crate) const KEXT_ALLOCATION_SIZES_25G76: [usize; 24] = [ + 0x2708, 0x0c18, 0x1048, 0x0e10, 0x6a30, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe440, 0x06a8, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e78, 0x00e4, +]; + +/// KEXT_PROVEN: 26A428 __const 0xfffffe000714f0d8. Only entries 0 (hw_data), +/// 4 (perf/policy), 12 (status B), 13 (status config) and 22 (region C) +/// differ from 25G76. +pub(crate) const KEXT_ALLOCATION_SIZES_26A428: [usize; 24] = [ + 0x2848, 0x0c18, 0x1048, 0x0e10, 0x7800, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe730, 0x06d0, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e80, 0x00e4, +]; + +pub(crate) const KX_HW_DATA: usize = 0; + +pub(crate) const KX_BLOCK_C18: usize = 1; + +pub(crate) const KX_BLOCK_1048: usize = 2; + +pub(crate) const KX_BLOCK_E10: usize = 3; + +pub(crate) const KX_POLICY: usize = 4; + +pub(crate) const KX_BLOCK_68: usize = 5; + +pub(crate) const KX_BLOCK_800: usize = 6; + +pub(crate) const KX_BLOCK_88: usize = 7; + +pub(crate) const KX_DM_RING: usize = 8; + +pub(crate) const KX_MAIN_CONFIG: usize = 9; + +pub(crate) const KX_CONTROL_RING: usize = 10; + +pub(crate) const KX_REGION_C: usize = 22; + +/// `getSizeOfFWBRNTable` returns 0 in both kexts (KEXT_PROVEN +/// 0xfffffe00089ee74c / 0xfffffe0008c507e4). +pub(crate) const KEXT_BRN_TABLE_SIZE: usize = 0; + +/// Both shared allocations are rounded to one 16 KiB page (allocateSharedData). +pub(crate) const KEXT_ALLOCATION_PAGE: usize = 0x4000; + +pub(crate) const SHARED_ALLOCATION_ENTRIES: usize = 25; + +const fn round_up_usize(value: usize, alignment: usize) -> usize { + (value + alignment - 1) & !(alignment - 1) +} + +/// The allocateSharedData packing rule (KEXT_PROVEN 0xfffffe0008c038d8): +/// each next entry starts at `(previous offset + previous size + 0x3f) & !0x3f`. +pub(crate) const fn next_packed_allocation(offset: usize, size: usize) -> usize { + round_up_usize(offset + size, 0x40) +} + +/// Offsets of `sizes` packed by [`next_packed_allocation`], the first at 0. +pub(crate) const fn packed_allocation_offsets(sizes: &[usize; N]) -> [usize; N] { + let mut out = [0usize; N]; + let mut index = 1; + while index < N { + out[index] = next_packed_allocation(out[index - 1], sizes[index - 1]); + index += 1; + } + out +} + +/// The 25 entries of shared allocation #1 in allocation order: hw_data, the +/// 0xc18/0x1048/0xe10 blocks, perf/policy, the 0x68/0x800 blocks, the BRN +/// table, the 0x88 block, twelve data-master rings, then main config and +/// device-control ring for role 0 and for role 1 (KEXT_PROVEN descriptor +/// build, 25G76 g17g_key.c:1224-1310 / 26A428 g18g_key.c:1236-1320). +pub(crate) const fn shared_allocation_sizes(table: &[usize; 24]) -> [usize; SHARED_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; SHARED_ALLOCATION_ENTRIES]; + sizes[0] = table[KX_HW_DATA]; + sizes[1] = table[KX_BLOCK_C18]; + sizes[2] = table[KX_BLOCK_1048]; + sizes[3] = table[KX_BLOCK_E10]; + sizes[4] = table[KX_POLICY]; + sizes[5] = table[KX_BLOCK_68]; + sizes[6] = table[KX_BLOCK_800]; + sizes[7] = KEXT_BRN_TABLE_SIZE; + sizes[8] = table[KX_BLOCK_88]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + sizes[9 + ring] = table[KX_DM_RING]; + ring += 1; + } + sizes[21] = table[KX_MAIN_CONFIG]; + sizes[22] = table[KX_CONTROL_RING]; + sizes[23] = table[KX_MAIN_CONFIG]; + sizes[24] = table[KX_CONTROL_RING]; + sizes +} + +/// Offsets (from the start of the shared allocation #1 backing) of every +/// object in it. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedClusterLayout { + pub(crate) hw_data: usize, + pub(crate) block_c18: usize, + pub(crate) block_1048: usize, + pub(crate) block_e10: usize, + pub(crate) policy: usize, + pub(crate) block_68: usize, + pub(crate) block_800: usize, + pub(crate) brn_table: usize, + pub(crate) block_88: usize, + /// Data-master rings in allocation order (see [`WORK_CHANNEL_DM_INDEX`]). + pub(crate) dm_rings: [usize; WORK_CHANNEL_COUNT], + pub(crate) main_config: [usize; 2], + pub(crate) control_ring: [usize; 2], + pub(crate) end: usize, + pub(crate) total: usize, +} + +pub(crate) const fn shared_cluster_layout(table: &[usize; 24]) -> SharedClusterLayout { + let sizes = shared_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_rings = [0usize; WORK_CHANNEL_COUNT]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + dm_rings[ring] = offsets[9 + ring]; + ring += 1; + } + let end = offsets[24] + sizes[24]; + SharedClusterLayout { + hw_data: offsets[0], + block_c18: offsets[1], + block_1048: offsets[2], + block_e10: offsets[3], + policy: offsets[4], + block_68: offsets[5], + block_800: offsets[6], + brn_table: offsets[7], + block_88: offsets[8], + dm_rings, + main_config: [offsets[21], offsets[23]], + control_ring: [offsets[22], offsets[24]], + end, + total: round_up_usize(end, KEXT_ALLOCATION_PAGE), + } +} + +/// The 26A428 layouts: hw_data 0x2848 at 0, blocks at 0x2880/0x34c0/0x4540, +/// perf/policy 0x7800 at 0x5380, 0x68/0x800 at 0xcb80/0xcc00, BRN and 0x88 at +/// 0xd400, rings from 0xd4c0, main r0 0x1f4c0, control r0 0x1f980, main r1 +/// 0x23980, control r1 0x23e40; status B 0x340, status config 0xea80, status +/// A 0xf180/0xc3440, tail 0x177700/0x177740/0x1777c0. Totals 0x28000 and +/// 0x178000, as in 25G76. +pub(crate) const G18G_26A428_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&KEXT_ALLOCATION_SIZES_26A428); + +/// Main `+0x254` views (KEXT_PROVEN 26A428 initFirmwareSharedData: the FW VAs +/// of the 0xc18, 0x1048, 0xe10, 0x68 and 0x800 blocks, this+0x310..0x338). +pub(crate) const G18G_26A428_BUNDLE_VIEW_OFFSETS: [usize; 5] = [ + G18G_26A428_SHARED_LAYOUT.block_c18, + G18G_26A428_SHARED_LAYOUT.block_1048, + G18G_26A428_SHARED_LAYOUT.block_e10, + G18G_26A428_SHARED_LAYOUT.block_68, + G18G_26A428_SHARED_LAYOUT.block_800, +]; + +pub(crate) const G18G_26A428_BUNDLE_VIEW_EXTENTS: [usize; 5] = [ + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_C18], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_1048], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_E10], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_68], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_800], +]; + +/// Role-1 main `+0x481`: FW VA of the perf/policy allocation (KEXT_PROVEN). +pub(crate) const G18G_26A428_MAIN_SECONDARY_POLICY_PTR: usize = 0x481; + +/// Role-1 main `+0x300`. INFERRED: the J813 value (26A428 source accel+0x10a74). +pub(crate) const G18G_26A428_MAIN_SECONDARY_KIND: u32 = 0xc; + +/// J873 ADT gpu-num-perf-states. ADT_PROVEN. +pub(crate) const J873_GPU_NUM_PERF_STATES: u32 = 13; + +/// Region-C ADT tunables. KEXT_PROVEN: +0x7c8 = accel+0x106b0 +/// (gpu-idleoff-standby-timer; J813 +0x7c4 carried its ADT 1250) and +0x7e4 = +/// accel+0x106cc (gpu-idleoff-min_conf); ADT_PROVEN values 1500 and 0.8f. +/// +0x9c = this+0xf14 * 100 (25G76 +0x98 = this+0xf40 * 100 = 1200 on J813): +/// INFERRED to be the perf-state count again. +pub(crate) const G18G_26A428_REGION_C_ADT_WORDS: [(usize, u32); 3] = [ + (0x7c8, 1500), + (0x7e4, 0x3f4c_cccd), + (0x09c, J873_GPU_NUM_PERF_STATES * 100), +]; + +/// Stable region-C inputs from the same eight initialization snapshots. +/// +0x38 is the new 26A428 feature bit, no longer a guessed zero. +pub(crate) const G18G_26A428_REGION_C_CAPTURED_WORDS: [(usize, u32); 7] = [ + (0x0038, 1), (0x0088, 0x00008f16), (0x008c, 0x00000514), + (0x0090, 0x00000514), (0x09b0, 0x00000200), + (0x09dc, 0x00000032), (0x0e0c, 0x000000fa), +]; + +/// Encode the G18G region C: the J813 bytes, the 25G76 fields from +0x38 on +/// moved up by 4 (FW_PROVEN), the J873 ADT tunables and stable native J873 +/// inputs. The new +0x38 feature bit (accel+0x6b9 bit 2) is captured as one. +pub(crate) fn encode_g18g_26a428_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + encode_region_c_for_profile(RootProfile::J813G17G25G76, out)?; + // The J813 object never reaches its 0xe78 end; shifting in place keeps the + // tail zero and needs no second 4 KiB buffer on the kernel stack. + let shifted = KEXT_ALLOCATION_SIZES_25G76[KX_REGION_C]; + if out[shifted..].iter().any(|byte| *byte != 0) { + return Err(InitdataError::BufferSize); + } + out.copy_within(0x38..shifted, 0x3c); + out[0x38..0x3c].fill(0); + for (offset, value) in G18G_26A428_REGION_C_ADT_WORDS { + put_u32(out, offset, value); + } + for (offset, value) in G18G_26A428_REGION_C_CAPTURED_WORDS { + put_u32(out, offset, value); + } + // Idle power-off policy word (Region C +0x30). The firmware loads + // it on every cold wake, which undoes the IdlePowerOff(0) sent once at boot. + // 0 keeps the scheduler idle event from powering the cores down between + // jobs. Measured on the M6: 8.6 percent less time per op. + put_u32(out, 0x30, 0); + Ok(()) +} + +/// FW_PROVEN (jump table 0xfffffc0000006c60, 56 entries from opcode 4): the +/// opening/region-C reload handler (b000 0x3ed0) is reached by 0x15, 0x2a +/// and 0x30. +pub(crate) const G18G_26A428_CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x15; + +pub(crate) const G18G_26A428_CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; diff --git a/drivers/gpu/drm/asahi/m5/live_boot.rs b/drivers/gpu/drm/asahi/m5/live_boot.rs new file mode 100644 index 0000000000000..89bf0b08c7124 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/live_boot.rs @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5-specific initialization and retained application submission session. +//! DRM admission remains in the device adapter; no G17P manager is selected. + +use super::provider::{validate_provider, ExpectedProvider, G17PAscwrapV6Role, + G17PMailboxProvider, G17PNoRtkitBuffer}; +use crate::{driver::AsahiDrmDevice, m5::resources::M5FirmwareMemory, + m5::fw_msg, mmu, regs}; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use crate::m5::completion::{graphics_pointer_head, MAX_GRAPHICS_GENERATION}; + +/// Application jobs are polled for retirement. A 1 ms step put up to a +/// millisecond of idle GPU after every job, several per video frame. +/// (The first render job keeps 1 ms steps; see submit_application.) +const APPLICATION_POLL_US: i64 = 50; +const APPLICATION_POLLS_PER_MS: u32 = 1000 / APPLICATION_POLL_US as u32; +use kernel::{device::Core, macros::vtable, platform, prelude::*, + soc::apple::rtkit, sync::{aref::ARef, Arc}, time::{delay::fsleep, Delta}}; + +struct M5RoleState { + dev: ARef, + role: usize, + acked: AtomicBool, + crashed: AtomicBool, + events: AtomicU32, +} + +struct M5RtkitOps; + +#[vtable] +impl rtkit::Operations for M5RtkitOps { + type Data = Arc; + type Buffer = G17PNoRtkitBuffer; + + fn recv_message_early( + data: ::Borrowed<'_>, + endpoint: u8, + message: u64, + ) -> bool { + // Both native M5 roles reply with exactly 0x0009000000000000. + if endpoint == 0x20 && message == 0x0009_0000_0000_0000 { + data.acked.store(true, Ordering::Release); + true + } else { + false + } + } + + fn recv_message( + data: ::Borrowed<'_>, + endpoint: u8, + message: u64, + ) { + let ordinal = data.events.fetch_add(1, Ordering::Relaxed); + if ordinal < 8 { + dev_dbg!(data.dev.as_ref(), "M5 message role={} ep={:#x} payload={:#x}\n", + data.role, endpoint, message); + } + } + + fn crashed( + data: ::Borrowed<'_>, + _crashlog: Option<&[u8]>, + ) { + data.crashed.store(true, Ordering::Release); + dev_err!(data.dev.as_ref(), "M5 firmware crashed role={}\n", data.role); + } +} + +struct M5Role { + provider: G17PMailboxProvider, + rtkit: rtkit::RtKit, + state: Arc, + started: bool, +} + +struct M5Session { + roles: [M5Role; 2], + memory: Option>, + uat: Arc, + /// The GPU mappings of both firmware crash log buffers. + _crash_views: [mmu::KernelMapping; 2], + registers: regs::G17Resources, + dev: ARef, +} + +impl M5Session { + fn stop_cpus(&mut self) -> Result { + let mut failure = None; + // Reverse the measured M5 GFX1/GFX start order. + for index in [0, 1] { + if self.roles[index].started { + match self.roles[index].provider.stop_cpu() { + Ok(()) => self.roles[index].started = false, + Err(error) => failure = Some(error), + } + } + } + failure.map_or(Ok(()), Err) + } + + fn initialize(&mut self) -> Result { + self.registers.m5_match_pmgr_boot_control()?; + for index in [1, 0] { + let role = &mut self.roles[index]; + role.provider.start_cpu()?; + role.started = true; + Pin::new(&mut role.rtkit).boot()?; + dev_dbg!(self.dev.as_ref(), "M5_INIT role={} rtkit-ready\n", index); + } + // These helpers operate only on the proven shared dual-root layout; + // the Neo-specific contexts, metrics and manager remain unselected. + let roots = self.uat.adopt_firmware_root_entries()?; + let timestamp: u64; + // SAFETY: Read-only architectural counter, available on this target. + unsafe { core::arch::asm!("mrs {value}, cntvct_el0", value = out(reg) timestamp, + options(nomem, nostack, preserves_flags)); } + self.memory = Some(M5FirmwareMemory::new_with_uat( + &self.dev, self.uat.clone(), timestamp)?); + self.uat.confirm_firmware_root_entries(roots)?; + self.uat.mirror_secondary_root()?; + dev_dbg!(self.dev.as_ref(), "M5_INIT retained memory graph mapped\n"); + self.registers.enable_g17_gating()?; + + // M5's own trace publishes and ACKs GFX1 first (133/137), then GFX + // (160/164). The Neo's opposite ordering is not used here. + for index in [1, 0] { + let address = self.memory.as_ref().ok_or(EIO)?.root(index == 1); + let message = fw_msg::encode_initdata_doorbell(address).map_err(|_| EINVAL)?; + let role = &mut self.roles[index]; + Pin::new(&mut role.rtkit).start_endpoint(0x20)?; + Pin::new(&mut role.rtkit).start_endpoint(0x21)?; + Pin::new(&mut role.rtkit).send_message(0x20, message)?; + dev_dbg!(self.dev.as_ref(), "M5_INIT role={} published root={:#x}\n", index, address); + for _ in 0..200 { + if self.memory.as_mut().ok_or(EIO)?.drain_reports(&self.dev)? { + return Err(EIO); + } + if role.state.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if role.state.acked.load(Ordering::Acquire) { + break; + } + fsleep(Delta::from_millis(10)); + } + if !role.state.acked.load(Ordering::Acquire) { + return Err(ETIMEDOUT); + } + dev_dbg!(self.dev.as_ref(), "M5_INIT role={} initdata-ack\n", index); + } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + self.retire_openings() + } + + fn install_graphics_queues(&mut self) -> Result { + let memory = self.memory.as_mut().ok_or(EIO)?; + dev_dbg!(self.dev.as_ref(), "M5_GRAPHICS install before counters={:x?} registration={:x?}\n", + memory.graphics_status()?, memory.graphics_installation_status()?); + memory.publish_graphics_install()?; + Pin::new(&mut self.roles[0].rtkit).send_message(0x21, 0x0083_0000_0000_000a)?; + for poll in 0..1000 { + if memory.drain_reports(&self.dev)? { return Err(EIO); } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + let (outer, inner, _) = memory.graphics_status()?; + if outer == [2; 3] && inner == [[1; 3]; 2] + && memory.graphics_queues_installed()? { + dev_dbg!(self.dev.as_ref(), "M5_GRAPHICS installed TA/fragment queues; registration={:x?}; no draws published\n", + memory.graphics_installation_status()?); + return Ok(()); + } + if [0, 100].contains(&poll) { + dev_dbg!(self.dev.as_ref(), "M5_GRAPHICS install poll={} counters={:x?} registration={:x?}\n", + poll, memory.graphics_status()?, memory.graphics_installation_status()?); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.dev.as_ref(), "M5_GRAPHICS install timeout counters={:x?} registration={:x?}\n", + memory.graphics_status()?, memory.graphics_installation_status()?); + Err(ETIMEDOUT) + } + + fn retire_idle_policy(&mut self) -> Result { + let memory = self.memory.as_mut().ok_or(EIO)?; + // Retire the idle-policy request. + memory.publish_idle_wake()?; + Pin::new(&mut self.roles[0].rtkit).send_message(0x21, 0x0084_0000_0000_0011)?; + for _ in 0..1000 { + if memory.drain_reports(&self.dev)? { return Err(EIO); } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + dev_err!(self.dev.as_ref(), "M5: firmware crashed retiring the idle policy\n"); + return Err(EIO); + } + if memory.control_counters()? == [[2; 3], [1; 3]] { + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.dev.as_ref(), "M5: idle policy not retired, counters={:?}\n", + memory.control_counters()?); + Err(ETIMEDOUT) + } + + fn retire_openings(&mut self) -> Result { + let mut counters = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + dev_dbg!(self.dev.as_ref(), "M5_OPEN before power-transition-end {:?}\n", counters); + // M5's own cold trace sends one 0x89 to GFX, then GFX1 (events + // 167/171), immediately after the two initdata ACKs. No other control + // record or work item is published in this bounded experiment. + for index in [0, 1] { + Pin::new(&mut self.roles[index].rtkit) + .send_message(0x21, fw_msg::encode_power_transition_end())?; + } + for _ in 0..1000 { + if self.memory.as_mut().ok_or(EIO)?.drain_reports(&self.dev)? { + return Err(EIO); + } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + counters = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + if counters == [[1; 3]; 2] { + dev_dbg!(self.dev.as_ref(), "M5_OPEN both native control openings retired {:?}\n", counters); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.dev.as_ref(), "M5_OPEN control openings did not retire {:?}\n", counters); + Err(ETIMEDOUT) + } + +} + +/// Sizes of the GFX and GFX1 `__DATA` segments the 25G76 firmware runs with. +const DATA_SIZES: [u64; 2] = [0x134000, 0x130000]; +/// The firmware's crash log buffer, at this offset of each `__DATA` segment. +const CRASH_LOG_OFFSET: u64 = 0xa0000; +const CRASH_LOG_SIZE: usize = 0x8000; + +/// Creates the UAT both firmware instances share, as the A18 Pro's: each instance publishes +/// its own kernel root, the secondary one 0x40000 into `pagetables`. Each instance asks for +/// its crash log at the GPU address of `__DATA` + 0xa0000 truncated to 40 bits, which is +/// mapped here, before either instance starts. +fn new_uat(dev: &AsahiDrmDevice) -> Result<(Arc, [mmu::KernelMapping; 2])> { + let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let mut crash_logs = [0u64; 2]; + for (index, name) in [c"gfx-data", c"gfx1-data"].into_iter().enumerate() { + let region = node.reserved_mem_region_to_resource_byname(name)?; + if region.size() != DATA_SIZES[index] || region.start() & (mmu::UAT_PGSZ as u64 - 1) != 0 + { + return Err(EINVAL); + } + crash_logs[index] = region.start() + CRASH_LOG_OFFSET; + } + let uat = Arc::new( + mmu::Uat::new( + dev.into(), + mmu::UatGeometry { ias: 42, oas: 42 }, + false, + mmu::UatFirmware::PublishedRoots { secondary_root_offset: Some(0x40000) }, + mmu::UatQuirks::DEFAULT, + )?, + GFP_KERNEL, + )?; + let map = |pa: u64| { + uat.kernel_lower_vm().map_io( + dev.as_ref().as_ref(), + pa & ((1u64 << 40) - 1), + pa.try_into()?, + CRASH_LOG_SIZE, + mmu::PROT_FW_SHARED_RW, + ) + }; + let views = [map(crash_logs[0])?, map(crash_logs[1])?]; + Ok((uat, views)) +} + +pub(crate) struct M5Runtime { + session: Option>, + application: bool, + faulted: bool, + application_generation: u32, + application_compute_generation: u32, +} + +impl M5Runtime { + fn prepare( + pdev: &platform::Device>, + dev: ARef, + registers: regs::G17Resources, + ) -> Result { + let gfx = G17PMailboxProvider::get(pdev.as_ref(), 0)?; + let gfx1 = G17PMailboxProvider::get(pdev.as_ref(), 1)?; + for (provider, expected) in [ + (&gfx, ExpectedProvider { role: G17PAscwrapV6Role::Gfx, + wrapper: (0x582600000, 0x88000), iop_vbar: (0x582050000, 8) }), + (&gfx1, ExpectedProvider { role: G17PAscwrapV6Role::Gfx1, + wrapper: (0x582e00000, 0x88000), iop_vbar: (0x582850000, 8) }), + ] { + validate_provider(provider, expected)?; + provider.require_safe_cpu_lifecycle()?; + } + let (uat, crash_views) = new_uat(&dev)?; + let make_role = |index, provider| -> Result { + let state = Arc::new(M5RoleState { dev: dev.clone(), role: index, + acked: AtomicBool::new(false), crashed: AtomicBool::new(false), + events: AtomicU32::new(0) }, GFP_KERNEL)?; + let rtkit = rtkit::RtKit::::new_with_firmware_crashlog( + dev.as_ref().as_ref(), None, index, state.clone())?; + Ok(M5Role { provider, rtkit, state, started: false }) + }; + let session = KBox::new(M5Session { + roles: [make_role(0, gfx)?, make_role(1, gfx1)?], + memory: None, uat, _crash_views: crash_views, registers, dev, + }, GFP_KERNEL)?; + // From this point every path returns the retained platform owner, + // even when firmware rejects the graph. Failed probe unwinding must + // never release UAT or transports beneath a running firmware CPU. + // Keep callback code resident even if a failed stop retains owners. + // This explicit RAM-boot experiment is released by reboot only. + unsafe { kernel::bindings::__module_get((*kernel::module::this_module::()).as_ptr()); } + Ok(Self { session: Some(session), application: false, faulted: false, + application_generation: 0, application_compute_generation: 0 }) + } + + /// Boot without a native program or job graph and leave both processors + /// running for explicit application submissions. This does not register + /// a render node or claim that Mesa command translation is complete. + pub(crate) fn start_application( + pdev: &platform::Device>, dev: ARef, registers: regs::G17Resources, + ) -> Result { + // Periodic power/submission traces need continuous draining while + // the session idles. Recovery bit 9 emits only on a fault, which the + // synchronous submit loop drains before retaining a failed graph. + if crate::module_parameters::m5_fw_trace.value() & !0x200 != 0 { + return Err(EINVAL); + } + let mut owner = Self::prepare(pdev, dev, registers)?; + owner.application = true; + let result = (|| { + let session = owner.session.as_mut().ok_or(EIO)?; + session.initialize()?; + session.retire_idle_policy()?; + Ok(()) + })(); + if let Err(error) = result { + owner.faulted = true; + return Err(error); + } + dev_info!(pdev.as_ref(), "M5: GPU firmware 25G76 running\n"); + Ok(owner) + } + + pub(crate) fn new_user_vm(&self, id: u64, kernel_range: core::ops::Range) -> Result { + if !self.application || self.is_crashed() { return Err(ENODEV); } + crate::m5::vm_layout::compact_range(&kernel_range).ok_or(EINVAL)?; + self.session.as_ref().ok_or(EIO)?.uat.new_vm(id, kernel_range)?.with_status() + } + + pub(crate) fn is_crashed(&self) -> bool { + self.faulted || self.session.as_ref().is_none_or(|session| + session.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire))) + } + + /// The adapter serializes this call and validates the translated command + /// plus every user mapping before entry. Return success only after paired + /// queue retirement, scheduler unlink, stamps and firmware notifications. + /// A failed published job retains and quarantines its VM until reboot. + #[allow(clippy::too_many_arguments)] + pub(crate) fn submit_application( + &mut self, vm: &mmu::Vm, parameters: crate::m5::render_desc::G17pRenderParameters, + controls: crate::m5::render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], + ) -> Result { + if !self.application || self.is_crashed() { return Err(ENODEV); } + // Reuse retired ring storage while keeping completion stamps unique. + if self.application_generation >= MAX_GRAPHICS_GENERATION { return Err(EBUSY); } + let generation = self.application_generation + 1; + let session = self.session.as_deref_mut().ok_or(EIO)?; + if generation == 1 { + session.memory.as_mut().ok_or(EIO)?.initialize_application_graphics( + &session.dev, vm, parameters, controls, clusters, usc_required)?; + } + // From installation onward, even a failed mailbox send can have made + // memory visible to firmware. Never unwind or recycle the job graph. + self.application_generation = generation; + let result = (|| { + if generation == 1 { session.install_graphics_queues()?; } + let memory = session.memory.as_mut().ok_or(EIO)?; + if generation == 1 { + memory.publish_graphics_job()?; + } else { + memory.publish_next_application_job(&session.dev, vm, parameters, controls, + clusters, usc_required, generation)?; + } + Pin::new(&mut session.roles[0].rtkit).send_message(0x21, 0x0083_0000_0000_000a)?; + for _ in 0..10000 * APPLICATION_POLLS_PER_MS { + if memory.drain_reports(&session.dev)? + || session.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + let (outer, inner, _) = memory.graphics_status()?; + let completion = memory.graphics_completion()?; + if outer == [memory.application_outer_head(); 3] + && inner == [[graphics_pointer_head(generation); 3]; 2] + && M5FirmwareMemory::graphics_pair_complete(completion, generation) { + dev_dbg!(session.dev.as_ref(), + "M5_APPLICATION generation={} completed; both queues retired\n", + generation); + return Ok(completion); + } + // The first job also installs the queues; polling it this + // often met a firmware recovery on its fragment queue. + fsleep(Delta::from_micros(if generation == 1 { 1000 } else { APPLICATION_POLL_US })); + } + Err(ETIMEDOUT) + })(); + if let Err(error) = result { + if let Some(memory) = session.memory.as_mut() { + for _ in 0..4 { let _ = memory.drain_reports(&session.dev); } + dev_err!(session.dev.as_ref(), + "M5_APPLICATION failed graph counters={:x?} completion={:x?}\n", + memory.graphics_status(), memory.graphics_completion()); + } + self.faulted = true; + if let Some(status) = vm.status() { status.record(error); } + if let Some(memory) = &session.memory { memory.quarantine_graphics(); } + // A successful stop request alone does not prove a halt. Keep + // the full session even after this best-effort stop succeeds. + let stopped = session.stop_cpus(); + dev_err!(session.dev.as_ref(), + "M5_APPLICATION submission failed {:?}; stop={:?}; VM and firmware owners retained\n", + error, stopped); + } + result + } + + /// Execute a compute command in the already retained application VM. + /// Installation, dispatch and retirement are separate firmware transitions. + /// The first submission installs queue 12 and dispatches generation 1; + /// every later one reuses that owner and dispatches the next generation, + /// as submit_application() does for the graphics pair. + pub(crate) fn submit_application_compute(&mut self, vm: &mmu::Vm, + command: &crate::m5::command::TranslatedComputeCommand) -> Result { + if !self.application || self.is_crashed() || self.application_generation == 0 { + return Err(ENOTSUPP); + } + if self.application_compute_generation >= MAX_GRAPHICS_GENERATION { + return Err(EBUSY); + } + let generation = self.application_compute_generation + 1; + let session = self.session.as_deref_mut().ok_or(EIO)?; + if generation == 1 { + session.memory.as_mut().ok_or(EIO)?.initialize_application_compute( + &session.dev, vm, command, self.application_generation)?; + } else { + session.memory.as_mut().ok_or(EIO)?.prepare_next_application_compute( + &session.dev, vm, command, generation)?; + } + // The owner is retained from installation onward. Even a failed mailbox + // send can follow firmware seeing the new producer, so never unwind. + self.application_compute_generation = generation; + let result = (|| { + let memory = session.memory.as_mut().ok_or(EIO)?; + // Installation is published once; the dispatch record every time. + let stages: &[bool] = if generation == 1 { &[false, true] } else { &[true] }; + for &dispatch in stages { + memory.publish_application_compute(dispatch, generation)?; + Pin::new(&mut session.roles[0].rtkit) + .send_message(0x21, 0x0083_0000_0000_000a)?; + let mut retired = false; + for _ in 0..if dispatch { 10000 } else { 1000 } * APPLICATION_POLLS_PER_MS { + if memory.drain_reports(&session.dev)? + || session.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + let (outer, _, _) = memory.graphics_status()?; + if outer == [memory.application_outer_head(); 3] + && if dispatch { memory.application_compute_complete(generation)? } + else { memory.application_compute_installed()? } { + retired = true; + break; + } + fsleep(Delta::from_micros(APPLICATION_POLL_US)); + } + if !retired { return Err(ETIMEDOUT); } + dev_dbg!(session.dev.as_ref(), + "M5_APPLICATION_COMPUTE {} generation={} queue=12 outer={} status={:x?}\n", + if dispatch { "completed" } else { "installed" }, generation, + memory.application_outer_head(), memory.application_compute_status()); + } + Ok(()) + })(); + if let Err(error) = result { + self.faulted = true; + if let Some(status) = vm.status() { status.record(error); } + if let Some(memory) = session.memory.as_mut() { + for _ in 0..4 { let _ = memory.drain_reports(&session.dev); } + memory.quarantine_graphics(); + dev_err!(session.dev.as_ref(), + "M5_APPLICATION_COMPUTE failed {:?}; outer/graphics={:x?} compute={:x?}\n", + error, memory.graphics_status(), memory.application_compute_status()); + } + let stopped = session.stop_cpus(); + dev_err!(session.dev.as_ref(), + "M5_APPLICATION_COMPUTE stop={:?}; VM and owners retained until reboot\n", stopped); + } + result + } +} + +impl Drop for M5Runtime { + fn drop(&mut self) { + if let Some(mut session) = self.session.take() { + if let Some(memory) = &session.memory { memory.quarantine_graphics(); } + let stopped = session.stop_cpus(); + dev_dbg!(session.dev.as_ref(), + "M5 session release requested; stop={:?}; halt unqualified, owners retained until reboot\n", + stopped); + core::mem::forget(session); + } + } +} diff --git a/drivers/gpu/drm/asahi/m5/mapped.rs b/drivers/gpu/drm/asahi/m5/mapped.rs new file mode 100644 index 0000000000000..2ccc37b82531b --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/mapped.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Kernel-owned, write-combined GEM objects mapped at one GPU address. + +use core::ops::Range; +use kernel::prelude::*; + +use crate::{driver::AsahiDrmDevice, gem, mmu}; + +/// A zeroed, write-combined kernel object and its owning GPU mapping. +pub(crate) struct MappedObject { + _mapping: mmu::KernelMapping, + object: gem::ObjectRef, + iova: u64, + /// Size of the object in bytes, as requested. + pub(crate) logical_size: usize, + /// The VM of the owning mapping. + home: u64, +} + +impl MappedObject { + fn new_object(dev: &AsahiDrmDevice, size: usize) -> Result { + let mut object = gem::new_kernel_object_wc(dev, size)?; + let len = object.size(); + let vmap = object.vmap()?; + // SAFETY: the vmap covers the whole page-rounded allocation. + unsafe { core::ptr::write_bytes(vmap.addr.cast::(), 0, len) }; + Ok(object) + } + + fn from_mapping( + object: gem::ObjectRef, vm: &mmu::Vm, mapping: mmu::KernelMapping, size: usize, + ) -> Self { + let iova = mapping.iova(); + Self { _mapping: mapping, object, iova, logical_size: size, home: vm.id() } + } + + /// Maps a new object of `size` bytes at `address`. + pub(crate) fn new_wc_at( + dev: &AsahiDrmDevice, vm: &mmu::Vm, address: u64, size: usize, prot: mmu::Prot, + ) -> Result { + let mut object = Self::new_object(dev, size)?; + let mapping = object.map_at(vm, address, prot, false)?; + Ok(Self::from_mapping(object, vm, mapping, size)) + } + + /// Maps a new object of `size` bytes at a free address in `range`. + pub(crate) fn new_wc_in_range( + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: Range, size: usize, alignment: u64, + prot: mmu::Prot, + ) -> Result { + let mut object = Self::new_object(dev, size)?; + let mapping = object.map_into_range(vm, range, alignment, prot, true)?; + Ok(Self::from_mapping(object, vm, mapping, size)) + } + + /// As [`Self::new_wc_in_range`], followed by `guard_pages` unmapped pages. + pub(crate) fn new_wc_in_range_with_guards( + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: Range, size: usize, alignment: u64, + prot: mmu::Prot, guard_pages: usize, + ) -> Result { + let object = Self::new_object(dev, size)?; + let guard = guard_pages.checked_mul(mmu::UAT_PGSZ).ok_or(EOVERFLOW)?; + let mapping = vm.map_in_range_with_guard_size( + &object.gem, 0..object.size(), alignment, range, prot, guard, + )?; + Ok(Self::from_mapping(object, vm, mapping, size)) + } + + /// Maps this object again at `address` of `vm`. + pub(crate) fn map_alias_at( + &mut self, vm: &mmu::Vm, address: u64, prot: mmu::Prot, + ) -> Result { + self.object.map_at(vm, address, prot, false) + } + + /// Maps this object at its own address in another client VM, followed by `guard_pages` + /// unmapped pages. In the VM of the owning mapping, where the object already is at that + /// address, there is nothing to map. + pub(crate) fn map_alias_at_with_guards( + &mut self, vm: &mmu::Vm, address: u64, prot: mmu::Prot, guard_pages: usize, + ) -> Result> { + if vm.id() == self.home { + return if address == self.iova { Ok(None) } else { Err(EINVAL) }; + } + let guard = guard_pages.checked_mul(mmu::UAT_PGSZ).ok_or(EOVERFLOW)?; + vm.map_at_with_guard_size(address, self.object.size(), &self.object.gem, prot, guard) + .map(Some) + } + + /// GPU address of the object. + pub(crate) fn iova(&self) -> u64 { + self.iova + } + + /// Runs `f` on the CPU view of the object's `size` bytes. + pub(crate) fn with_bytes_mut(&mut self, f: impl FnOnce(&mut [u8]) -> Result) -> Result { + let size = self.logical_size; + let vmap = self.object.vmap()?; + // SAFETY: the vmap covers the whole allocation, which is at least `size` bytes, and + // `&mut self` excludes other CPU views made through this object. + let bytes = unsafe { core::slice::from_raw_parts_mut(vmap.addr.cast::(), size) }; + f(bytes) + } + + fn offset(&self, address: u64, width: usize) -> Result { + let offset: usize = address.checked_sub(self.iova).ok_or(EINVAL)?.try_into()?; + let end = offset.checked_add(width).ok_or(EINVAL)?; + if end > self.logical_size { + return Err(EINVAL); + } + Ok(offset) + } + + /// Reads the little-endian word at GPU address `address` (any alignment). + pub(crate) fn read_u32(&mut self, address: u64) -> Result { + let offset = self.offset(address, 4)?; + self.with_bytes_mut(|bytes| { + Ok(u32::from_le_bytes(bytes[offset..offset + 4].try_into().map_err(|_| EINVAL)?)) + }) + } + + /// Reads the little-endian double word at GPU address `address` (any alignment). + pub(crate) fn read_u64(&mut self, address: u64) -> Result { + let offset = self.offset(address, 8)?; + self.with_bytes_mut(|bytes| { + Ok(u64::from_le_bytes(bytes[offset..offset + 8].try_into().map_err(|_| EINVAL)?)) + }) + } + + /// Writes the naturally aligned little-endian word at GPU address `address` with one store. + pub(crate) fn write_u32(&mut self, address: u64, value: u32) -> Result { + let offset = self.offset(address, 4)?; + if offset % 4 != 0 { + return Err(EINVAL); + } + self.with_bytes_mut(|bytes| { + // SAFETY: checked bounds and natural alignment. + unsafe { + core::ptr::write_volatile(bytes.as_mut_ptr().add(offset).cast::(), value.to_le()) + }; + Ok(()) + }) + } +} diff --git a/drivers/gpu/drm/asahi/m5/mod.rs b/drivers/gpu/drm/asahi/m5/mod.rs new file mode 100644 index 0000000000000..96beaac4391f4 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/mod.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple M5 (T8142, G17G) GPU support for the 25G76 firmware (RTKit-3255.160.4). +//! +//! The M5 has two GPU coprocessors, GFX and GFX1, as the A18 Pro does. The boot loader (aura) +//! loads their firmware, checks both loaded TEXT images against the 25G76 build and says so +//! in `/chosen/aurora,m5-gpu-firmware-profile`. The firmware interface is HAL300 and differs +//! from the 26A428 and 25G83 builds the shared G17 code runs, so this backend builds its own +//! initialization graph ([`initdata`], [`resources`]) and talks to the firmware over its own +//! channels ([`channel`], [`sksm`], [`completion`]). +//! +//! # Plugging into the shared driver +//! +//! - `ProbeConfig::M5` (`apple,agx-t8142`) calls [`probe`], which starts both coprocessors +//! ([`live_boot`]) and returns a [`Gpu`] implementing [`crate::gpu::Gpu`]. +//! - The UAT is the shared [`crate::mmu::Uat`] with published roots, as on the A18 Pro, plus +//! [`crate::mmu::M5ExecutionContext`] for application contexts. User VMs carry a status and +//! a job lifetime, and the backend uses the tracked submission model of `file.rs`. +//! - The UAPI is the shared one. The backend reports `VM_STATUS` (with its mirror) and +//! `SOFT_FAULTS`, and one hardware command per submission. +//! +//! # Submission +//! +//! One render pair (TA and fragment queues) and one compute queue serve every client: the +//! queues are installed by the first render and handed from client VM to client VM, with +//! the driver's pools aliased at the same addresses in each. A submission is published, +//! then polled to retirement before the ioctl returns, so there is one job in flight. +//! Renders and compute are admitted only within the state the firmware programs have been +//! verified with ([`render`], [`compute`]); anything else is refused with `ENOTSUPP` +//! before it reaches the firmware. +//! +//! # Failures +//! +//! There is no recovery yet. A job that fails or does not retire marks its VM, stops both +//! coprocessors and retains every firmware-visible object until reboot; the device is lost +//! for all clients. The module stays loaded for the same reason. + +mod channel; +mod command; +mod completion; +mod compute; +mod drm; +mod fw_msg; +mod initdata; +mod live_boot; +mod mapped; +mod provider; +mod render; +mod render_desc; +mod render_target; +mod resources; +mod sksm; +mod submission; +mod vm_layout; + +use kernel::{device::Core, of, platform, prelude::*, sync::Arc}; + +use crate::{driver::AsahiDrmDevice, regs}; + +pub(crate) use drm::M5DrmGpu as Gpu; + +/// The firmware profile aura publishes in `/chosen/aurora,m5-gpu-firmware-profile` after it +/// checked both loaded TEXT images against the exact 25G76 build. +const FIRMWARE_PROFILE: &[u8] = b"M5-RTKit-3255.160.4.release\0"; + +/// Starts both firmware instances and returns the GPU, ready for DRM registration. +pub(crate) fn probe( + pdev: &platform::Device>, + dev: &AsahiDrmDevice, + res: regs::G17Resources, +) -> Result> { + let chosen = of::chosen().ok_or(ENODEV)?; + let profile: KVec = chosen.get_property(c"aurora,m5-gpu-firmware-profile")?; + if profile.as_slice() != FIRMWARE_PROFILE { + dev_err!(pdev.as_ref(), "M5: GPU firmware profile not published or not 25G76\n"); + return Err(ENODEV); + } + let id = res.get_g17_id()?; + let cores: u32 = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + // M5 GPUs have 10 cores, or 8 with two fused off (MacBook Air). + if id.family != 0xb || id.num_dies != 1 || !matches!(cores, 8 | 10) { + dev_err!(pdev.as_ref(), "M5: unexpected GPU family {:#x}, {} dies, {} cores\n", + id.family, id.num_dies, cores); + return Err(ENODEV); + } + // Before the session: a CLPC that has not probed yet defers the GPU. + let clpc = drm::M5ClpcInput::new(pdev.as_ref())?; + let runtime = live_boot::M5Runtime::start_application(pdev, dev.into(), res)?; + let runtime = Arc::pin_init(kernel::new_mutex!(runtime), GFP_KERNEL)?; + Ok(Arc::new(Gpu::new(runtime, id, clpc), GFP_KERNEL)?) +} diff --git a/drivers/gpu/drm/asahi/m5/provider.rs b/drivers/gpu/drm/asahi/m5/provider.rs new file mode 100644 index 0000000000000..d54a591f85448 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/provider.rs @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The GFX and GFX1 ASCWrap v6 mailbox providers, as seen by the M5 firmware session. + +use core::mem::MaybeUninit; + +use kernel::{ + bindings, + error::{from_err_ptr, to_result}, + io::{IoSysMap, Region}, + prelude::*, + soc::apple::rtkit, +}; + +use crate::mmu; + +const G17P_CRASHLOG_MAX_BYTES: usize = 256 * 1024; + +const G17P_RTKIT_UAT_DVA_MASK: u64 = (1u64 << 40) - 1; + +/// Module-local, read-only view of one existing ASCWrap mailbox provider. +/// +/// The C provider owns the mapping and all lifecycle transitions. This view +/// neither maps an alias nor changes callback ownership; it only borrows the +/// provider's established mailbox-register pointer for two control reads. +pub(super) struct G17PMailboxProvider { + mbox: *mut bindings::apple_mbox, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) enum G17PAscwrapV6Role { + Gfx, + Gfx1, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) struct G17PAscwrapV6Lifecycle { + role: G17PAscwrapV6Role, + wrapper: (u64, u64), + iop_vbar: (u64, u64), + missing: u32, +} + +impl G17PMailboxProvider { + + pub(super) fn get(dev: &kernel::device::Device, index: usize) -> Result { + // SAFETY: `dev` is live and apple_mbox_get() establishes a device link + // to keep the resolved provider alive for this consumer. + let mbox = unsafe { + from_err_ptr(bindings::apple_mbox_get( + dev.as_raw(), + index.try_into()?, + ))? + }; + Ok(Self { mbox }) + } + + pub(super) fn lifecycle(&self) -> Result { + let mut raw = MaybeUninit::::uninit(); + // SAFETY: `self.mbox` is the live provider returned by + // apple_mbox_get() and `raw` points to writable storage. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_lifecycle(self.mbox, raw.as_mut_ptr()) + })?; + // SAFETY: the successful C call initialized every field. + let raw = unsafe { raw.assume_init() }; + let role = match raw.role as u32 { + 1 => G17PAscwrapV6Role::Gfx, + 2 => G17PAscwrapV6Role::Gfx1, + _ => return Err(EINVAL), + }; + Ok(G17PAscwrapV6Lifecycle { + role, + wrapper: (raw.wrapper_start, raw.wrapper_size), + iop_vbar: (raw.iop_vbar_start, raw.iop_vbar_size), + missing: raw.missing, + }) + } + + pub(super) fn require_safe_cpu_lifecycle(&self) -> Result { + // SAFETY: this is a capability check on the live provider pointer. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(self.mbox) + }) + } + + pub(super) fn start_cpu(&self) -> Result { + // SAFETY: lifecycle serialization and MMIO ownership remain in C. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_start_cpu(self.mbox) }) + } + + pub(super) fn stop_cpu(&self) -> Result { + // SAFETY: lifecycle serialization and MMIO ownership remain in C. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }) + } + +} + +unsafe impl Send for G17PMailboxProvider {} + +unsafe impl Sync for G17PMailboxProvider {} + +pub(super) const fn g17p_rtkit_uat_address(dva: u64) -> u64 { + dva & G17P_RTKIT_UAT_DVA_MASK +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(super) enum G17PProcessor { + Gfx = 0, + Gfx1 = 1, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) struct G17PFirmwareCrashBuffer { + dva: u64, + size: usize, +} + +impl G17PProcessor { + + pub(super) const fn firmware_crash_buffer(self) -> G17PFirmwareCrashBuffer { + // Both endpoint-1 buffer requests are stable across the retained + // G17P captures. Message 0x0010810001e1c000 decodes to the primary + // DVA below and size 0x8000; GFX1 advertises the second DVA with the + // same size. + match self { + Self::Gfx => G17PFirmwareCrashBuffer { + dva: 0x100_01e1_c000, + size: 0x8000, + }, + Self::Gfx1 => G17PFirmwareCrashBuffer { + dva: 0x100_01f5_0000, + size: 0x8000, + }, + } + } +} + +pub(super) const fn bounded_crashlog_size(size: usize) -> usize { + if size < G17P_CRASHLOG_MAX_BYTES { + size + } else { + G17P_CRASHLOG_MAX_BYTES + } +} + +const _: () = { + assert!(bounded_crashlog_size(0) == 0); + assert!(bounded_crashlog_size(4096) == 4096); + assert!(bounded_crashlog_size(G17P_CRASHLOG_MAX_BYTES + 1) == G17P_CRASHLOG_MAX_BYTES); + let gfx = G17PProcessor::Gfx.firmware_crash_buffer(); + let gfx1 = G17PProcessor::Gfx1.firmware_crash_buffer(); + assert!(gfx.dva == 0x100_01e1_c000 && gfx1.dva == 0x100_01f5_0000); + assert!(g17p_rtkit_uat_address(gfx.dva) == 0x01e1_c000); + assert!(g17p_rtkit_uat_address(gfx1.dva) == 0x01f5_0000); + assert!(gfx.size == 0x8000 && gfx1.size == 0x8000); + assert!(gfx.dva & (mmu::UAT_PGSZ as u64 - 1) == 0); + assert!(gfx1.dva & (mmu::UAT_PGSZ as u64 - 1) == 0); + assert!(gfx.dva + gfx.size as u64 <= 1u64 << 42); + assert!(gfx1.dva + gfx1.size as u64 <= 1u64 << 42); +}; + +/// Active G17P boot does not ask Rust to back system-endpoint buffers. The +/// generic RTKit core uses its coherent-DMA fallback when these callbacks are +/// absent. +pub(super) struct G17PNoRtkitBuffer; + +impl rtkit::Buffer for G17PNoRtkitBuffer { + fn iova(&self) -> Result { + Err(ENOTSUPP) + } + + fn buf(&mut self) -> Result> { + Err(ENOTSUPP) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) struct ExpectedProvider { + pub(super) role: G17PAscwrapV6Role, + pub(super) wrapper: (u64, u64), + pub(super) iop_vbar: (u64, u64), +} + +pub(super) fn validate_provider( + provider: &G17PMailboxProvider, + expected: ExpectedProvider, +) -> Result { + let lifecycle = provider.lifecycle()?; + if lifecycle.role != expected.role + || lifecycle.wrapper != expected.wrapper + || lifecycle.iop_vbar != expected.iop_vbar + { + return Err(EINVAL); + } + + Ok(lifecycle) +} diff --git a/drivers/gpu/drm/asahi/m5/render.rs b/drivers/gpu/drm/asahi/m5/render.rs new file mode 100644 index 0000000000000..22bede0991343 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/render.rs @@ -0,0 +1,508 @@ +// SPDX-License-Identifier: MIT +//! M5/G17G graphics register construction from owned render resources. +//! +//! Common resource lowering is shared with G17P. Register order, tile packing +//! and per-stage controls follow the native 25G76 M5 reference. This codec is +//! not a claim that Linux TA/3D submission or presentation is qualified. + +use crate::m5::render_desc::{ + self, G17pRenderParameters, RegisterWrite, RenderBuildError, RenderDescriptorMetadata, + RenderDescriptorObjects, +}; + +pub(crate) const TA_REGISTER_COUNT: usize = 82; +pub(crate) const FRAGMENT_REGISTER_COUNT: usize = 90; + +/// Encode the TA parameter-manager metrics slot in the compact GART window. +/// The native 25G76 slot at 0x1000080040 produces register 1c910 = 0x80041: +/// the low bit enables the record; the remaining bits address its 64 bytes. +pub(crate) fn parameter_buffer_metrics_record( + scene_metrics: u64, +) -> Result { + let offset = scene_metrics.checked_sub(0x10_0000_0000) + .ok_or(M5RenderBuildError::OwnershipMismatch)?; + if offset >= (1u64 << 32) || offset & 0x3f != 0 { + return Err(M5RenderBuildError::OwnershipMismatch); + } + Ok(offset | 1) +} + +/// Initial M5 PB slot with fresh ring cursors. The owning job chooses the +/// hardware slot separately and retains the page-list mapping. +pub(crate) fn initial_parameter_buffer_descriptor( + page_list: u64, + page_count: u32, +) -> Result<[u32; 4], M5RenderBuildError> { + if page_list == 0 || page_list >= (1u64 << 42) || page_list & 0x7f != 0 + || page_count == 0 || page_count >= (1 << 22) { + return Err(M5RenderBuildError::OwnershipMismatch); + } + // 25G76 halLoadPBDesc, 89de844..89de8dc. The descriptor has four + // dwords; +0's low flags and both cursors start at zero. Native's + // untagged user address is translated into its PB segment encoding. + let segment = ((page_list + 0x70_0000_0000) & 0x70_0000_0000) >> 8; + Ok([((page_list >> 4) as u32) & !7, + (segment as u32 & 0xf000_0000) | page_count, 0, 0]) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5RenderBuildError { + Common(RenderBuildError), + SharedRegisterLayout, + TileCountOutOfRange, + OverlappingTileFlags, + DispatchBitAlreadySet, + InvalidContext, + OwnershipMismatch, + InvalidUscState, +} + +impl From for M5RenderBuildError { + fn from(error: RenderBuildError) -> Self { + Self::Common(error) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderStageControls { + /// Native host descriptor +0x1120 (TA) / +0x7e8 (fragment). + pub(crate) resource_flags: u64, + /// Explicit per-stage d419/d429 input, independent of the work key. + pub(crate) resource_control: u64, + /// Native +0x1128 / +0x7f0; a zero source selects the native 0x60 fallback. + pub(crate) ptc_max: u32, + pub(crate) work_key: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderControls { + pub(crate) ta: M5RenderStageControls, + pub(crate) fragment: M5RenderStageControls, + pub(crate) ta_tile_control: u64, + /// Base value for the two 1c898 writes. Bit zero is supplied by the codec. + pub(crate) ta_dispatch_control: u64, + pub(crate) fragment_raster_control: u64, + /// Non-geometry bits for 16068. Never includes tile counts, utile or layers. + pub(crate) fragment_tile_flags: u64, + /// M5 adds this second pair of store-program bindings at 153b1/153b9. + pub(crate) store_pipeline_2_bind: u64, + pub(crate) store_pipeline_2: u64, + pub(crate) partial_store_pipeline_2_bind: u64, + pub(crate) partial_store_pipeline_2: u64, + pub(crate) partial_tile_flags: u64, +} + +/// Per-submission USC private-memory accounting, separate from user status +/// registers and queue completion stamps. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderUscState { + pub(crate) control: u64, + /// AGXUMADataRec +0x50. Native allocatePagesEnd derives this from pool + /// sizes and 4-KiB allocation units; it is a byte count, not a GPU VA. + pub(crate) required_bytes: u64, + /// Capacity actually backed by this job's retained USC allocation pages. + pub(crate) allocated_bytes: u64, + /// A 64-byte slot in the firmware-visible statistics ring. + pub(crate) statistics: u64, +} + +impl M5RenderUscState { + fn valid(self) -> bool { + let firmware_range = |address: u64, size: u64| { + address >= 0xffff_fc00_0000_0000 + && address & 0x3f == 0 + && address.checked_add(size).is_some() + }; + firmware_range(self.control, 0x80) + && firmware_range(self.statistics, 0x40) + && self.required_bytes != 0 + && self.required_bytes <= self.allocated_bytes + && self.allocated_bytes & 0xfff == 0 + } +} + +/// Fill the native USC fields only after the caller owns the pool and the +/// statistics slot. Other runtime fields still need queue/link publication. +/// Validate both stages before modifying either descriptor. +pub(crate) fn apply_usc_submission_state( + ta: &mut [u8], + fragment: &mut [u8], + ta_state: M5RenderUscState, + fragment_state: M5RenderUscState, +) -> Result<(), M5RenderBuildError> { + if ta.len() < render_desc::TA_DESCRIPTOR_SIZE + || fragment.len() < render_desc::FRAGMENT_DESCRIPTOR_SIZE + { + return Err(RenderBuildError::BufferTooSmall.into()); + } + if !ta_state.valid() || !fragment_state.valid() { + return Err(M5RenderBuildError::InvalidUscState); + } + // 25G76 AGXTAChannelSKSM::submitBuffer: 8a244c4..8a244ec. + // AGX3DChannelSKSM::submitBuffer takes the corresponding value from + // host +0xba0 = AGXUMADataRec +0x50 at 895b70c..895b718. + for (raw, state, control, required, statistics) in [ + (ta, ta_state, 0x934, 0x93d, 0x945), + (fragment, fragment_state, 0x21ce, 0x21d7, 0x21df), + ] { + put_u64(raw, control, state.control); + put_u64(raw, required, state.required_bytes); + put_u64(raw, statistics, state.statistics); + } + Ok(()) +} + +const EMPTY_REGISTER: RegisterWrite = RegisterWrite { + number: 0, + value: 0, +}; + +struct RegisterList { + values: [RegisterWrite; N], + count: usize, +} + +impl RegisterList { + fn new() -> Self { + Self { + values: [EMPTY_REGISTER; N], + count: 0, + } + } + + fn push(&mut self, number: u32, value: u64) -> Result<(), M5RenderBuildError> { + if self.count == N { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + self.values[self.count] = RegisterWrite { number, value }; + self.count += 1; + Ok(()) + } + + fn finish(self) -> Result<[RegisterWrite; N], M5RenderBuildError> { + if self.count != N { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + Ok(self.values) + } +} + +fn value(registers: &[RegisterWrite], number: u32) -> Result { + registers + .iter() + .find(|r| r.number == number) + .map(|r| r.value) + .ok_or(M5RenderBuildError::SharedRegisterLayout) +} + +const fn ptc_control(maximum: u32) -> u64 { + let maximum = if maximum == 0 { 0x60 } else { maximum }; + ((maximum as u64) << 32) | 0x400020 +} + +fn tile_state(p: &G17pRenderParameters, flags: u64) -> Result { + let tiles_x = (p.width as u64 + 31) / 32; + let tiles_y = (p.height as u64 + 31) / 32; + if !(1..=1024).contains(&tiles_x) || !(1..=1024).contains(&tiles_y) { + return Err(M5RenderBuildError::TileCountOutOfRange); + } + // 25G76 fragment generator: bfi at 89dcb98, 89dcbb8 and 89dcbbc. + let geometry_mask = (0xfffff_u64 << 42) | (0xf_u64 << 38) | (1_u64 << 32); + if flags & geometry_mask != 0 { + return Err(M5RenderBuildError::OverlappingTileFlags); + } + Ok(flags + | ((tiles_x - 1) << 42) + | ((tiles_y - 1) << 52) + | ((p.utile_config & 0xf000) << 26) + | if p.layers > 1 { 1_u64 << 32 } else { 0 }) +} + +fn tile_blocks(pixels: u32) -> u64 { + let tiles = (pixels as u64 + 31) / 32; + let first = (((tiles + 3) / 4) + 3) & !3; + // Native TA host +1208/+120c supplies 0x40200c for the reference: + // boundaries 12, 8, 4 in ten-bit fields, versus G17P's nine-bit fields. + (3 * first) | ((2 * first) << 10) | (first << 20) +} + +pub(crate) fn build_ta_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], M5RenderBuildError> { + // M5's captured register program addresses heap metadata directly. + // G17P's layered path now uses a separate layer table in these slots. + let mut common_parameters = *p; + common_parameters.layermeta = p.heapmeta; + let common = render_desc::build_ta_registers(&common_parameters)?; + if controls.ta_dispatch_control & 1 != 0 { + return Err(M5RenderBuildError::DispatchBitAlreadySet); + } + // Validate the M5 tile fields even when only constructing the TA half. + tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterList::new(); + for mut register in common { + match register.number { + 0x0a309 | 0x0a5a1 | 0x0d419 => continue, + 0x1c8f8 => { + out.push(0x0d499, controls.ta.resource_flags)?; + out.push(0x0d419, controls.ta.resource_control)?; + out.push(0x0a5a1, ptc_control(controls.ta.ptc_max))?; + register.value = controls.ta_tile_control; + } + 0x10171 => { + for (number, source) in [(0x1ca50, 0x101b9), (0x1ca58, 0x10149), (0x1ca60, 0x10141)] + { + out.push(number, value(&common, source)?)?; + } + out.push(0x1ca70, tile_blocks(p.width))?; + out.push(0x1ca78, tile_blocks(p.height))?; + out.push(0x1ca68, value(&common, 0x1c081)?)?; + } + 0x1c069 => register.value = tile_blocks(p.width), + 0x1c071 => register.value = tile_blocks(p.height), + 0x1c898 => { + if register.value > 1 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + register.value |= controls.ta_dispatch_control; + } + 0x10799 => out.push(0x1a149, 0)?, + 0x10209 => { + out.push(0x0a739, 0)?; + register.value = controls.ta.work_key; + } + 0x1c9f0 | 0x14320 => register.value = controls.ta.work_key, + 0x14308 => register.value = p.usc_flist_hardware_buffer_id as i32 as i64 as u64, + _ => {} + } + out.push(register.number, register.value)?; + if register.number == 0x01740 { + out.push(0x0a739, 0)?; + } + } + out.finish() +} + +pub(crate) fn build_fragment_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], M5RenderBuildError> { + // M5's captured register program addresses heap metadata directly. + // G17P's layered path now uses a separate layer table in these slots. + let mut common_parameters = *p; + common_parameters.layermeta = p.heapmeta; + let common = render_desc::build_fragment_registers(&common_parameters)?; + let geometry = tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterList::new(); + let mut dispatch_writes = 0; + let mut tile_writes = 0; + for mut register in common { + match register.number { + 0x0a301 | 0x15109 | 0x15101 | 0x160a8 | 0x0a5a9 | 0x0d429 => continue, + 0x100b1 => register.number = 0x16469, + 0x100d9 => register.number = 0x16471, + 0x100b8 => register.number = 0x16138, + 0x10791 => register.value = controls.fragment_raster_control, + 0x15321 => { + out.push(0x0d491, controls.fragment.resource_flags)?; + out.push(0x0d429, controls.fragment.resource_control)?; + out.push(0x0a5a9, ptc_control(controls.fragment.ptc_max))?; + } + 0x16068 => { + tile_writes += 1; + if tile_writes == 1 { + register.value = geometry; + } + } + 0x16020 => { + dispatch_writes += 1; + if dispatch_writes == 3 { + if register.value != 0 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + continue; + } + } + 0x160e0 => out.push(0x1a151, 0)?, + 0x10211 => { + out.push(0x0a741, 0)?; + register.value = controls.fragment.work_key; + } + 0x10420 => register.value = controls.fragment.work_key, + 0x14048 => register.value = p.usc_flist_hardware_buffer_id as i32 as i64 as u64, + _ => {} + } + out.push(register.number, register.value)?; + if register.number == 0x15381 { + out.push(0x153b1, controls.store_pipeline_2_bind)?; + out.push(0x153b9, controls.store_pipeline_2)?; + } + if register.number == 0x01731 { + out.push(0x0a741, 0)?; + } + } + if dispatch_writes != 3 || tile_writes != 2 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + out.finish() +} + +fn partial_registers( + common: &[RegisterWrite], + insert_after: u32, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; N], M5RenderBuildError> { + let mut out = RegisterList::new(); + for register in common { + let bits = if register.number == 0x10039 { + controls.partial_tile_flags + } else { + 0 + }; + out.push(register.number, register.value | bits)?; + if register.number == insert_after { + out.push(0x153b1, controls.partial_store_pipeline_2_bind)?; + out.push(0x153b9, controls.partial_store_pipeline_2)?; + } + } + out.finish() +} + +pub(crate) fn build_partial_store_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 18], M5RenderBuildError> { + partial_registers(&render_desc::partial_store_registers(p), 0x15381, controls) +} + +pub(crate) fn build_partial_resume_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 25], M5RenderBuildError> { + partial_registers(&render_desc::partial_resume_registers(p), 0x15381, controls) +} + +pub(crate) fn build_partial_load_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 12], M5RenderBuildError> { + partial_registers(&render_desc::partial_load_registers(p), 0x15309, controls) +} + +fn work_key( + p: &G17pRenderParameters, + metadata: RenderDescriptorMetadata, +) -> Result { + if metadata.context_id == 0 || metadata.context_id > 255 { + return Err(M5RenderBuildError::InvalidContext); + } + if metadata.ta_hardware_buffer_id != p.ta_hardware_buffer_id { + return Err(M5RenderBuildError::OwnershipMismatch); + } + metadata + .submission_ordinal + .checked_add(metadata.submission_ordinal / 2) + .ok_or(M5RenderBuildError::Common( + RenderBuildError::ArithmeticOverflow, + ))?; + Ok(((metadata.context_id as u64) << 8) | (metadata.command_buffer_state_word & 0x3f) as u64) +} + +fn alias_offset(alias: u64, offset: u64) -> Result { + alias.checked_add(offset).ok_or(M5RenderBuildError::Common( + RenderBuildError::ArithmeticOverflow, + )) +} + +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Construct the M5 TA record's common layout and all register programs. +/// Runtime ownership/completion fields must be populated before publication. +#[inline(never)] +pub(crate) fn build_ta_descriptor_base( + p: &G17pRenderParameters, + controls: &M5RenderControls, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + low_alias: u64, + raw: &mut [u8], +) -> Result<(), M5RenderBuildError> { + let mut controls = *controls; + controls.ta.work_key = work_key(p, metadata)?; + let program = alias_offset(low_alias, 0x60)?; + alias_offset(low_alias, render_desc::TA_DESCRIPTOR_SIZE as u64)?; + let registers = build_ta_registers(p, &controls)?; + render_desc::build_ta_descriptor_base(p, objects, metadata, ®isters, raw)?; + // Native boundaries: main work ends before the final a739/deflake/dispatch. + for (offset, count) in [ + (0x38, TA_REGISTER_COUNT - 3), + (0x3a, TA_REGISTER_COUNT), + (0x3c, TA_REGISTER_COUNT), + ] { + put_u16(raw, offset, count as u16); + } + put_u64(raw, 0x760, program); + Ok(()) +} + +/// Construct the M5 fragment record, including all three partial programs. +/// The shared encoder is pure and does not apply Neo diagnostic overrides. +#[inline(never)] +pub(crate) fn build_fragment_descriptor_base( + p: &G17pRenderParameters, + controls: &M5RenderControls, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + low_alias: u64, + raw: &mut [u8], +) -> Result<(), M5RenderBuildError> { + let mut controls = *controls; + controls.fragment.work_key = work_key(p, metadata)?; + alias_offset(low_alias, render_desc::FRAGMENT_DESCRIPTOR_SIZE as u64)?; + let main = build_fragment_registers(p, &controls)?; + let store = build_partial_store_registers(p, &controls)?; + let resume = build_partial_resume_registers(p, &controls)?; + let load = build_partial_load_registers(p, &controls)?; + render_desc::build_fragment_descriptor_base( + p, + objects, + metadata, + render_desc::G17pFragmentRegisters { + main: &main, + store: &store, + resume: &resume, + load: &load, + }, + raw, + )?; + put_u32(raw, 0x50, 1); + put_u32(raw, 0x90, 1); + for (offset, count) in [ + (0x80, FRAGMENT_REGISTER_COUNT - 3), + (0x82, FRAGMENT_REGISTER_COUNT - 1), + (0x84, FRAGMENT_REGISTER_COUNT - 1), + (0x88, FRAGMENT_REGISTER_COUNT), + ] { + put_u16(raw, offset, count as u16); + } + for (offset, target) in [ + (0x7a0, 0xa0), + (0xec0, 0x7c0), + (0x15e0, 0xee0), + (0x1d00, 0x1600), + ] { + put_u64(raw, offset, alias_offset(low_alias, target)?); + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/m5/render_desc.rs b/drivers/gpu/drm/asahi/m5/render_desc.rs new file mode 100644 index 0000000000000..8e73ee27c5749 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/render_desc.rs @@ -0,0 +1,1129 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +const TA_REGISTER_COUNT: usize = 73; + +const FRAGMENT_REGISTER_COUNT: usize = 89; + +/// Both embedded register lists match the kext EXACTLY -- same writes, same +/// unique registers, same order -- verified against 2048-byte captures of the +/// native TA and 3D command records (macos-trace recs2048-gfx.txt): +/// +/// TA 73 writes, 71 unique, list ends at +0x3cc +/// 3D 89 writes, 86 unique, list ends at +0x4cc +/// +/// A 1024-byte capture cuts the 3D list at exactly +0x400 and hides its last +/// 17 writes, which briefly looked like 14 registers "Metal never writes" -- +/// they were entries 72..85. Assert the counts so a future truncated capture +/// cannot produce that conclusion again. +const _: () = assert!(TA_REGISTER_COUNT == 73); + +const _: () = assert!(FRAGMENT_REGISTER_COUNT == 89); + +const PARTIAL_STORE_REGISTER_COUNT: usize = 16; + +const PARTIAL_RESUME_REGISTER_COUNT: usize = 23; + +const PARTIAL_LOAD_REGISTER_COUNT: usize = 10; + +const REGISTER_BYTES: usize = 12; + +pub(crate) const TA_DESCRIPTOR_SIZE: usize = 0x9c0; + +pub(crate) const FRAGMENT_DESCRIPTOR_SIZE: usize = 0x2240; + +/// Native render-setup field copied through command descriptor +0xc00 to the +/// final 3D command tail at +0x2110. This is independent of SGX's GPC +/// performance-state map register. +const G17P_FRAGMENT_TAIL_SETUP_WORD: u32 = u32::MAX; + +/// J700 shared-runtime flag snapshot copied by native 3D submit into type-1 +/// command lanes +0x2124/+0x2128. +/// +/// The kext resolves the accelerator's shared event-stamp/HWDSID internal +/// resource at +0x1a130, obtains its CPU mapping, reads byte bit 0 at mapping +/// +0x2ad/+0x2ae, and widens them before per-draw register generation. The +/// working Neo Metal snapshot contains 1/1. +const G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT: [u32; 2] = [1, 1]; + +pub(crate) const G17P_EOT_RESOURCE_SPEC: u64 = 0x8; + +pub(crate) const G17P_LOAD_RESOURCE_SPEC: u64 = 0x0007_8000_0000_0040; + +pub(crate) const G17P_SOURCE_BG_PIPELINE: u64 = 0x0000_0100_0199_0240; + +pub(crate) const G17P_SOURCE_EOT_PIPELINE: u64 = 0x0000_0100_0199_0640; + +const TILE_WIDTH: u64 = 32; + +const TILE_HEIGHT: u64 = 32; + +const MACRO_TILES_X: u64 = 4; + +const MACRO_TILES_Y: u64 = 4; + +const REGION_ENTRY_SIZE: u64 = 5; + +/// Base added by G17P's compact VDM control-stream register (0x1c880). +/// The register retains the legacy 40-bit {TTBR selector, 39-bit offset} +/// shape, so lower-root user addresses are encoded relative to this base. +const G17P_USER_BASE: u64 = 0x10_0000_0000; + +const G17P_VDM_USER_WINDOW_SIZE: u64 = 1 << 39; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RenderBuildError { + ZeroDimensions, + LayerCountOutOfRange, + UtileDimensions, + AddressBeforeContext, + UserAddressOutOfRange, + VdmAddressOutOfRange, + BufferTooSmall, + ArithmeticOverflow, +} + +impl RenderBuildError { +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWrite { + pub(crate) number: u32, + pub(crate) value: u64, +} + +impl RegisterWrite { + const fn new(number: u32, value: u64) -> Self { + Self { number, value } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17pRenderParameters { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) context_base: u64, + /// Allocation epoch of the trusted execution-context lease, shared by TA/3D. + pub(crate) context_generation: u8, + pub(crate) tilemap: u64, + /// Per-layer tiler metadata, which native places between the tilemap and + /// the heap metadata. It is empty, and so equal to `heapmeta`, for a + /// single-layer target. + pub(crate) layermeta: u64, + pub(crate) heapmeta: u64, + pub(crate) tpc: u64, + /// Geometry size of the owned TPC subregion, before capacity page rounding. + pub(crate) tpc_size: u64, + pub(crate) deflake_1: u64, + pub(crate) deflake_2: u64, + pub(crate) deflake_3: u64, + pub(crate) encoder: u64, + pub(crate) ta_status: u64, + pub(crate) fragment_status: u64, + pub(crate) store_pipeline_bind: u64, + pub(crate) store_pipeline: u64, + pub(crate) load_pipeline_bind: u64, + pub(crate) load_pipeline: u64, + pub(crate) partial_store_pipeline_bind: u64, + pub(crate) partial_store_pipeline: u64, + pub(crate) partial_load_pipeline_bind: u64, + pub(crate) partial_load_pipeline: u64, + pub(crate) scissor_array: u64, + pub(crate) depth_bias_array: u64, + pub(crate) aux_fb: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth_buffer: u64, + pub(crate) stencil_buffer: u64, + pub(crate) depth_aux_buffer: u64, + pub(crate) stencil_aux_buffer: u64, + pub(crate) depth_stride: u64, + pub(crate) stencil_stride: u64, + pub(crate) depth_aux_stride: u64, + pub(crate) stencil_aux_stride: u64, + pub(crate) depth_clear_value_bits: u32, + pub(crate) stencil_clear_value: u32, + pub(crate) depth_flags: u64, + /// ZLS control used by every firmware partial-store/load/resume phase. + /// Unlike the pass-boundary control, an attached depth/stencil surface + /// must be both loaded and stored across an internal split. + pub(crate) partial_depth_flags: u64, + pub(crate) depth_dimensions: u64, + pub(crate) merge_upper_x_bits: u32, + pub(crate) merge_upper_y_bits: u32, + pub(crate) layers: u16, + /// Utile dimensions in pixels, including the sample-layout adjustment. + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) utile_config: u64, + pub(crate) multisample_control: u64, + pub(crate) ppp_control: u64, + pub(crate) tib_blocks: u64, + pub(crate) tile_config: u64, + pub(crate) aux_fb_flags: u64, + pub(crate) aux_fb_page_count: u64, + pub(crate) cycle: u64, + pub(crate) record_index: u64, + /// TA event correlation written to 0x1ca10/0x014a1/0x0a349. + /// Native sources this independently from the fragment correlation. + pub(crate) lifecycle: u64, + /// Fragment event correlation written to 0x160e0/0x01499/0x0a341. + pub(crate) fragment_lifecycle: u64, + pub(crate) sampler_array: u64, + pub(crate) sampler_count: u32, + pub(crate) process_empty_tiles: bool, + pub(crate) ta_timestamp_start: u64, + pub(crate) ta_timestamp_end: u64, + pub(crate) fragment_timestamp_start: u64, + pub(crate) fragment_timestamp_end: u64, + /// Write the VDM control-stream register (0x1c880) as a full GPU virtual + /// address instead of the native user-process compact form. + /// + /// SCOPE CORRECTED against the macOS oracle. This flag once covered every + /// register `ta_offset` feeds, justified from the legacy G14X path where + /// all of them are whole pointers. A real Metal render pass on THIS chip + /// says otherwise: the captured TA command record writes tilemap, TPC and + /// heapmeta as small context-relative offsets (see `ta_offset`). The legacy + /// encoding does not transfer to G17P, so those six registers are always + /// relative now and this flag applies to the VDM stream alone. Native first + /// lowers the task VA to a task-relative offset, maps the same backing at + /// 0x1000000000 + offset in the GART window, and writes that offset here. + /// Linux userspace supplies the full GART-alias VA; this kernel subtracts + /// the architectural 0x1000000000 base. Hardware adds it back for compact + /// pointers whose bit 39 is clear. + pub(crate) absolute_pointers: bool, + /// + /// Legacy diagnostic OR-mask for the BG resource-specifier high 17 bits. + /// Native calls this field PassControlMaskType and derives it from shader + /// outputs, including extension of the highest active pass bit; it is not + /// an address class or a universal constant. Main and partial shaders can + /// legitimately differ. Default zero preserves the userspace mask; + /// a nonzero value is an explicit diagnostic override only. + pub(crate) bg_resource_prefix: u64, + /// Historical opt-in diagnostic changing byte 2 of the USC private-memory + /// budget qword at TA +0x93d. It must not override +0x94d: that byte is the + /// allocation generation of the execution context, not a PM mode flag. + /// The captured budget is not yet derived from our pool allocation. + pub(crate) native_pm_bytes: bool, + /// Hardware-buffer ID allocated for this TA command. Native allocates it + /// from the accelerator's 0x7f-entry `AGXHardwareBufferIDManager`, stores + /// it in the host-side `AGXTACommandDescriptor` at +0x1248/+0x928, and the + /// register generator emits that value as register 0x1c830. Those two host + /// offsets are not fields in this final 0x9c0-byte firmware command. + pub(crate) ta_hardware_buffer_id: u32, + /// HardwareBufferID of the retained USC private-memory FList shared by TA + /// and 3D. Native allocates this from the USC manager (accelerator + /// +0x133c8), independently of the TA parameter-buffer manager above, and + /// emits the same ID in TA register 0x14308 and fragment register 0x14048. + pub(crate) usc_flist_hardware_buffer_id: u32, + /// Diagnostic capture-value bitmask retained for fields whose proper + /// inputs are not modeled yet. In particular, the 0xa5a* values are + /// assembled from workload-derived USC pool limits. Their captured values + /// are not architectural literals and must not become defaults. + pub(crate) native_ta_registers: u32, + /// Retained GPU-ID diagnostic snapshot, not the fragment parameter-buffer + /// owner. The old 0x1c838 use confused host descriptor +0x928 with a + /// different object's transient raw-command +0x928 field. + pub(crate) gpc_perf_state_map: u32, + /// Low nibble of SGX 0xe01480, copied by native 3D submit to tail +0x210c. + pub(crate) gpc_perf_state_map_low: u32, + /// Bit 0 of SGX 0xe0141c, copied by native 3D submit to tail +0x2104. + pub(crate) gpc_perf_state_control: u32, + /// GPU address of the selected native 0x80 Scene-state entry, or 0 for + /// none. The command-record binding is carried by `RenderDescriptorObjects` + /// in both TA and 3D; this copy is retained for diagnostics. + pub(crate) parameter_buffer: u64, +} + +impl G17pRenderParameters { + + pub(crate) const fn source_rsrc8() -> Self { + Self { + width: 64, + height: 64, + context_base: 0x0000_0010_0000_0000, + context_generation: 1, + tilemap: 0x0000_0010_001b_0000, + layermeta: 0x0000_0010_001b_5000, + heapmeta: 0x0000_0010_001b_5000, + tpc: 0x0000_0010_0024_0000, + tpc_size: 0x800, + deflake_1: 0x0000_0010_0006_82a0, + deflake_2: 0x0000_0010_0006_8020, + deflake_3: 0x0000_0010_0006_8000, + encoder: 0x0000_0010_0001_8000, + ta_status: 0x0000_0010_0007_8000, + fragment_status: 0x0000_0010_001a_8000, + store_pipeline_bind: G17P_EOT_RESOURCE_SPEC, + store_pipeline: G17P_SOURCE_EOT_PIPELINE, + load_pipeline_bind: G17P_LOAD_RESOURCE_SPEC, + load_pipeline: G17P_SOURCE_BG_PIPELINE, + partial_store_pipeline_bind: 0, + partial_store_pipeline: 0, + partial_load_pipeline_bind: 0, + partial_load_pipeline: 0, + scissor_array: 0x0000_0100_019a_0000, + depth_bias_array: 0x0000_0100_01af_8000, + aux_fb: 0x0000_0100_01aa_8000, + occlusion_query_base: 0, + depth_buffer: 0, + stencil_buffer: 0, + depth_aux_buffer: 0, + stencil_aux_buffer: 0, + depth_stride: 0, + stencil_stride: 0, + depth_aux_stride: 0, + stencil_aux_stride: 0, + depth_clear_value_bits: 0x3f80_0000, + stencil_clear_value: 0, + depth_flags: 0, + partial_depth_flags: 0, + depth_dimensions: 0, + merge_upper_x_bits: 0x3cdd_b3d9, + merge_upper_y_bits: 0x3cdd_b3d9, + layers: 1, + utile_width: 32, + utile_height: 32, + utile_config: 0xa000, + multisample_control: 0x88, + ppp_control: 0x202, + tib_blocks: 8, + tile_config: 0x10280, + aux_fb_flags: 0xc001, + aux_fb_page_count: 0x10_0000, + cycle: 0x17_8020, + record_index: 0x8_0005, + lifecycle: 0, + fragment_lifecycle: 0, + sampler_array: 0, + sampler_count: 0, + process_empty_tiles: true, + ta_timestamp_start: 0xffff_fc20_0002_4c68, + ta_timestamp_end: 0, + fragment_timestamp_start: 0xffff_fc20_0002_4c68, + fragment_timestamp_end: 0xffff_fc20_0002_4c70, + absolute_pointers: false, + bg_resource_prefix: 0, + native_pm_bytes: false, + ta_hardware_buffer_id: 0, + usc_flist_hardware_buffer_id: 0, + native_ta_registers: 0, + gpc_perf_state_map: 0, + gpc_perf_state_map_low: 0, + gpc_perf_state_control: 0, + parameter_buffer: 0, + } + } +} + +#[derive(Debug, Copy, Clone)] +struct Geometry { + tiles_x: u64, + tiles_y: u64, + size1: u64, + size2: u64, + size3: u64, + x_blocks: u64, + y_blocks: u64, + screen: u64, + pixels: u64, + macro_size: u64, +} + +const fn align(value: u64, alignment: u64) -> u64 { + (value + alignment - 1) & !(alignment - 1) +} + +fn geometry(parameters: &G17pRenderParameters) -> Result { + if parameters.width == 0 || parameters.height == 0 { + return Err(RenderBuildError::ZeroDimensions); + } + if parameters.layers == 0 || parameters.layers > 2048 { + return Err(RenderBuildError::LayerCountOutOfRange); + } + + let width = parameters.width as u64; + let height = parameters.height as u64; + let tiles_x = (width + TILE_WIDTH - 1) / TILE_WIDTH; + let tiles_y = (height + TILE_HEIGHT - 1) / TILE_HEIGHT; + let (utile_width, utile_height) = (parameters.utile_width, parameters.utile_height); + if !matches!((utile_width, utile_height), (32, 32) | (32, 16) | (16, 16)) { + return Err(RenderBuildError::UtileDimensions); + } + let utiles_per_tile_x = TILE_WIDTH / utile_width as u64; + let utiles_per_tile_y = TILE_HEIGHT / utile_height as u64; + // The UAPI's utile height already halves at four samples. Use the actual + // dimensions once; a second sample-count factor overallocates tile lists + // and publishes a macro size that disagrees with the native register set. + let utiles_per_tile = utiles_per_tile_x * utiles_per_tile_y; + let mtile_x1 = align((tiles_x + MACRO_TILES_X - 1) / MACRO_TILES_X, 4); + let mtile_y1 = align((tiles_y + MACRO_TILES_Y - 1) / MACRO_TILES_Y, 4); + let mtile_x2 = 2 * mtile_x1; + let mtile_x3 = 3 * mtile_x1; + let mtile_y2 = 2 * mtile_y1; + let mtile_y3 = 3 * mtile_y1; + let mtile_stride = mtile_x1 * mtile_y1; + let size1 = (REGION_ENTRY_SIZE * mtile_stride * utiles_per_tile + 3) / 4; + + Ok(Geometry { + tiles_x, + tiles_y, + size1, + size2: mtile_stride, + size3: 2 * mtile_stride * utiles_per_tile, + x_blocks: mtile_x3 | (mtile_x2 << 9) | (mtile_x1 << 18), + y_blocks: mtile_y3 | (mtile_y2 << 9) | (mtile_y1 << 18), + screen: ((tiles_y - 1) << 12) | (tiles_x - 1), + pixels: (width - 1) | ((height - 1) << 16), + macro_size: mtile_y1 * utiles_per_tile_y | ((mtile_x1 * utiles_per_tile_x) << 16), + }) +} + +/// Context-relative offset. ALWAYS relative -- a real Metal render pass on this +/// chip writes small offsets into these registers, not addresses. +/// +/// Oracle: the TA command record captured from `mtlg.m` (macos-trace +/// `recs-gfx.txt`, dumped by `recs.d` at `requestChannelCommandTA`) carries an +/// embedded register list that decodes field-for-field against this function's +/// output -- same registers, same order, 12-byte entries of u32 number plus u64 +/// value -- with these values: +/// +/// 0x01748 = 1 (matches) +/// 0x10141 = 0x200 (matches) +/// 0x1c039 = 0x0119ba00 tilemap <- OFFSET, not a VA +/// 0x1c9c8 = 0x0119ba00 tilemap +/// 0x1c0a1 = 0x01290000 tpc <- OFFSET +/// 0x1c031 = 0x800000000119ca00 heapmeta | bit63 (matches the form) +/// 0x1c9c0 = 0x800000000119ca00 heapmeta +/// 0x1c051 = 0x003a0012006b0003 (matches the constant exactly) +/// +/// Writing full virtual addresses here -- which is what the legacy G14X path +/// does, and what an earlier version of this driver was changed to do -- points +/// the tiler's tilemap, TPC and heapmeta at values the firmware cannot use. +fn ta_resource_offset( + parameters: &G17pRenderParameters, + address: u64, +) -> Result { + address + .checked_sub(parameters.context_base) + .ok_or(RenderBuildError::AddressBeforeContext) +} + +/// Compact user/GART address consumed by the TA register and descriptor +/// fields. Native maps these objects in the architectural 0x1000000000 window +/// and publishes the offset, not the full address. Descriptor +0x7d6 is only +/// 32 bits wide, so reject aliases that cannot be represented consistently. +fn ta_user_offset(address: u64) -> Result { + let offset = address + .checked_sub(G17P_USER_BASE) + .ok_or(RenderBuildError::UserAddressOutOfRange)?; + + if offset > u32::MAX as u64 { + return Err(RenderBuildError::UserAddressOutOfRange); + } + + Ok(offset) +} + +fn vdm_user_offset(address: u64) -> Result { + let offset = address + .checked_sub(G17P_USER_BASE) + .ok_or(RenderBuildError::VdmAddressOutOfRange)?; + + if offset >= G17P_VDM_USER_WINDOW_SIZE { + return Err(RenderBuildError::VdmAddressOutOfRange); + } + + Ok(offset) +} + +pub(crate) fn build_ta_registers( + p: &G17pRenderParameters, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], RenderBuildError> { + build_ta_registers_for_work_key(p, render_work_key(1, 0)) +} + +fn build_ta_registers_for_work_key( + p: &G17pRenderParameters, + work_key: u64, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], RenderBuildError> { + let g = geometry(p)?; + let tilemap = ta_resource_offset(p, p.tilemap)?; + let layermeta = ta_resource_offset(p, p.layermeta)?; + let heapmeta = ta_resource_offset(p, p.heapmeta)?; + let tpc = ta_resource_offset(p, p.tpc)?; + let deflake_1 = ta_user_offset(p.deflake_1)?; + let deflake_2 = ta_user_offset(p.deflake_2)?; + let deflake_3 = ta_user_offset(p.deflake_3)?; + // The VDM control stream is the one pointer the 192-byte capture window + // does not reach (register 0x1c880 sits past the end of the dump), and it + // is the one that is not relative to the driver-private `context_base`. + // macOS first lowers the task VA to a task-relative offset, then maps the + // backing at 0x1000000000 + offset in the GART window. Linux userspace gives + // us that full alias VA, so this hardware encoder subtracts the GART base. + // Bit 39 selects the upper UAT root; refuse an offset that would silently + // change roots. Keep the full-address escape hatch diagnostic-only. + let encoder = if p.absolute_pointers { + p.encoder + } else { + vdm_user_offset(p.encoder)? + }; + // generateTARegisterListG16Common: the layer count less one with bit 15, + // plus bit 14 and a process-empty-tiles select for a layered target. The + // pipe mirror 0x1c9e8 keeps only the layered bit and the count. + let mut layer_mode = 0x8000 | (p.layers as u64 - 1); + if p.layers > 1 { + layer_mode |= 0x4000 | if p.process_empty_tiles { 0x2000 } else { 0x1000 }; + } + + Ok([ + RegisterWrite::new(0x01748, 1), + // Native N=1 uses 0x200 and N=2 uses 0x0. The one-layer source profile + // selects the N=1 value, matching descriptor byte +0x789 above. + RegisterWrite::new(0x10141, 0x200), + RegisterWrite::new(0x1c039, tilemap), + RegisterWrite::new(0x1c9c8, tilemap), + RegisterWrite::new(0x1c0a1, tpc), + RegisterWrite::new(0x1c031, heapmeta | 0x8000_0000_0000_0000), + RegisterWrite::new(0x1c9c0, heapmeta | 0x8000_0000_0000_0000), + RegisterWrite::new(0x1c051, 0x003a_0012_006b_0003), + RegisterWrite::new(0x1c061, 1), + RegisterWrite::new(0x10149, p.utile_config), + RegisterWrite::new(0x10139, p.multisample_control), + RegisterWrite::new(0x10111, deflake_1), + RegisterWrite::new(0x1c9b0, deflake_1), + RegisterWrite::new(0x10119, deflake_2), + RegisterWrite::new(0x1c9b8, deflake_2), + RegisterWrite::new(0x1c958, 1), + RegisterWrite::new(0x1c950, deflake_3 | 0x0004_0000_0000_0000), + RegisterWrite::new(0x1c930, 0), + RegisterWrite::new(0x1c880, encoder), + RegisterWrite::new(0x1c079, layermeta), + RegisterWrite::new(0x1c9d8, layermeta), + RegisterWrite::new(0x10151, 0), + RegisterWrite::new(0x1c199, 0), + RegisterWrite::new(0x1c1a1, 0), + RegisterWrite::new(0x1c1a9, 0), + RegisterWrite::new(0x1c1b1, 0), + RegisterWrite::new(0x1c1b9, 0), + RegisterWrite::new(0x1c8f8, 0x8860), + RegisterWrite::new(0x1c0b1, g.size1), + RegisterWrite::new(0x1c850, g.size1), + // Silicon holds 0x10131 at 0x88 at ONE, TWO and FOUR samples while + // 0x10139 carries the real sample position table (all five captures in + // logs/m5-msaa-native-hv-20260915-101837). Driving this from + // multisample_control would write the position table here, which + // Apple never does. + RegisterWrite::new(0x10131, 0x88), + RegisterWrite::new(0x10121, p.ppp_control), + RegisterWrite::new(0x10129, g.pixels), + RegisterWrite::new(0x101b9, g.screen), + RegisterWrite::new(0x1c069, g.x_blocks), + RegisterWrite::new(0x1c071, g.y_blocks), + RegisterWrite::new(0x1c081, g.size2), + RegisterWrite::new(0x1c0a9, g.size3), + RegisterWrite::new(0x10171, 0x100), + RegisterWrite::new(0x10169, layer_mode), + RegisterWrite::new(0x0a309, 0), + RegisterWrite::new(0x1c8e0, u64::MAX), + RegisterWrite::new(0x1c8e8, u64::MAX), + RegisterWrite::new(0x1c898, 0), + RegisterWrite::new(0x101e1, 0x1c), + RegisterWrite::new(0x1c9e8, layer_mode & 0x47ff), + RegisterWrite::new(0x1a099, 0), + RegisterWrite::new(0x1a0a1, 0), + RegisterWrite::new(0x1a069, 0), + RegisterWrite::new(0x1a071, 0), + RegisterWrite::new(0x1a0c9, 0), + RegisterWrite::new(0x1a0d1, 0), + RegisterWrite::new(0x101c9, 0), + RegisterWrite::new(0x0d471, 0), + RegisterWrite::new(0x1a0f1, 8), + RegisterWrite::new(0x10799, 0xff_0000), + // Native sources this from AGXHardwareBufferIDManager. It is neither a + // PBDesc ring index nor a capture-stable literal. + RegisterWrite::new(0x1c830, p.ta_hardware_buffer_id as u64), + RegisterWrite::new(0x1ca30, p.cycle), + RegisterWrite::new(0x16c39, p.cycle), + RegisterWrite::new(0x1c910, p.record_index), + // Native derives the high word from the TA AGXUMADataRec PTC/DUPM + // maximum at host descriptor +0x1488, copied through command +0x1158. + // halUpdatePTCMaxCount computes max(0x60, min(pool_input / active_kicks, + // 0x200)); Linux does not yet carry that pool input. Follow the kext's + // exact zero-source fallback 0x60_0040_0020. Bit 1 retains one working + // Metal trace's workload-derived 0xfe value as a diagnostic only. + RegisterWrite::new( + 0x0a5a1, + if p.native_ta_registers & 2 != 0 { + 0x0000_00fe_0040_0020 + } else { + 0x0000_0060_0040_0020 + }, + ), + RegisterWrite::new(0x0d419, 0x0000_0002_0000_0001), + RegisterWrite::new(0x1ca10, p.lifecycle), + RegisterWrite::new(0x014a1, p.lifecycle), + RegisterWrite::new(0x0a349, p.lifecycle), + RegisterWrite::new(0x10209, work_key), + RegisterWrite::new(0x1c9f0, work_key), + RegisterWrite::new(0x14320, work_key), + // AGXTACommandDescriptor::prepare receives the shared USC FList ID + // through descriptor +0x1410. Optional GART/feature payloads are absent + // in Linux, so the retained owner's ID is the complete value. + RegisterWrite::new(0x14308, p.usc_flist_hardware_buffer_id as u64), + RegisterWrite::new(0x14318, p.ta_status | 1), + RegisterWrite::new(0x01740, 1), + RegisterWrite::new(0x1c880, deflake_1), + RegisterWrite::new(0x1c898, 1), + ]) +} + +pub(crate) fn build_fragment_registers( + p: &G17pRenderParameters, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], RenderBuildError> { + build_fragment_registers_for_work_key(p, render_work_key(1, 1)) +} + +fn build_fragment_registers_for_work_key( + p: &G17pRenderParameters, + work_key: u64, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], RenderBuildError> { + let g = geometry(p)?; + let tile_state = 0x3717f + | ((g.tiles_x - 1) << 44) + | ((g.tiles_y - 1) << 53) + | 0x20_0000_0000 + | (if p.layers > 1 { 0x1_0000_0000 } else { 0 }) + | ((p.utile_config & 0xf000) << 28); + + Ok([ + RegisterWrite::new(0x01739, 1), + RegisterWrite::new(0x10009, p.utile_config), + RegisterWrite::new(0x15379, p.store_pipeline_bind), + RegisterWrite::new(0x15381, p.store_pipeline), + RegisterWrite::new(0x15369, p.load_pipeline_bind), + RegisterWrite::new(0x15371, p.load_pipeline), + RegisterWrite::new(0x15131, p.merge_upper_x_bits as u64), + RegisterWrite::new(0x15139, p.merge_upper_y_bits as u64), + RegisterWrite::new(0x100a1, 0), + RegisterWrite::new(0x15069, 0), + RegisterWrite::new(0x15071, 0), + RegisterWrite::new(0x16058, 0), + RegisterWrite::new(0x10019, p.multisample_control), + RegisterWrite::new(0x100b1, g.macro_size), + RegisterWrite::new(0x16030, g.macro_size), + RegisterWrite::new(0x100d9, g.screen), + RegisterWrite::new(0x0a301, 0), + RegisterWrite::new(0x10791, 0xff_0200), + RegisterWrite::new(0x16098, p.heapmeta), + RegisterWrite::new(0x15109, p.scissor_array), + RegisterWrite::new(0x15101, p.depth_bias_array), + RegisterWrite::new(0x15021, p.aux_fb_flags), + RegisterWrite::new(0x15211, ((p.height as u64) << 32) | p.width as u64), + RegisterWrite::new(0x15049, p.aux_fb_page_count), + RegisterWrite::new(0x10051, p.tib_blocks), + RegisterWrite::new(0x15321, p.depth_dimensions), + RegisterWrite::new(0x15301, p.depth_clear_value_bits as u64), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15311, p.occlusion_query_base), + RegisterWrite::new(0x15319, p.depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + RegisterWrite::new(0x15351, 0), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x15231, 0), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15239, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15401, p.depth_stride), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15409, p.stencil_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15411, p.depth_aux_stride), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15419, p.stencil_aux_stride), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x16429, p.tilemap), + RegisterWrite::new(0x16060, p.layermeta), + RegisterWrite::new(0x16431, (4 * g.size1) << 24), + RegisterWrite::new(0x10039, p.tile_config), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16451, 0), + RegisterWrite::new(0x15359, 0), + RegisterWrite::new(0x100b8, 0x8860), + RegisterWrite::new(0x16461, p.aux_fb), + RegisterWrite::new(0x16090, p.aux_fb), + RegisterWrite::new(0x101e9, 0x1c), + RegisterWrite::new(0x160a8, 0), + RegisterWrite::new(0x16068, tile_state), + RegisterWrite::new(0x1a0a9, 0), + RegisterWrite::new(0x1a0b1, 0), + RegisterWrite::new(0x1a079, 0), + RegisterWrite::new(0x1a081, 0), + RegisterWrite::new(0x1a0d9, 0), + RegisterWrite::new(0x1a0e1, 0), + RegisterWrite::new(0x101c1, 0), + RegisterWrite::new(0x0d469, 0), + RegisterWrite::new(0x1a0f9, 8), + // Fragment analog of TA 0x0a5a1. Native derives its high word from the + // 3D AGXUMADataRec PTC/DUPM maximum at host descriptor +0xc00, copied + // through command +0x818. Linux has no owned pool-size input yet, so + // use the exact 0x60 zero-source fallback. Bit 3 retains one captured, + // workload-derived 0xd5 value only as a diagnostic. + RegisterWrite::new( + 0x0a5a9, + if p.native_ta_registers & 8 != 0 { + 0x0000_00d5_0040_0020 + } else { + 0x0000_0060_0040_0020 + }, + ), + RegisterWrite::new(0x0d429, 0x0000_0002_0000_0001), + RegisterWrite::new(0x160e0, p.fragment_lifecycle), + RegisterWrite::new(0x01499, p.fragment_lifecycle), + RegisterWrite::new(0x0a341, p.fragment_lifecycle), + // The fragment must use the same parameter-buffer owner as TA 1c830. + // Native TA::prepare stores its allocated HWPB ID at host +928; the + // same derived host descriptor is submitted through both channels. + // generate3DRegisterList reads that HOST field at 8a5c7cc. Its wrapper + // at 8a515d8 tail-calls with x1=raw command, x2=host descriptor: the + // unused channel argument in the mangled common-function name is NOT + // present in the optimized calling convention. Raw command +928 has + // a different producer and must not be used to infer this register. + RegisterWrite::new(0x1c838, p.ta_hardware_buffer_id as u64), + RegisterWrite::new(0x1ca28, p.cycle), + RegisterWrite::new(0x10211, work_key), + RegisterWrite::new(0x10420, work_key), + // AGX3DCommandDescriptor::prepare receives this ID from the USC pool's + // HardwareBufferIDManager and generate3DRegisterList ORs it with an + // optional GART payload and feature bit. Linux currently has neither + // optional source, so publish the retained USC FList owner's exact ID. + RegisterWrite::new(0x14048, p.usc_flist_hardware_buffer_id as u64), + RegisterWrite::new(0x14080, p.fragment_status | 1), + RegisterWrite::new(0x01731, 1), + RegisterWrite::new(0x16020, 1), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16068, 0x4_0000), + ]) +} + +pub(crate) fn partial_store_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_STORE_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15379, p.partial_store_pipeline_bind), + RegisterWrite::new(0x15381, p.partial_store_pipeline), + RegisterWrite::new(0x10039, p.tile_config), + RegisterWrite::new(0x15359, 0x20), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +pub(crate) fn partial_resume_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_RESUME_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15379, p.partial_store_pipeline_bind), + RegisterWrite::new(0x15381, p.partial_store_pipeline), + RegisterWrite::new(0x15369, p.partial_load_pipeline_bind), + RegisterWrite::new(0x15371, p.partial_load_pipeline), + RegisterWrite::new(0x10039, p.tile_config & 0xffff), + RegisterWrite::new(0x15359, 0x20), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +pub(crate) fn partial_load_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_LOAD_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15369, p.partial_load_pipeline_bind), + RegisterWrite::new(0x15371, p.partial_load_pipeline), + RegisterWrite::new(0x10039, p.tile_config & 0xffff), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +fn put_u8(raw: &mut [u8], offset: usize, value: u8) { + raw[offset] = value; +} + +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn encode_registers(raw: &mut [u8], offset: usize, registers: &[RegisterWrite]) { + for (index, register) in registers.iter().enumerate() { + let entry = offset + index * REGISTER_BYTES; + put_u32(raw, entry, register.number); + put_u64(raw, entry + 4, register.value); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RenderDescriptorKind { + Tiling, + Fragment, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderDescriptorObjects { + pub(crate) record_a: u64, + pub(crate) shared: u64, + pub(crate) record_b: u64, + pub(crate) zero: u64, +} + +impl RenderDescriptorObjects { + +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderDescriptorMetadata { + pub(crate) context_id: u32, + pub(crate) submission_ordinal: u64, + /// Monotonic state tag allocated by the command buffer's 36-entry state + /// ring. This is neither the selected ring index nor any QID/HWID. Native + /// retains the full word in the selected state object and places only its + /// low six bits in the stage work key. + pub(crate) command_buffer_state_word: u32, + /// TA HardwareBufferID copied to firmware descriptor +0x18. + /// + /// Every working native descriptor corpus couples this field to register + /// 0x1c830 exactly (for example 2/2, 4/4, and 5/5). The same allocator + /// result is also PBState +0x0c and the descriptor-ring index consumed by + /// `halLoadPBDesc`. + pub(crate) ta_hardware_buffer_id: u32, + pub(crate) submit_sequence: u64, +} + +impl RenderDescriptorMetadata { +} + +/// Native's TA/3D register-list producer couples the command-buffer state tag +/// to the descriptor's allocated application-GART context. The physical QID, +/// QoS HardwareBufferID, descriptor-array slot and 36-entry state-ring index +/// are independent namespaces and must not be substituted here. +const fn render_work_key(context_id: u32, command_buffer_state_word: u32) -> u64 { + ((context_id as u64) << 8) | ((command_buffer_state_word & 0x3f) as u64) +} + +fn descriptor_work_ordinal(ordinal: u64) -> u32 { + (ordinal as u32).wrapping_add((ordinal / 2) as u32) +} + +fn write_common_descriptor( + kind: RenderDescriptorKind, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + registers: &[RegisterWrite], + raw: &mut [u8], +) { + let (selector, pointers, register_offset, pointer_gap) = match kind { + RenderDescriptorKind::Tiling => (0u32, 0x10usize, 0x60usize, 8usize), + RenderDescriptorKind::Fragment => (1u32, 0x20usize, 0xa0usize, 0usize), + }; + put_u32(raw, 0, selector); + put_u64(raw, 4, metadata.submit_sequence); + put_u32(raw, 0x0c, metadata.context_id); + + let addresses = [ + objects.record_a, + objects.shared, + objects.record_b, + objects.zero, + ]; + let mut offset = pointers; + for (index, address) in addresses.iter().enumerate() { + put_u64(raw, offset, *address); + offset += 8; + if index == 0 { + offset += pointer_gap; + } + } + encode_registers(raw, register_offset, registers); + + let work = descriptor_work_ordinal(metadata.submission_ordinal); + match kind { + RenderDescriptorKind::Tiling => { + put_u32(raw, 0x18, metadata.ta_hardware_buffer_id); + put_u32(raw, 0x48, work); + } + RenderDescriptorKind::Fragment => {} + } +} + +pub(crate) fn build_ta_descriptor_base( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + registers: &[RegisterWrite], + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < TA_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + if registers.is_empty() || registers.len() > (0x660 - 0x60) / REGISTER_BYTES { + return Err(RenderBuildError::BufferTooSmall); + } + raw[..TA_DESCRIPTOR_SIZE].fill(0); + write_common_descriptor( + RenderDescriptorKind::Tiling, + objects, + metadata, + registers, + raw, + ); + + // Like the fragment boundaries, these are generated for every native TA + // command (25G83 88bdfc4/88be090/88be1a4), not only a queue's first kick. + // The final three entries are 01740, 1c880, 1c898; firmware reads +38 and + // +3a/+3c to select the corresponding embedded RCE boundaries. + put_u16(raw, 0x38, (TA_REGISTER_COUNT - 2) as u16); + put_u16(raw, 0x3a, TA_REGISTER_COUNT as u16); + put_u16(raw, 0x3c, TA_REGISTER_COUNT as u16); + + let deflake = ta_user_offset(parameters.deflake_1)?; + let sampler_max = if parameters.sampler_count == 0 { + 0 + } else { + parameters.sampler_count + 1 + }; + put_u32( + raw, + 0x0768, + (((registers.len() * REGISTER_BYTES) as u32) << 16) | registers.len() as u32, + ); + put_u64(raw, 0x0780, parameters.tpc); + // Native copies RTMemory's geometry-derived TPC size beside its pointer + // on every submission (25G83 88fd970..974), independent of queue lifetime. + put_u64(raw, 0x0788, parameters.tpc_size); + put_u32(raw, 0x07d6, deflake as u32); + put_u32(raw, 0x0876, u32::MAX); + put_u64(raw, 0x087a, parameters.sampler_array); + put_u32(raw, 0x0882, parameters.sampler_count); + put_u32(raw, 0x0886, sampler_max); + // Present in 436/438 valid G17P TA records, including both exact 32x32 + // geometry matches. This belongs to the TA output/activation tail; leaving + // it zero is a structural mismatch before the linked 3D job is generated. + put_u32(raw, 0x0892, 1); + put_u8(raw, 0x0932, 0x44); + put_u8(raw, 0x093c, 1); + put_u8(raw, 0x094d, parameters.context_generation); + put_u64(raw, 0x08fe, parameters.ta_timestamp_start); + put_u64(raw, 0x0906, parameters.ta_timestamp_end); + // +0x090e/+0x0916 are caller-visible timestamp destinations. Firmware + // writes them only inside its timestamp aperture, so they stay zero; the + // completion publishes the pass envelope to userspace instead. + // AGXTAChannelSKSM::submitBuffer mirrors the same dependency/current pair + // consumed by the TA RCE program into the firmware command. These fields + // are deliberately unaligned in the native 0x9c0-byte command layout. + put_u32(raw, 0x086e, (parameters.lifecycle >> 32) as u32); + put_u64(raw, 0x08ce, parameters.lifecycle as u32 as u64); + Ok(()) +} + +fn put_embedded_registers( + raw: &mut [u8], + header_offset: usize, + program_offset: usize, + registers: &[RegisterWrite], +) { + let bytes = registers.len() * REGISTER_BYTES; + put_u32( + raw, + header_offset, + ((bytes as u32) << 16) | registers.len() as u32, + ); + encode_registers(raw, program_offset, registers); +} + +pub(crate) struct G17pFragmentRegisters<'a> { + pub(crate) main: &'a [RegisterWrite], + pub(crate) store: &'a [RegisterWrite], + pub(crate) resume: &'a [RegisterWrite], + pub(crate) load: &'a [RegisterWrite], +} + +pub(crate) fn build_fragment_descriptor_base( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + programs: G17pFragmentRegisters<'_>, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < FRAGMENT_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + if programs.main.is_empty() + || programs.main.len() > (0x6a0 - 0xa0) / REGISTER_BYTES + || programs.store.len() > (0xec0 - 0x7c0) / REGISTER_BYTES + || programs.resume.len() > (0x15e0 - 0xee0) / REGISTER_BYTES + || programs.load.len() > (0x1d00 - 0x1600) / REGISTER_BYTES + { + return Err(RenderBuildError::BufferTooSmall); + } + raw[..FRAGMENT_DESCRIPTOR_SIZE].fill(0); + let geometry = geometry(parameters)?; + let register_count = programs.main.len(); + write_common_descriptor( + RenderDescriptorKind::Fragment, + objects, + metadata, + programs.main, + raw, + ); + + put_u64(raw, 0x40, parameters.tilemap); + put_u64(raw, 0x48, parameters.multisample_control); + // Native copies the render target's sample count through host +0x8c8 + // into every fragment descriptor at +0x50 (25G83: 0x8839d80, + // 0x8837af8..0x8837b08). It is not first-work metadata. The UAPI has + // already encoded log2(samples) in UTILE_CONFIG bits [1:0]. + put_u32(raw, 0x50, 1u32 << (parameters.utile_config & 3)); + put_u32(raw, 0x54, geometry.macro_size as u32); + put_u32(raw, 0x68, parameters.merge_upper_x_bits); + put_u32(raw, 0x6c, parameters.merge_upper_y_bits); + put_u64(raw, 0x78, geometry.tiles_x * geometry.tiles_y); + + // RCE boundaries belong to every command, including retained queue kicks. + // Native emits these from the running register count (25G83 88c0740, + // 88c07ac, 88c08c8, 88c0970); firmware consumes +80/+82/+84/+88 when + // selecting an entry in this command's embedded program. The fixed recipe + // ends with 01731=1, 16020=1, 16020=0, 16068=40000. Boundaries are after + // 01731, after the first 16020, that same entry, and the complete program. + // Keeping these only in the cold wrapper left ordinary restages at zero. + put_u16(raw, 0x80, (register_count - 3) as u16); + put_u16(raw, 0x82, (register_count - 2) as u16); + put_u16(raw, 0x84, (register_count - 2) as u16); + put_u16(raw, 0x88, register_count as u16); + + put_u32( + raw, + 0x07a8, + (((register_count * REGISTER_BYTES) as u32) << 16) | register_count as u32, + ); + put_embedded_registers(raw, 0x0ec8, 0x07c0, programs.store); + put_embedded_registers(raw, 0x15e8, 0x0ee0, programs.resume); + put_embedded_registers(raw, 0x1d08, 0x1600, programs.load); + + put_u64(raw, 0x1d20, parameters.depth_bias_array); + put_u64(raw, 0x1d30, parameters.scissor_array); + put_u64(raw, 0x1d40, parameters.occlusion_query_base); + put_u64(raw, 0x1e78, parameters.load_pipeline_bind); + put_u64(raw, 0x1e80, parameters.load_pipeline); + put_u64(raw, 0x1ea8, parameters.partial_load_pipeline_bind); + put_u64(raw, 0x1eb0, parameters.partial_load_pipeline); + put_u32(raw, 0x1ec0, 0x0404_0404); + put_u64(raw, 0x1f38, parameters.tib_blocks); + put_u64(raw, 0x1f40, parameters.aux_fb_flags); + put_u32(raw, 0x1f48, parameters.width); + put_u32(raw, 0x1f4c, parameters.height); + put_u64(raw, 0x1f50, parameters.aux_fb_page_count); + put_u64(raw, 0x1f58, parameters.tile_config); + put_u32(raw, 0x1f78, parameters.store_pipeline_bind as u32); + put_u64(raw, 0x1f7c, parameters.store_pipeline); + put_u32(raw, 0x1f98, parameters.partial_store_pipeline_bind as u32); + put_u64(raw, 0x1f9c, parameters.partial_store_pipeline); + put_u32(raw, 0x1fa8, parameters.depth_clear_value_bits); + put_u64(raw, 0x1fac, (parameters.tib_blocks << 33) | 0x300); + // Native copies isMemorylessRender through the UMD render record +0x240, + // parsed record +0x1b8, and host descriptor +0x8bf into this byte. It is + // not a TIB-size predicate: native's memoryless policy also selects the + // shortened SPM phase programs. The admitted render UAPI represents + // memory-backed resources and has no memoryless-render policy or matching + // SPM-program construction. Keep that policy clear even for >8 TIB + // blocks; supporting memoryless renders requires modeling both together. + // Empty-tile processing remains independent in register 0x10039 bit 16. + put_u8(raw, 0x2100, 0); + // Exact G17P tail copied by AGX3DChannelSKSM::submitBuffer. +0x2104 is the + // accelerator control snapshot and +0x210c its low map nibble. +0x2110 is + // a separate render-setup field: prepare copies descriptor +0xc00 to setup + // +0x818, processRenderSetup carries it to +0x89c, and submitBuffer emits + // it here. Working Metal commands carry the invariant all-ones value. + put_u32(raw, 0x2104, parameters.gpc_perf_state_control); + // AGX3DChannelSKSM::submitBuffer mirrors the same dependency/current pair + // consumed by the fragment RCE program into the firmware command. + put_u32(raw, 0x2108, (parameters.fragment_lifecycle >> 32) as u32); + put_u32(raw, 0x210c, parameters.gpc_perf_state_map_low); + put_u32(raw, 0x2110, G17P_FRAGMENT_TAIL_SETUP_WORD); + // Native submitBuffer zeros +0x2114..+0x2123. Sampler state reaches the + // GPU through its real register/program path, not these command-tail + // lanes. The first two following flag lanes reproduce the working Neo + // shared-runtime snapshot; the final two come from per-command descriptor + // bytes and retain their constructor-zero values until those inputs are + // modeled. + put_u32( + raw, + 0x2124, + G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT[0], + ); + put_u32( + raw, + 0x2128, + G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT[1], + ); + put_u64(raw, 0x2168, parameters.fragment_lifecycle as u32 as u64); + put_u64(raw, 0x2198, parameters.fragment_timestamp_start); + put_u64(raw, 0x21a0, parameters.fragment_timestamp_end); + // +0x21a8/+0x21b0: caller-visible destinations, zero as in the TA record. + put_u8(raw, 0x21cc, 0x53); + put_u8(raw, 0x21d6, 1); + put_u8(raw, 0x21e7, parameters.context_generation); + put_u8(raw, 0x2208, 1); + put_u8(raw, 0x220c, 1); + put_u8(raw, 0x222d, g17p_frg_hwdsid_mode()); + Ok(()) +} + +/// FRG +0x222d mode byte: Linux has always written 1; all 160 native light +/// FRG captures hold 0. +fn g17p_frg_hwdsid_mode() -> u8 { + 1 +} + diff --git a/drivers/gpu/drm/asahi/m5/render_target.rs b/drivers/gpu/drm/asahi/m5/render_target.rs new file mode 100644 index 0000000000000..a277377cb5851 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/render_target.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Render-target allocation geometry and M5 viewport state. + +pub(crate) const PAGE_BYTES: usize = 0x4000; +pub(crate) const AUX_BYTES: usize = 0x8000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct M5RenderTargetLayout { + pub(crate) tilemap_bytes: usize, + pub(crate) heapmeta_offset: usize, + pub(crate) heapmeta_bytes: usize, + pub(crate) target_bytes: usize, + pub(crate) tpc_bytes: usize, +} + +fn align(value: usize, alignment: usize) -> Option { + Some(value.checked_add(alignment - 1)? & !(alignment - 1)) +} + +impl M5RenderTargetLayout { + pub(crate) fn new( + width: u32, height: u32, layers: u16, utile_width: u8, utile_height: u8, + clusters: u32, + ) -> Option { + if !(1..=16384).contains(&width) || !(1..=16384).contains(&height) + || !(1..=2048).contains(&layers) || clusters == 0 + || !matches!((utile_width, utile_height), (32, 32) | (32, 16) | (16, 16)) { + return None; + } + // Same 4x4 macrotile geometry as the register builders. M5 packs its + // boundaries in ten-bit fields; that does not change allocation size. + let tiles_x = (width as usize + 31) / 32; + let tiles_y = (height as usize + 31) / 32; + let per_mtile = align((tiles_x + 3) / 4, 4)? + .checked_mul(align((tiles_y + 3) / 4, 4)?)?; + let utiles = (32 / utile_width as usize) * (32 / utile_height as usize); + let tilemap_bytes = align(5usize.checked_mul(per_mtile)?.checked_mul(utiles)?, 4)? + .checked_mul(16)?.checked_mul(layers as usize)?; + let heapmeta_offset = align(tilemap_bytes, 0x1000)?; + let heapmeta_bytes = if layers > 1 { 0x300 } else { 0x200 }; + let target_bytes = align(heapmeta_offset.checked_add(heapmeta_bytes)?, PAGE_BYTES)?; + let tpc_bytes = align(8usize.checked_mul(per_mtile)?.checked_mul(utiles)? + .checked_mul(16)?.checked_mul(layers as usize)? + .checked_mul(clusters as usize)?, PAGE_BYTES)?; + Some(Self { tilemap_bytes, heapmeta_offset, heapmeta_bytes, target_bytes, tpc_bytes }) + } +} + +fn half_float_bits(value: u32) -> u32 { + // The admitted integer dimensions are exactly representable in binary32. + let highest = 31 - value.leading_zeros(); + ((highest + 126) << 23) | ((value << (23 - highest)) & 0x7fffff) +} + +/// Native M5 deflake page: viewport at +0x9c0, command header 0x3000. +/// G17P uses +0x900/0xc00 and must not supply this page. The eight nonzero +/// dwords match the M5 25G76 resource capture; dimensions are caller-owned. +pub(crate) fn initialize_deflake(page: &mut [u8], width: u32, height: u32) -> Option<()> { + if page.len() != PAGE_BYTES || !(1..=16384).contains(&width) + || !(1..=16384).contains(&height) { return None; } + page.fill(0); + let x = half_float_bits(width); + let y = half_float_bits(height); + for (offset, value) in [(0x9c0, 0x3000), + (0x9c4, 0x80000000 | ((width + 31) / 32 - 1)), + (0x9c8, (height + 31) / 32 - 1), (0x9d0, x), (0x9d4, x), + (0x9d8, y), (0x9dc, y | 0x80000000), (0x9e4, 0x3f800000)] { + page[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + Some(()) +} + diff --git a/drivers/gpu/drm/asahi/m5/resources.rs b/drivers/gpu/drm/asahi/m5/resources.rs new file mode 100644 index 0000000000000..30d94f4d97148 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources.rs @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![allow(dead_code)] + +//! Retained RAM for the J813 G17G/25G76 initialization graph. +//! +//! Construction only: no processor start, mailbox publication, work submission +//! or DRM registration. The eventual boot session must retain this entire +//! owner until BOTH firmware processors are stopped, including on failure. +//! M5 UAT lifecycle admission remains separate from this memory layout. + +use super::mapped::MappedObject; +use crate::{driver::AsahiDrmDevice, m5::initdata as init, mmu}; +use crate::m5::completion::{graphics_pointer_head, MAX_GRAPHICS_GENERATION}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; +use kernel::sync::Arc; + +mod firmware_view; +mod graphics_memory; +mod graphics_usc; +mod parameter_buffer; +mod target_memory; +mod graphics_job; +mod graphics_queues; +mod application_compute; + +const KERNEL_BASE: u64 = 0xffff_fc20_0000_0000; +const PRIVATE_BASE: u64 = KERNEL_BASE + 0x2_0000; +const ROOTS_BASE: u64 = KERNEL_BASE + 0x1a_8000; +const SHARED_BASE: u64 = KERNEL_BASE + 0xc078_8000; +const FWCTL_BASE: u64 = KERNEL_BASE + 0x1c_0000; +const SENTINEL_BASE: u64 = KERNEL_BASE + 0xc07e_0000; +const PBDESC_BASE: u64 = KERNEL_BASE + 0x15e_0000; +const UMA_BASE: u64 = KERNEL_BASE + 0x15e_8000; + +/// Owning mappings are dropped before the UAT. Aliases retain the SAME GEM +/// backing; no firmware pointer refers to a copied snapshot or temporary map. +pub(crate) struct M5FirmwareMemory { + primary_root: MappedObject, + secondary_root: MappedObject, + shared: MappedObject, + private: MappedObject, + region_a: MappedObject, + region_c: MappedObject, + fwctl: MappedObject, + aon: MappedObject, + qos: MappedObject, + pbdesc: MappedObject, + uma: MappedObject, + completions: KVec, + aliases: KVec, + registers: KVec, + graphics: Option<(graphics_job::M5GraphicsJob, graphics_queues::M5GraphicsQueues)>, + application_compute: Option, + application_outer_head: u32, + report_counts: [[u32; 2]; 2], + trace_printed: [u32; 2], + compute_kick_observed: bool, + compute_dispatch_returned: bool, + graphics_stamp_flags: u128, + uat: Arc, +} + +impl M5FirmwareMemory { + /// Internal defaults for the initial M5 render-state envelope. The DRM + /// translator replaces all program/input addresses, and owned target + /// allocation replaces private storage before any queue publication. + pub(crate) fn reference_render_profile() -> + (crate::m5::render_desc::G17pRenderParameters, crate::m5::render::M5RenderControls) { + graphics_job::triangle_parameters() + } + + /// All fallible preparation happens before publication. The caller must + /// pass a prepared M5 UAT while both processors await initdata. The + /// caller retains its own Arc until both CPUs stop, including when this + /// unpublished graph fails construction. There is no Neo tuning fallback. + #[inline(never)] + pub(crate) fn new_with_uat( + dev: &AsahiDrmDevice, + uat: Arc, + boot_timestamp: u64, + ) -> Result> { + let memory = Self::allocate_with_uat(dev, uat, boot_timestamp)?; + fence(Ordering::SeqCst); + Ok(memory) + } + + #[inline(never)] + fn allocate_with_uat( + dev: &AsahiDrmDevice, + uat: Arc, + boot_timestamp: u64, + ) -> Result> { + let mut registers = KVec::new(); + for window in init::M5_25G76_REGISTER_WINDOWS { + let (physical, virtual_address, span) = window.mapped_pages().map_err(|_| EINVAL)?; + let protection = match window.flag { + 0 => mmu::PROT_FW_MMIO_RO, + 2 => mmu::PROT_FW_MMIO_RW, + _ => return Err(EINVAL), + }; + // Page-table writes only. No register contents are accessed. + registers.push(uat.kernel_vm().map_io(dev.as_ref().as_ref(), virtual_address, physical.try_into()?, + span.try_into()?, protection)?, GFP_KERNEL)?; + } + let fixed = |address, size, protection| MappedObject::new_wc_at( + dev, uat.kernel_vm(), address, size, protection); + // Install every fixed address before allocator-selected objects. + let mut private = fixed(PRIVATE_BASE, init::M5_25G76_PRIVATE_CLUSTER_SIZE, mmu::PROT_FW_SHARED_RW)?; + // Separate mappings retain the empty page between the two roots. + let mut primary_root = fixed(ROOTS_BASE, init::ROOT_SIZE_PRIMARY, mmu::PROT_FW_SHARED_RO)?; + let mut secondary_root = fixed(ROOTS_BASE + init::SECONDARY_ROOT_DELTA, + init::ROOT_SIZE_SECONDARY, mmu::PROT_FW_SHARED_RO)?; + let mut shared = fixed(SHARED_BASE, init::NATIVE_SHARED_CLUSTER_SIZE, mmu::PROT_FW_PRIV_RW)?; + let region_a = fixed(KERNEL_BASE + 0x1b_8000, init::REGION_A_SIZE, mmu::PROT_FW_SHARED_RO)?; + let mut region_c = fixed(KERNEL_BASE + 0x1a_0000, init::REGION_C_SIZE, mmu::PROT_FW_SHARED_RO)?; + let fwctl = fixed(FWCTL_BASE, 0x4000, mmu::PROT_FW_SHARED_RW)?; + // These two tables have zero-backed mappings at handoff. Mode6 + // prepares its own UMA slot before any compute publication. + let mut pbdesc = fixed(PBDESC_BASE, 0x4000, mmu::PROT_FW_SHARED_RW)?; + let mut uma = fixed(UMA_BASE, 0x4000, mmu::PROT_FW_SHARED_RW)?; + let mut aliases = KVec::new(); + for (object, address) in [(&mut pbdesc, init::REGION_VIEW_LOW_ADDRS[0]), + (&mut uma, init::REGION_VIEW_LOW_ADDRS[1])] { + aliases.push(object.map_alias_at(uat.kernel_lower_vm(), address, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + if uat.kernel_vm().translate_iova(object.iova())? + != uat.kernel_lower_vm().translate_iova(address)? { + return Err(EFAULT); + } + } + + let mut completions = KVec::new(); + let mut completion_bindings = [init::G17PKsmCompletionAliases { low: 0, high: 0 }; 4]; + for (ordinal, binding) in completion_bindings.iter_mut().enumerate() { + // The fixed alias pairs are measured on M5, including ordinals + // 1 and 3 which are absent from the older Neo initialization. + let low = 0x70_0184_8000 + ordinal as u64 * 0x2_8000; + let high = KERNEL_BASE + 0xc07e_8000 + ordinal as u64 * 0x2_8000; + let mut backing = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, + init::M5_25G76_COMPLETION_BACKING_SIZE, mmu::PROT_FW_PRIV_RW)?; + aliases.push(backing.map_alias_at(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..init::M5_25G76_COMPLETION_BACKING_SIZE).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset as u64)? + != uat.kernel_lower_vm().translate_iova(low + offset as u64)? { + return Err(EFAULT); + } + } + *binding = init::G17PKsmCompletionAliases { low, high }; + completions.push(backing, GFP_KERNEL)?; + } + // Keep allocator-selected objects below the high fixed graph, and + // above the PB/UMA tables. This cannot occupy the deferred blank + // sentinel address or its neighboring guard page. + let dynamic = KERNEL_BASE + 0x168_0000..KERNEL_BASE + 0x1000_0000; + let qos = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), dynamic.clone(), + 0xe40, mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW)?; + let mut aon = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), dynamic, + init::M5_25G76_AON_PTD_SIZE, + mmu::UAT_PGSZ as u64, mmu::PROT_FW_PRIV_RW)?; + aon.with_bytes_mut(|out| init::encode_m5_25g76_aon_ptd(out).map_err(|_| EINVAL))?; + region_c.with_bytes_mut(|out| init::encode_region_c_for_profile( + init::RootProfile::J813G17G25G76, out).map_err(|_| EINVAL))?; + private.with_bytes_mut(|out| init::encode_m5_25g76_private_cluster( + FWCTL_BASE, out).map_err(|_| EINVAL))?; + + let addresses = init::SharedAddresses { + region_a: region_a.iova(), region_c: region_c.iova(), + hw_data_bundle: SHARED_BASE, hw_data_bundle_alloc: init::NATIVE_SHARED_CLUSTER_SIZE, + }; + let primary = init::InstanceAddresses { + root: ROOTS_BASE, main_config: SHARED_BASE + init::NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET as u64, + status_a: PRIVATE_BASE + 0xee40, status_b: PRIVATE_BASE + 0x340, + secondary_extras: [0, 0], + }; + let secondary = init::InstanceAddresses { + root: ROOTS_BASE + init::SECONDARY_ROOT_DELTA, + main_config: SHARED_BASE + init::NATIVE_SECONDARY_MAIN_BUNDLE_OFFSET as u64, + status_a: PRIVATE_BASE + 0xc3100, status_b: 0, + secondary_extras: [primary.status_b + init::PRIMARY_STATUS_B_CONFIG_OFFSET as u64, + PRIVATE_BASE + 0x177400], + }; + let views = init::PrimaryRegionViews { + sentinel_high: SENTINEL_BASE, alias_high: [PBDESC_BASE, UMA_BASE], + }; + let trace_mask = crate::module_parameters::m5_fw_trace.value(); + if trace_mask & !0x203 != 0 { return Err(EINVAL); } + // Exact M5 GFX firmware 0x1fadc retains root+0xa8 (status A), + // then 0x3074..0x3088 copies status-A+0 into its trace flags. + // Bits 0 and 1 gate power/submission records; 0x6260 reads the + // NEXT byte and tests bit 1 (full-word bit 9) for recovery summary + // 0x909. Publish through the host field before either descriptor. + let native_trace = private.read_u32(primary.status_a)?; + private.write_u32(primary.status_a, native_trace | trace_mask)?; + shared.with_bytes_mut(|out| { + init::encode_m5_25g76_hw_data(completion_bindings, qos.iova(), + &mut out[..init::M5_25G76_HW_DATA_SIZE]).map_err(|_| EINVAL)?; + init::encode_m5_25g76_bundle_policy(init::M5PolicyBindings { + state: PRIVATE_BASE + 0x177480, aon_ptd: aon.iova(), boot_timestamp, + }, out).map_err(|_| EINVAL)?; + for (role, instance, grid, views) in [ + (init::InstanceRole::Primary, &primary, PRIVATE_BASE, Some(&views)), + (init::InstanceRole::Secondary, &secondary, PRIVATE_BASE + 0x1770c0, None), + ] { + let channels = init::derive_channel_table(role, grid, instance.status_a, + instance.main_config, SHARED_BASE); + let offset = (instance.main_config - SHARED_BASE) as usize; + init::encode_main_config_for_profile(init::RootProfile::J813G17G25G76, + role, &addresses, instance, &channels, views, + &mut out[offset..offset + init::MAIN_CONFIG_SIZE]).map_err(|_| EINVAL)?; + } + Ok(()) + })?; + primary_root.with_bytes_mut(|out| { + init::encode_root_for_profile(init::RootProfile::J813G17G25G76, + init::InstanceRole::Primary, &addresses, &primary, + out).map_err(|_| EINVAL) + })?; + secondary_root.with_bytes_mut(|out| { + init::encode_root_for_profile(init::RootProfile::J813G17G25G76, + init::InstanceRole::Secondary, &addresses, &secondary, + out).map_err(|_| EINVAL) + })?; + // Unlike PB/UMA, this future blank sentinel is unmapped at native + // handoff. It becomes zero-backed later in macOS initialization. + if uat.kernel_vm().translate_iova(SENTINEL_BASE).is_ok() { + return Err(EINVAL); + } + // Writes to all CPU WC mappings precede any future handoff. Only + // the two measured control openings are present; work rings and + // later compute/render descriptors remain unsubmitted. + fence(Ordering::SeqCst); + // Keep the retained graph off the kernel stack, including + // construction and error-return temporaries in the boot caller. + KBox::init(try_init!(Self { primary_root, secondary_root, shared, private, region_a, region_c, fwctl, aon, qos, + pbdesc, uma, completions, aliases, registers, graphics: None, + report_counts: [[0; 2]; 2], trace_printed: [0; 2], + compute_kick_observed: false, compute_dispatch_returned: false, + graphics_stamp_flags: 0, application_compute: None, + application_outer_head: 0, uat }), GFP_KERNEL) + } + + /// Prepare a first application job while retaining its actual VM. No + /// native program bundle or captured input/output pages are requested. + /// The caller has validated the command and all user mapping coverage. + #[allow(clippy::too_many_arguments)] + pub(crate) fn initialize_application_graphics( + &mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm, + parameters: crate::m5::render_desc::G17pRenderParameters, + controls: crate::m5::render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], + ) -> Result { + if self.graphics.is_some() { return Err(EBUSY); } + let job = graphics_job::M5GraphicsJob::from_vm( + dev, &self.uat, vm, parameters, controls, clusters, usc_required)?; + self.attach_graphics(dev, job) + } + + fn attach_graphics(&mut self, dev: &AsahiDrmDevice, job: graphics_job::M5GraphicsJob) -> Result { + if self.graphics.is_some() { return Err(EBUSY); } + let queues = graphics_queues::M5GraphicsQueues::new(dev, &self.uat, &job)?; + let pb = job.parameter_descriptor()?; + self.pbdesc.with_bytes_mut(|out| { + let base = graphics_job::PB_HARDWARE_ID as usize * 16; + for (index, word) in pb.into_iter().enumerate() { + out[base + index * 4..base + index * 4 + 4].copy_from_slice(&word.to_le_bytes()); + } + Ok(()) + })?; + let usc = job.uma_descriptor(); + self.uma.with_bytes_mut(|out| { + let base = graphics_usc::HARDWARE_ID as usize * 32; + for (index, word) in usc.into_iter().enumerate() { + out[base + index * 8..base + index * 8 + 8].copy_from_slice(&word.to_le_bytes()); + } + Ok(()) + })?; + self.graphics = Some((job, queues)); + Ok(()) + } + + pub(crate) fn quarantine_graphics(&self) { + if let Some((job, _)) = &self.graphics { job.quarantine(); } + } + + /// Drain M5 report channels using the native producer contracts. + /// GFX 0x6ca8..0x6d18 emits 256 x 0x48 event records; 0x17e18 + /// emits 512 x 0x48 trace records. In each cursor object the host owns + /// +0 and firmware owns +0x20. Copy before releasing each slot. + pub(crate) fn drain_reports(&mut self, dev: &AsahiDrmDevice) -> Result { + let mut recovery = false; + for (role, status_offset) in [0xee40u64, 0xc3100].into_iter().enumerate() { + for (channel, (cursor_offset, ring_offset, capacity)) in + [(0x40u64, 0x2c0u64, 256u32), (0x240, 0xa6ac0, 512)] + .into_iter().enumerate() { + let cursor = PRIVATE_BASE + status_offset + cursor_offset; + let ring = (status_offset + ring_offset) as usize; + for _ in 0..128 { + let consumer = self.private.read_u32(cursor)?; + let producer = self.private.read_u32(cursor + 0x20)?; + if consumer >= capacity || producer >= capacity { return Err(EIO); } + if consumer == producer { break; } + fence(Ordering::Acquire); + let offset = ring + consumer as usize * 0x48; + let mut raw = [0u8; 0x48]; + self.private.with_bytes_mut(|bytes| { + raw.copy_from_slice(bytes.get(offset..offset + 0x48).ok_or(EINVAL)?); + Ok(()) + })?; + let word = |off: usize| u32::from_le_bytes(raw[off..off + 4].try_into().unwrap()); + let qword = |off: usize| u64::from_le_bytes(raw[off..off + 8].try_into().unwrap()); + let count = self.report_counts[role][channel]; + if channel == 0 { + // Type 1 (graphics stamps) arrives for every job. + if word(0) == 1 { + dev_dbg!(dev.as_ref(), "M5_EVENT role={} ordinal={} type={} payload={:x?}\n", + role, count, word(0), [qword(4), qword(12), qword(20), qword(28), qword(36)]); + } else { + dev_dbg!(dev.as_ref(), "M5_EVENT role={} ordinal={} type={} payload={:x?}\n", + role, count, word(0), [qword(4), qword(12), qword(20), qword(28), qword(36)]); + } + recovery |= word(0) == 4; + if role == 0 && word(0) == 1 { + self.graphics_stamp_flags |= u128::from(qword(4)) + | (u128::from(qword(12)) << 64); + } + } else { + let args = [qword(12), qword(20), qword(28), qword(36)]; + // This one-shot diagnostic owns qid11/stamp1. The + // exact M5 trace point precedes the hardware kick; + // the admission probe also requires its later callback + // return trace. Firmware-private counters can be cached. + if role == 0 && word(0) == 5 && word(44) == 0x41 + && args == [11, 1, 2, 1] { + self.compute_kick_observed = true; + } + // 0x26aa4 invokes the pending-work callback, which + // dispatches tag14 and writes SGX+0x21018 at0x11f5c. + // 0x26ac4 emits 0x130 only after that callback returns. + if role == 0 && self.compute_kick_observed && word(0) == 5 + && word(44) == 0x130 && args == [4, 0, 0, 0] { + self.compute_dispatch_returned = true; + } + // Exact M5 0x24eb4/0x26754 emit zero-argument + // processing-loop brackets. Count and consume them + // without spending the diagnostic print budget. + let loop_marker = word(0) == 5 && args == [0; 4] + && matches!(word(44) & 0x00ff_ffff, 0x105 | 0x106); + if !loop_marker && self.trace_printed[role] < if role == 0 { 512 } else { 16 } { + dev_dbg!(dev.as_ref(), "M5_TRACE role={} ordinal={} timestamp={} type={} id={:#x} args={:x?}\n", + role, count, qword(4), word(0), word(44), args); + self.trace_printed[role] += 1; + } + } + self.report_counts[role][channel] = count.saturating_add(1); + fence(Ordering::SeqCst); + self.private.write_u32(cursor, (consumer + 1) % capacity)?; + } + } + } + Ok(recovery) + } + + pub(crate) fn graphics_status(&mut self) -> Result<([u32; 3], [[u32; 3]; 2], [u32; 5])> { + let mut outer = [0; 3]; + let base = init::WORK_STATE_GRID_OFFSETS[8]; + for index in 0..3 { + outer[index] = self.private.read_u32(PRIVATE_BASE + + (base + index * init::CHANNEL_STATE_SPACING) as u64)?; + } + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + Ok((outer, queues.counters()?, job.progress()?)) + } + + pub(crate) fn graphics_completion(&mut self) -> Result { + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + let progress = job.progress()?; + let scheduler = job.scheduler_status()?; + Ok(crate::m5::completion::M5GraphicsCompletion { + submitted: progress[0], completed: scheduler[0], linked: scheduler[1], + list: queues.completion_list()?, stamps: [progress[1], progress[3]], + signalled: self.graphics_stamp_flags, + }) + } + + pub(crate) fn graphics_pair_complete( + completion: crate::m5::completion::M5GraphicsCompletion, + generation: u32, + ) -> bool { + (1..=MAX_GRAPHICS_GENERATION).contains(&generation) + && completion.pair_complete_at(graphics_queues::JOB_LIST, generation * 0x100) + } + + pub(crate) fn graphics_installation_status(&mut self) -> Result<([[u64; 5]; 2], [u64; 2], [u64; 2])> { + self.graphics.as_mut().ok_or(EINVAL)?.1.installation_status() + } + + pub(crate) fn graphics_queues_installed(&mut self) -> Result { + self.graphics.as_mut().ok_or(EINVAL)?.1.installed() + } + + pub(crate) fn publish_graphics_install(&mut self) -> Result { + let (outer, inner, progress) = self.graphics_status()?; + if outer != [0; 3] || inner != [[0, 0, 1]; 2] + || progress != [2, 0, 0, 0, 0] { return Err(EBUSY); } + if self.graphics_installation_status()? != ([[0; 5]; 2], [0; 2], [0xffff, 0]) { + return Err(EBUSY); + } + self.publish_graphics_records(0, 1) + } + + pub(crate) fn publish_graphics_job(&mut self) -> Result { + let (outer, inner, progress) = self.graphics_status()?; + if outer != [2; 3] || inner != [[1; 3]; 2] + || progress != [2, 0, 0, 0, 0] { return Err(EBUSY); } + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + if !queues.installed()? { return Err(EBUSY); } + self.graphics_stamp_flags = 0; + job.publish_context()?; + queues.publish_draw()?; + self.publish_graphics_records(2, 4) + } + + /// Reuse only a fully retired pair, replacing its application descriptors + /// while preserving firmware-owned PB/USC state and the installed queues. + #[allow(clippy::too_many_arguments)] + /// A different submitting VM takes over the retired job in place: same + /// context slot, descriptor graph, queues and PB/USC pools. The compute + /// owner shares that context, so its operands follow the job. + fn hand_off_application(&mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let (job, _) = self.graphics.as_mut().ok_or(EINVAL)?; + if job.owns_vm(vm) { return Ok(()); } + job.rebind_vm(dev, vm).inspect_err(|e| pr_err!("M5: rebind_vm failed: {:?}\n", e))?; + job.publish_context().inspect_err(|e| pr_err!("M5: publish_context failed: {:?}\n", e))?; + Ok(()) + } + + pub(crate) fn publish_next_application_job( + &mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm, + parameters: crate::m5::render_desc::G17pRenderParameters, + controls: crate::m5::render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], generation: u32, + ) -> Result { + if !(2..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + let previous = generation - 1; + let (outer, inner, _) = self.graphics_status()?; + if outer != [self.application_outer_head; 3] + || inner != [[graphics_pointer_head(previous); 3]; 2] + || !Self::graphics_pair_complete(self.graphics_completion()?, previous) { + return Err(EBUSY); + } + self.hand_off_application(dev, vm)?; + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + job.prepare_application_frame(dev, vm, parameters, controls, clusters, + usc_required, generation).inspect_err(|e| pr_err!("M5: prepare_application_frame failed: {:?}\n", e))?; + queues.publish_next_draw(job, generation).inspect_err(|e| pr_err!("M5: publish_next_draw failed: {:?}\n", e))?; + self.graphics_stamp_flags = 0; + self.publish_graphics_records(self.application_outer_head as usize, + graphics_pointer_head(generation) as u16) + } + + fn publish_graphics_records(&mut self, outer_index: usize, inner_head: u16) -> Result { + // Only recycle storage after every outer consumer reaches the old + // producer. The application caller also requires paired retirement. + if outer_index >= 256 || self.graphics_status()?.0 != [outer_index as u32; 3] { + return Err(EBUSY); + } + self.shared.with_bytes_mut(|out| { + for stage in 0..2 { + let offset = init::WORK_RING_BUNDLE_OFFSETS[8] + ((outer_index + stage) % 256) * 24; + let record = &mut out[offset..offset + 24]; + record.fill(0); + let address = graphics_queues::M5GraphicsQueues::queue_address(stage)?; + record[8..16].copy_from_slice(&address.to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&inner_head.to_le_bytes()); + record[22] = graphics_job::QUEUE_IDS[stage]; + record[23] = 1; + } + Ok(()) + })?; + fence(Ordering::SeqCst); + // Both stage-tagged SKSM queues travel through the CL_2 command ring. + let producer = init::WORK_STATE_GRID_OFFSETS[8] + 2 * init::CHANNEL_STATE_SPACING; + self.private.write_u32(PRIVATE_BASE + producer as u64, ((outer_index + 2) % 256) as u32)?; + self.application_outer_head = ((outer_index + 2) % 256) as u32; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn application_outer_head(&self) -> u32 { self.application_outer_head } + + /// Initial admission is deliberately one compute job after retired graphics. + /// No private diagnostic context, queue 11, or replacement UMA is installed. + pub(crate) fn initialize_application_compute(&mut self, dev: &AsahiDrmDevice, + vm: &mmu::Vm, command: &crate::m5::command::TranslatedComputeCommand, + graphics_generation: u32) -> Result { + if self.application_compute.is_some() || self.application_outer_head + 2 > 256 { + return Err(ENOTSUPP); + } + let (outer, inner, _) = self.graphics_status()?; + if graphics_generation == 0 || outer != [self.application_outer_head; 3] + || inner != [[graphics_pointer_head(graphics_generation); 3]; 2] + || !Self::graphics_pair_complete(self.graphics_completion()?, graphics_generation) { + return Err(EBUSY); + } + self.hand_off_application(dev, vm)?; + let job = &self.graphics.as_ref().ok_or(EINVAL)?.0; + self.application_compute = Some(application_compute::M5ApplicationCompute::new( + dev, &self.uat, job, command)?); + Ok(()) + } + + pub(crate) fn application_compute_installed(&mut self) -> Result { + let compute = self.application_compute.as_mut().ok_or(EINVAL)?; + Ok(compute.counters()? == [1; 3] && compute.installed()?) + } + + /// Publish one command-ring record for queue 12. `dispatch` false is the + /// installation record; true is dispatch generation `generation`, whose + /// inner head is the producer that generation's three pointers reach. + pub(crate) fn publish_application_compute(&mut self, dispatch: bool, + generation: u32) -> Result { + let outer_index = self.application_outer_head as usize; + if outer_index >= 256 || self.graphics_status()?.0 != [outer_index as u32; 3] { + return Err(EBUSY); + } + let inner_head = u16::try_from(application_compute::compute_pointer_head( + if dispatch { generation } else { 0 })).map_err(|_| EINVAL)?; + let compute = self.application_compute.as_mut().ok_or(EINVAL)?; + if dispatch { + // Preserve the retired graphics notifications for later render reuse. + self.graphics_stamp_flags &= !(1u128 << application_compute::QUEUE_ID); + compute.publish_dispatch(generation)?; + } else if compute.counters()? != [0, 0, 1] { return Err(EBUSY); } + let offset = init::WORK_RING_BUNDLE_OFFSETS[8] + outer_index * 24; + self.shared.with_bytes_mut(|out| { + let record = &mut out[offset..offset + 24]; + // Every outer consumer reached this slot (checked above), so its + // prior user, graphics or compute, has retired. Graphics records + // wrap this ring after 128 renders, so even the first compute + // record can land on a used slot. + record.fill(0); + record[8..16].copy_from_slice( + &application_compute::M5ApplicationCompute::queue_address().to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&inner_head.to_le_bytes()); + record[22] = application_compute::QUEUE_ID; + record[23] = 1; + Ok(()) + })?; + fence(Ordering::SeqCst); + let head = ((outer_index + 1) % 256) as u32; + let producer = init::WORK_STATE_GRID_OFFSETS[8] + 2 * init::CHANNEL_STATE_SPACING; + self.private.write_u32(PRIVATE_BASE + producer as u64, head)?; + self.application_outer_head = head; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Stage a further dispatch on the already installed compute queue. The + /// installed owner keeps its VM, USC and operand storage; only the command + /// stream and the generation's stamps change. + pub(crate) fn prepare_next_application_compute(&mut self, dev: &AsahiDrmDevice, + vm: &mmu::Vm, command: &crate::m5::command::TranslatedComputeCommand, + generation: u32) -> Result { + if !(2..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + self.hand_off_application(dev, vm)?; + let (job, _) = self.graphics.as_ref().ok_or(EINVAL)?; + self.application_compute.as_mut().ok_or(EINVAL)? + .prepare_next_dispatch(job, command, generation) + } + + pub(crate) fn application_compute_complete(&mut self, generation: u32) -> Result { + let (outer, _, _) = self.graphics_status()?; + Ok(outer == [self.application_outer_head; 3] + && self.application_compute.as_mut().ok_or(EINVAL)? + .complete(self.graphics_stamp_flags, generation)?) + } + + pub(crate) fn application_compute_status(&mut self) + -> Result<([u32; 3], crate::m5::completion::M5ComputeCompletion)> { + let compute = self.application_compute.as_mut().ok_or(EINVAL)?; + Ok((compute.counters()?, compute.completion(self.graphics_stamp_flags)?)) + } + + pub(crate) fn publish_idle_wake(&mut self) -> Result { + if self.control_counters()? != [[1; 3]; 2] { return Err(EBUSY); } + self.shared.with_bytes_mut(|out| { + let record = &mut out[0x1eac0..0x1eb00]; + if record.iter().any(|x| *x != 0) { return Err(EBUSY); } + record[..4].copy_from_slice(&0x0au32.to_le_bytes()); + Ok(()) + })?; + fence(Ordering::SeqCst); + let producer = init::WORK_STATE_GRID_OFFSETS[init::CONTROL_CHANNEL_INDEX] + + 2 * init::CHANNEL_STATE_SPACING; + self.private.write_u32(PRIVATE_BASE + producer as u64, 2)?; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn root(&self, secondary: bool) -> u64 { + if secondary { self.secondary_root.iova() } else { self.primary_root.iova() } + } + + /// Read the actual shared control state, not a detached host copy. The + /// offsets and three pointers per role are checked by the M5 capture + /// verifier against both native main objects and the private arena. + pub(crate) fn control_counters(&mut self) -> Result<[[u32; 3]; 2]> { + self.private.with_bytes_mut(|bytes| { + let mut counters = [[0; 3]; 2]; + for (role, grid) in [0usize, 0x1770c0].into_iter().enumerate() { + let base = grid + init::WORK_STATE_GRID_OFFSETS[init::CONTROL_CHANNEL_INDEX]; + for (index, value) in counters[role].iter_mut().enumerate() { + let offset = base + index * init::CHANNEL_STATE_SPACING; + let word = bytes.get(offset..offset + 4).ok_or(EINVAL)?; + // SAFETY: arena and native counter offsets are u32 aligned; + // this bounded WC mapping is retained while firmware runs. + *value = unsafe { core::ptr::read_volatile(word.as_ptr().cast::()) }; + } + } + fence(Ordering::Acquire); + Ok(counters) + }) + } + +} diff --git a/drivers/gpu/drm/asahi/m5/resources/application_compute.rs b/drivers/gpu/drm/asahi/m5/resources/application_compute.rs new file mode 100644 index 0000000000000..4f84edbd0875b --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/application_compute.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! One application compute dispatch sharing the retained graphics VM and USC. +//! +//! Queue 12 is separate from TA/fragment queues 10/11. All publication and +//! completion gates are explicit; constructing this owner does not submit it. + +use super::{graphics_job::M5GraphicsJob, graphics_usc, + firmware_view::FirmwareDataView, MappedObject}; +use crate::{driver::AsahiDrmDevice, m5::submission, m5::command, + m5::channel, m5::completion::{graphics_pointer_head, GRAPHICS_POINTER_COUNT, + MAX_GRAPHICS_GENERATION}, m5::compute, m5::sksm, mmu}; +use kernel::prelude::*; + +pub(super) const QUEUE_ID: u8 = 12; +const GRAPH: u64 = 0xffff_fc20_c008_0000; +const POINTERS: usize = 0x4000; +const ITEMS: usize = 0x8000; +/// Past the 0x500 command pointers the channel state declares, as for the +/// graphics queues, so the producer head wraps like theirs. +const CONFIG: usize = ITEMS + GRAPHICS_POINTER_COUNT as usize * 8; +const CONTROL: usize = 0xc000; +const _: () = assert!(CONFIG + 0x300 <= CONTROL); + +/// Installation publishes one command pointer; every dispatch generation then +/// appends the same three -- descriptor, channel config, kick -- so the +/// producer head runs 1, 4, 7, ... (mod 0x500) exactly as +/// graphics_pointer_head() does for queues 10/11. +pub(super) const fn compute_pointer_head(generation: u32) -> u32 { + graphics_pointer_head(generation) +} + +pub(super) const fn compute_generation_admitted(generation: u32) -> bool { + generation >= 1 && generation <= MAX_GRAPHICS_GENERATION +} + +const LIST: u64 = GRAPH + 0x10000; +const ENTRY: u64 = 0xffff_fc20_003d_0000; +const ENTRY_LOW: u64 = 0x70_0063_0000; +const DESC: u64 = 0xffff_fc20_c040_0000; +const DESC_LOW: u64 = 0x70_0038_0000; +const SCHEDULER: u64 = DESC + 0x4000; +const ZERO: u64 = DESC + 0x8000; +const SIGNALS: u64 = DESC + 0x10000; + +pub(super) struct M5ApplicationCompute { + aliases: KVec, + descriptor: MappedObject, + graph: MappedObject, + entry: MappedObject, + list: MappedObject, + signals: MappedObject, + /// GPU address of the job's compute storage. + storage: u64, + /// The operands mapped into the VM that took the graphics job over. + firmware: FirmwareDataView, +} + +impl M5ApplicationCompute { + pub(super) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, job: &M5GraphicsJob, + command: &command::TranslatedComputeCommand) -> Result { + // The adapter excludes helpers, timestamps, legacy sampler registers + // and the Neo add3 proof ABI before allocating or publishing anything. + if command.sampler_heap != 0 || command.sampler_count != 0 + || command.g17p_add3_buffers.is_some() { return Err(ENOTSUPP); } + // Validate observation support and vacancy before publishing a queue. + let firmware = FirmwareDataView::new(dev)?; + if firmware.queue_registration(QUEUE_ID)? != ([0; 5], 0) { return Err(EBUSY); } + // 128 KiB operands, a preemption page and a zero robustness page. + // Their addresses come from the application's reserved driver arena. + let storage = job.compute_storage(); + let mut descriptor = MappedObject::new_wc_at(dev, uat.kernel_vm(), DESC, + 0x10000, mmu::PROT_FW_PRIV_RW)?; + let mut entry = MappedObject::new_wc_at(dev, uat.kernel_vm(), ENTRY, + 0x20000, mmu::PROT_FW_PRIV_RW)?; + let mut aliases = KVec::new(); + for (object, low, size) in [(&mut descriptor, DESC_LOW, 0x10000), + (&mut entry, ENTRY_LOW, 0x20000)] { + aliases.push(object.map_alias_at(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(object.iova() + offset as u64)? + != uat.kernel_lower_vm().translate_iova(low + offset as u64)? { + return Err(EFAULT); + } + } + } + let mut signals = MappedObject::new_wc_at(dev, uat.kernel_vm(), SIGNALS, + 0x4000, mmu::PROT_FW_SHARED_RW)?; + // The submitted count is the host's and stays put; firmware republishes + // the completed count into the scheduler record each generation. + signals.write_u32(SIGNALS, 1)?; + Self::encode_descriptor(&mut descriptor, job, command, storage, 1)?; + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), GRAPH, + 0x10000, mmu::PROT_FW_PRIV_RW)?; + let mut list = MappedObject::new_wc_at(dev, uat.kernel_vm(), LIST, + 0x4000, mmu::PROT_FW_SHARED_RW)?; + list.with_bytes_mut(|out| { put64(out, 8, LIST); Ok(()) })?; + let prepared = Self::prepare_entry(1)?; + entry.with_bytes_mut(|out| { + let offset = prepared.offset as usize; + out[offset..offset + prepared.bytes.len()].copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let configuration = |install| channel::encode_m5_channel_config( + channel::M5ChannelConfig { + ring_low: ENTRY_LOW, ring_high: ENTRY, queue_id: QUEUE_ID, priority: 2, + install_queue: install, gpu_context: job.context_id(), + event: if install { None } else { Some(channel::M5ChannelEvent { + address: job.usc_control(), timestamp: 1, index: graphics_usc::HARDWARE_ID, + }) }, + channel_control: GRAPH + CONTROL as u64, update_scheduler: install, + scheduler_index: 0, channel_stamp_index: 0, + update_context: !install, owner_context: if install { 0 } else { 1 }, + extended_command: false, + }).map_err(|_| EINVAL); + let install = configuration(true)?; + let config = configuration(false)?; + let kick = Self::prepare_kick(1)?; + graph.with_bytes_mut(|out| { + let reset = submission::encode_m5_25g76_channel_state_reset( + submission::M5ChannelStateResetValues { + uncached_state: GRAPH + POINTERS as u64, + command_pointer_ring: GRAPH + ITEMS as u64, gpu_va_10: LIST, + word_48: 0, gpu_va_9c: GRAPH + CONTROL as u64, + }); + let queue = usize::from(QUEUE_ID) * 0xc0; + out[queue..queue + 0xc0].copy_from_slice(&reset); + for offset in [0x28, 0x2c, 0x40] { put32(out, queue + offset, 2); } + out[queue + 0x36..queue + 0x38].fill(0xff); + put32(out, POINTERS + 0x50, u32::MAX); + put32(out, POINTERS + 0x60, 0x500); + out[CONTROL..CONTROL + 2].fill(0xff); + out[CONTROL + 5] = 1; + out[CONTROL + 0x26] = 2; + out[CONTROL + 0x33] = 0xff; + for (offset, bytes) in [(CONFIG, &install[..]), (CONFIG + 0x100, &config[..]), + (CONFIG + 0x200, &kick[..])] { + out[offset..offset + bytes.len()].copy_from_slice(bytes); + } + for (index, address) in [GRAPH + CONFIG as u64, DESC, + GRAPH + CONFIG as u64 + 0x100, GRAPH + CONFIG as u64 + 0x200] + .into_iter().enumerate() { put64(out, ITEMS + index * 8, address); } + put32(out, POINTERS + 0x40, 1); + Ok(()) + })?; + Ok(Self { aliases, descriptor, graph, entry, list, signals, storage, + firmware }) + } + + pub(super) fn queue_address() -> u64 { GRAPH + QUEUE_ID as u64 * 0xc0 } + + /// Re-encode the whole descriptor for one dispatch generation. The + /// scheduler record is cleared and re-seeded exactly as + /// M5GraphicsJob::rebuild_retired_frame() does for queues 10/11: firmware + /// republishes the completed count into it, so a stale one would satisfy + /// the next generation's completion gate before the dispatch even ran. + fn encode_descriptor(descriptor: &mut MappedObject, job: &M5GraphicsJob, + command: &command::TranslatedComputeCommand, storage: u64, + generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + descriptor.with_bytes_mut(|out| { + out[0x4000..0x4000 + 0xc8].fill(0); + put64(out, 0x4000, SIGNALS); + put32(out, 0x4010, 0x50); + compute::encode_m5_compute_descriptor(compute::M5ComputeDescriptor { + registers: compute::M5ComputeParameters { + preempt_base: storage + 0x20000, + cdm_base: command.control_stream_base, + dispatch_identity: 0x0000_0001_0000_0001, + context_id: job.context_id(), work_ordinal: 0, + robustness: storage + 0x24000, + operand_state_base: storage, + private_pool_hw_id: u64::from(graphics_usc::HARDWARE_ID), + descriptor_control_530: 0x54b, descriptor_control_538: 0xf5, + descriptor_flag_60a: false, descriptor_flag_60b: false, + }, + submit_sequence: 0, scheduler_record: SCHEDULER, + descriptor_low_alias: DESC_LOW, + cdm_terminator: command.control_stream_terminator, + dispatch_stamps: [SIGNALS + 0x40, SIGNALS + 0x80], + event_machine_stamp: generation * 0x100, queue_id: u32::from(QUEUE_ID), + queue_ordinal: 0, has_event: false, submission_index: generation, + status: [SIGNALS + 0xc0, SIGNALS + 0xc8], + shared_control: job.usc_control(), shared_control_enabled: true, + event_metadata: [0; 16], zero_page: ZERO, owner_id: 1, + descriptor_work_ordinal: 0, + }, &mut out[..0x1000]).map_err(|_| EINVAL) + }) + } + + fn prepare_entry(generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + let stamp = u64::from(generation); + sksm::prepare_m5_sksm_entry(sksm::M5SksmQueue { + id: QUEUE_ID, priority: 2, current_stamp: stamp, submitted: 0, + previous_stamp: if generation == 1 { None } else { Some(stamp - 1) }, + }, sksm::M5SksmOperands { + barriers: &[], first_predecessor: 0, implicit_dependency_type: 1, + mcache: None, flag_bit1: false, flag_bit61: false, flag_bit63: false, + payload: [DESC, Self::queue_address()], qos_byte_32: 3, qos_class_40: 8, + qos_word_48: 0xffff, event_mask: [0; 4], rce_kind: 0, + rce: [sksm::M5SksmRce { address: DESC_LOW + 0x40, tag: 37 }, + sksm::M5SksmRce { address: DESC_LOW + 0x760, tag: 4 }, + sksm::M5SksmRce { address: 0, tag: 0 }, + sksm::M5SksmRce { address: 0, tag: 0 }], + auxiliary: 0x3f_ffff_ffff_ffff, + }).map_err(|_| EINVAL) + } + + fn prepare_kick(generation: u32) -> Result<[u8; 21]> { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + sksm::encode_m5_sksm_add_kicks(QUEUE_ID, 2, u64::from(generation), 1) + .map_err(|_| EINVAL) + } + + /// Stage a further dispatch on the installed queue. Nothing here is + /// published: the producer head moves only in publish_dispatch(). + pub(super) fn prepare_next_dispatch(&mut self, job: &M5GraphicsJob, + command: &command::TranslatedComputeCommand, generation: u32) -> Result { + if generation < 2 || !compute_generation_admitted(generation) { return Err(EINVAL); } + if self.counters()? != [compute_pointer_head(generation - 1); 3] + || !self.installed()? { return Err(EBUSY); } + Self::encode_descriptor(&mut self.descriptor, job, command, + self.storage, generation)?; + let prepared = Self::prepare_entry(generation)?; + self.entry.with_bytes_mut(|out| { + let offset = prepared.offset as usize; + let slot = &mut out[offset..offset + prepared.bytes.len()]; + // Every preceding dispatch has retired, including the prior user + // of this 256-entry SKSM slot. Preserve the first-use check. + if generation < 256 && slot.iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + slot.copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let kick = Self::prepare_kick(generation)?; + self.graph.with_bytes_mut(|out| { + out[CONFIG + 0x200..CONFIG + 0x200 + kick.len()].copy_from_slice(&kick); + for (index, address) in [DESC, GRAPH + CONFIG as u64 + 0x100, + GRAPH + CONFIG as u64 + 0x200].into_iter().enumerate() { + let absolute = 1 + (generation as usize - 1) * 3 + index; + let offset = ITEMS + (absolute % GRAPHICS_POINTER_COUNT as usize) * 8; + if absolute < GRAPHICS_POINTER_COUNT as usize + && out[offset..offset + 8].iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + put64(out, offset, address); + } + Ok(()) + }) + } + + pub(super) fn counters(&mut self) -> Result<[u32; 3]> { + let mut out = [0; 3]; + for (index, offset) in [0, 0x30, 0x40].into_iter().enumerate() { + out[index] = self.graph.read_u32(GRAPH + (POINTERS + offset) as u64)?; + } + Ok(out) + } + + pub(super) fn installed(&mut self) -> Result { + let (slot, scheduler) = self.firmware.queue_registration(QUEUE_ID)?; + let header = self.graph.read_u32(GRAPH + CONTROL as u64)?; + let bitmap = self.graph.read_u64(GRAPH + CONTROL as u64 + 0x12)?; + Ok(slot[0] & 1 != 0 && slot[1] == ENTRY && slot[2] == ENTRY_LOW + && slot[3] == 4u64 << 32 && slot[4] as u32 == 2 + && scheduler == GRAPH + CONTROL as u64 + && header & 0xff != 0xff && (header >> 8) & 0xff != 0xff + && (header >> 16) & 0xff == 4 && bitmap == 1u64 << QUEUE_ID) + } + + pub(super) fn publish_dispatch(&mut self, generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + if self.counters()? != [compute_pointer_head(generation - 1); 3] + || !self.installed()? { return Err(EBUSY); } + self.graph.write_u32(GRAPH + POINTERS as u64 + 0x40, + compute_pointer_head(generation)) + } + + pub(super) fn completion(&mut self, signalled: u128) + -> Result { + Ok(crate::m5::completion::M5ComputeCompletion { + submitted: self.signals.read_u32(SIGNALS)?, + completed: self.descriptor.read_u32(SCHEDULER + 0xc)?, + linked: self.descriptor.read_u32(SCHEDULER + 0x94)?, + list: [self.list.read_u64(LIST)?, self.list.read_u64(LIST + 8)?], + stamp: self.signals.read_u32(SIGNALS + 0x40)?, signalled, + }) + } + + pub(super) fn complete(&mut self, signalled: u128, generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + Ok(self.counters()? == [compute_pointer_head(generation); 3] + && self.completion(signalled)? + .complete_at(LIST, QUEUE_ID, generation * 0x100)) + } +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} diff --git a/drivers/gpu/drm/asahi/m5/resources/firmware_view.rs b/drivers/gpu/drm/asahi/m5/resources/firmware_view.rs new file mode 100644 index 0000000000000..54ed6c966ac76 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/firmware_view.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Read-only view of the GFX firmware's DATA segment (the `gfx-data` reserved region). + +use crate::driver::AsahiDrmDevice; +use core::sync::atomic::{fence, Ordering}; +use kernel::{c_str, io, prelude::*}; + +pub(super) struct FirmwareDataView { + map: io::mem::Mem, + cache_line: usize, +} + +// SAFETY: This owns a fixed mapping of reserved RAM, read only through +// bounded volatile accesses. It neither transfers ownership nor writes DATA. +unsafe impl Send for FirmwareDataView {} +// SAFETY: Concurrent read-only volatile observations do not mutate host state. +unsafe impl Sync for FirmwareDataView {} + +impl FirmwareDataView { + pub(super) fn new(dev: &AsahiDrmDevice) -> Result { + let resource = dev.as_ref().as_ref().of_node().ok_or(EINVAL)? + .reserved_mem_region_to_resource_byname(c_str!("gfx-data"))?; + if resource.size() != 0x134000 || resource.start() < 0x100_0000_0000 + || resource.start() & 0x3fff != 0 + || resource.start().checked_add(resource.size()).ok_or(EOVERFLOW)? > 0x104_0000_0000 + || 0x127430 - 0x4c000 + 13 * 0x28 > resource.size() as usize + || 0x1257e0 - 0x4c000 + 13 * 8 > resource.size() as usize + || 0x178458 - 0x4c000 + 8 > resource.size() as usize { + return Err(EINVAL); + } + let ctr: u64; + // SAFETY: CTR_EL0 is an architectural read-only cache geometry register. + unsafe { core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)) }; + let cache_line = 4usize << ((ctr >> 16) & 0xf); + if cache_line > 0x4000 { return Err(EINVAL); } + // SAFETY: WB matches reserved System RAM. The host never writes this + // extent; observations invalidate covering lines before volatile reads. + let map = unsafe { io::mem::Mem::try_new(resource, io::mem::MemFlag::WB.into()) }?; + fence(Ordering::SeqCst); + Ok(Self { map, cache_line }) + } + + fn words(&self, offset: usize) -> [u64; N] { + // All callers use constructor-checked queue, scheduler or power extents. + let line = self.cache_line; + let start = offset & !(line - 1); + let end = (offset + N * 8 + line - 1) & !(line - 1); + for offset in (start..end).step_by(line) { + // SAFETY: Whole covering lines are inside the retained DATA view. + // The host never dirties them; do not clean stale data to firmware. + unsafe { core::arch::asm!("dc ivac, {addr}", + addr = in(reg) self.map.ptr().add(offset), + options(nostack, preserves_flags)) }; + } + // SAFETY: Complete invalidations before volatile observations. + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; + let mut out = [0; N]; + for (index, value) in out.iter_mut().enumerate() { + // SAFETY: The constructor bounds-checks every supported extent. + *value = unsafe { core::ptr::read_volatile( + self.map.ptr().add(offset + index * 8).cast::()) }; + } + fence(Ordering::Acquire); + out + } + + pub(super) fn queue_registration(&self, queue_id: u8) -> Result<([u64; 5], u64)> { + // TA/fragment own 10/11; application compute owns 12. The constructor + // bounds both complete tables through the last admitted queue. + if !(10..=12).contains(&queue_id) { return Err(EINVAL); } + Ok((self.words::<5>(0x127430 - 0x4c000 + queue_id as usize * 0x28), + self.words::<1>(0x1257e0 - 0x4c000 + queue_id as usize * 8)[0])) + } +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_job.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_job.rs new file mode 100644 index 0000000000000..0245811e9b5db --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_job.rs @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned TA/fragment resource graph for the bounded native-program test. +//! Construction leaves roots and queues unpublished. Submission wiring must +//! retain this owner through completion, faults and firmware shutdown. + +use super::{graphics_memory::M5GraphicsMemory, graphics_usc, MappedObject}; +use super::parameter_buffer::{M5ParameterBuffer, BLOCK_BYTES, PAGE_LIST_BYTES, PAGE_COUNT}; +use super::target_memory::M5TargetMemory; +use crate::{driver::AsahiDrmDevice, m5::render_desc, m5::render, mmu}; +use crate::m5::completion::MAX_GRAPHICS_GENERATION; +use kernel::prelude::*; + +const HIGH: u64 = 0xffff_fc20_c100_0000; +const LOW: u64 = 0x70_0030_0000; +const BYTES: usize = 0x40000; +const TA: usize = 0; +const FRAGMENT: usize = 0x4000; +const SCHEDULER: usize = 0x8000; +const SIGNALS_HIGH: u64 = HIGH + 0x80000; +const SUBMITTED: usize = 0; +const PM: usize = 0xc000; +const PM_SHARED: usize = 0x10000; +const PM_COUNTER: usize = 0x14000; +const PM_BLOCKS: usize = 0x18000; +const USC: usize = 0x1c000; +const USC_INNER: usize = 0x20000; +const USC_STATISTICS: usize = 0x24000; +const SCENES: usize = 0x30000; +const SCENE_ZERO: usize = SCENES + 80 * 0x80; +const TIMESTAMPS: usize = 0x38000; +const STAMPS_A: usize = 0x4000; +const STAMPS_B: usize = 0x8000; +const PAGE_LIST_HIGH: u64 = HIGH + 0x40000; +const METRICS_HIGH: u64 = HIGH + 0x60000; +const USER_BASE: u64 = 0x10_0000_0000; +pub(super) const PB_HARDWARE_ID: u32 = 0; +pub(super) const QUEUE_IDS: [u8; 2] = [10, 11]; +// Match Mesa's framebuffer dimensions; target allocations follow each size. +// Parameter-pool growth for more complex scenes remains separate. +use crate::m5::command::M5_MAX_RENDER_DIMENSION as MAX_APPLICATION_DIMENSION; +/// Retired targets kept for reuse. A video player cycles through a handful of +/// pass sizes every frame and a compositor's frames alternate with it, so +/// allocating per size change put several milliseconds on most jobs. +const TARGET_CACHE_ENTRIES: usize = 8; +/// Compute operands (128 KiB), a preemption page and a zero robustness page. +const COMPUTE_STORAGE_BYTES: usize = 0x28000; + +pub(super) struct M5GraphicsJob { + // The first owner's context retracts its roots before page release. + program: M5GraphicsMemory, + parameter_buffer: M5ParameterBuffer, + /// Storage of application compute, reserved with the parameter buffer so that its + /// address is free in every VM the job is handed to. + compute_storage: MappedObject, + target_memory: M5TargetMemory, + aliases: KVec, + // PB/USC pool views in a VM this job was handed to; empty for the first + // owner, whose pools were allocated in its own VM. + vm_aliases: KVec, + graph_alias: mmu::KernelMapping, + graph: MappedObject, + signals: MappedObject, + descriptor_template: KVec, + scene_template: KVec, + generation: u32, + target_shape: [u32; 5], + /// The VM `target_memory` lives in; a handoff leaves it behind. + target_vm: u64, + /// Retired targets by (VM, shape), least recently used first. + target_cache: KVec<(u64, [u32; 5], M5TargetMemory)>, +} + +impl M5GraphicsJob { + + /// Build a job from the submitting VM and translated application state. + /// No captured program is loaded; the caller supplies stage controls and + /// USC requirements derived for its program. Submission remains explicit. + #[allow(clippy::too_many_arguments)] + pub(super) fn from_vm( + dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, + parameters: render_desc::G17pRenderParameters, controls: render::M5RenderControls, + clusters: u32, usc_required: [u64; 2], + ) -> Result { + let program = M5GraphicsMemory::from_vm(dev, uat, vm)?; + Self::from_memory(dev, uat, program, parameters, controls, clusters, usc_required) + } + + #[allow(clippy::too_many_arguments)] + fn from_memory( + dev: &AsahiDrmDevice, uat: &mmu::Uat, program: M5GraphicsMemory, + mut p: render_desc::G17pRenderParameters, controls: render::M5RenderControls, + clusters: u32, usc_required: [u64; 2], + ) -> Result { + if !(1..=MAX_APPLICATION_DIMENSION).contains(&p.width) + || !(1..=MAX_APPLICATION_DIMENSION).contains(&p.height) + || p.layers != 1 || clusters != 1 { + return Err(ENOTSUPP); + } + let mut parameter_buffer = M5ParameterBuffer::new_reference(dev, &program)?; + let compute_storage = program.allocate_compact( + dev, COMPUTE_STORAGE_BYTES, 0x8000, mmu::PROT_GPU_SHARED_RW)?; + let target_memory = M5TargetMemory::new(dev, &program, &p, clusters)?; + target_memory.apply(&mut p); + p.cycle = parameter_buffer.scene_cycle(); + // Keep the PM register and firmware scene descriptor on the same + // owned metrics slot. The captured address is unmapped in a new VM. + p.record_index = render::parameter_buffer_metrics_record( + parameter_buffer.metrics() + 0x40).map_err(|_| EINVAL)?; + p.ta_hardware_buffer_id = PB_HARDWARE_ID; + p.usc_flist_hardware_buffer_id = graphics_usc::HARDWARE_ID; + p.ta_timestamp_start = HIGH + TIMESTAMPS as u64; + p.ta_timestamp_end = p.ta_timestamp_start + 8; + p.fragment_timestamp_start = p.ta_timestamp_start; + p.fragment_timestamp_end = p.ta_timestamp_end; + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), HIGH, + BYTES, mmu::PROT_FW_PRIV_RW)?; + // Native submitted totals and host-visible stamps are uncached in + // firmware. Keep them separate from the cached descriptor graph. + let mut signals = MappedObject::new_wc_at(dev, uat.kernel_vm(), SIGNALS_HIGH, + 0xc000, mmu::PROT_FW_SHARED_RW)?; + signals.write_u32(SIGNALS_HIGH + SUBMITTED as u64, 2)?; + let graph_alias = graph.map_alias_at(uat.kernel_lower_vm(), LOW, + mmu::PROT_GPU_SHARED_RW)?; + for offset in (0..BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(HIGH + offset as u64)? + != uat.kernel_lower_vm().translate_iova(LOW + offset as u64)? { + return Err(EFAULT); + } + } + let aliases = parameter_buffer.firmware_aliases( + uat, program.vm(), PAGE_LIST_HIGH, METRICS_HIGH, + )?; + let block_page_bases = parameter_buffer.block_page_bases()?; + graph.with_bytes_mut(|out| { + put64(out, SCHEDULER, SIGNALS_HIGH + SUBMITTED as u64); + put32(out, SCHEDULER + 0x10, 0x50); + // Two submitted stage records; firmware compares its completed + // count with this total, then unlinks the scheduler. + for (block, base) in block_page_bases.into_iter().enumerate() { + put32(out, PM_BLOCKS + block * 8, base); + } + put32(out, PM_SHARED, 3); + put32(out, PM_SHARED + 4, 3); + put32(out, PM_SHARED + 0x60, 1); + put32(out, PM_COUNTER, 1); + for (offset, value) in [(0xc, PB_HARDWARE_ID), (0x30, PAGE_LIST_BYTES as u32), + (0x34, PAGE_COUNT), (0x38, 1650), (0x3c, 3), (0x54, PAGE_COUNT - 1), + (0x58, BLOCK_BYTES as u32), (0x7c, 1650 * 16), (0x80, 550 * 16)] { + put32(out, PM + offset, value); + } + for (offset, address) in [(0x20, PAGE_LIST_HIGH), (0x28, parameter_buffer.page_list()), + (0x44, HIGH + PM_BLOCKS as u64), (0x4c, HIGH + PM_SHARED as u64), + (0x64, HIGH + PM_COUNTER as u64), (0x84, parameter_buffer.discard_offset())] { + put64(out, PM + offset, address); + } + // This sole job owns scene 1. M5 metrics use 64-byte spacing, + // unlike Neo dwords. Other reserved scenes remain unassigned; + // do not guess a scratch-ring wrap rule from a single draw. + let selected_scene = SCENES + 0x80; + put64(out, selected_scene, parameter_buffer.metrics() + 0x40); + put64(out, selected_scene + 8, METRICS_HIGH + 0x40); + put64(out, selected_scene + 0x28, parameter_buffer.scene_cycle()); + put64(out, selected_scene + 0x40, HIGH + PM_SHARED as u64 + 0x40); + put64(out, SCENES + 0x80 + 0x4c, 1); + put32(out, USC_INNER, 1); + program.usc().encode_initial_owner(HIGH + USC_INNER as u64, + &mut out[USC..USC + 0x70])?; + + Self::encode_descriptors(out, &p, &controls, program.context_id(), usc_required)?; + Ok(()) + })?; + let mut descriptor_template = KVec::new(); + let mut scene_template = KVec::new(); + graph.with_bytes_mut(|out| { + descriptor_template.extend_from_slice(&out[..SCHEDULER], GFP_KERNEL)?; + scene_template.extend_from_slice(&out[SCENES + 0x80..SCENES + 0x100], GFP_KERNEL)?; + Ok(()) + })?; + let target_vm = program.vm().id(); + Ok(Self { program, parameter_buffer, compute_storage, target_memory, aliases, + vm_aliases: KVec::new(), + graph_alias, graph, signals, descriptor_template, scene_template, generation: 1, + target_shape: [p.width, p.height, u32::from(p.layers), + u32::from(p.utile_width), u32::from(p.utile_height)], + target_vm, target_cache: KVec::new() }) + } + + #[allow(clippy::too_many_arguments)] + fn encode_descriptors( + out: &mut [u8], p: &render_desc::G17pRenderParameters, + controls: &render::M5RenderControls, context_id: u8, + usc_required: [u64; 2], + ) -> Result { + let objects = render_desc::RenderDescriptorObjects { + record_a: HIGH + SCHEDULER as u64, + shared: HIGH + PM as u64, + record_b: HIGH + SCENES as u64 + 0x80, + zero: HIGH + SCENE_ZERO as u64, + }; + let metadata = render_desc::RenderDescriptorMetadata { + context_id: context_id as u32, submission_ordinal: 0, command_buffer_state_word: 0, + ta_hardware_buffer_id: PB_HARDWARE_ID, submit_sequence: 1, + }; + render::build_ta_descriptor_base(p, controls, objects, metadata, + LOW + TA as u64, &mut out[TA..TA + render_desc::TA_DESCRIPTOR_SIZE]) + .map_err(|_| EINVAL)?; + render::build_fragment_descriptor_base(p, controls, objects, + render_desc::RenderDescriptorMetadata { submit_sequence: 0, ..metadata }, + LOW + FRAGMENT as u64, + &mut out[FRAGMENT..FRAGMENT + render_desc::FRAGMENT_DESCRIPTOR_SIZE]) + .map_err(|_| EINVAL)?; + let (ta, fragment) = out.split_at_mut(FRAGMENT); + let ta_usc = render::M5RenderUscState { + control: HIGH + USC as u64, required_bytes: usc_required[0], + allocated_bytes: graphics_usc::CAPACITY_BYTES, + statistics: HIGH + USC_STATISTICS as u64, + }; + render::apply_usc_submission_state(ta, fragment, ta_usc, + render::M5RenderUscState { + required_bytes: usc_required[1], statistics: ta_usc.statistics + 0x40, ..ta_usc + }).map_err(|_| EINVAL)?; + // This bounded first submission owns hardware stamp1 in each + // queue and event-machine timestamp0x100. Do not import native + // allocator history (the captured queues had stamps2/3). + put32(ta, 0x79c, QUEUE_IDS[1] as u32); + put64(ta, 0x7a0, 0x100); + put64(ta, 0x7a8, 1); + ta[0x7b0] = 1; + ta[0x7b1] = 1; + put64(ta, 0x8a6, SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[0] as u64 * 4); + put64(ta, 0x8ae, SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[0] as u64 * 4); + put32(ta, 0x8b6, 0x100); + put32(ta, 0x8ba, QUEUE_IDS[0] as u32); + put64(ta, 0x8c6, (1u64 << 40) | 1); + put64(fragment, 0x2140, SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[1] as u64 * 4); + put64(fragment, 0x2148, SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[1] as u64 * 4); + put32(fragment, 0x2150, 0x100); + put32(fragment, 0x2154, QUEUE_IDS[1] as u32); + put32(fragment, 0x215c, 0); + put64(fragment, 0x2160, (1u64 << 40) | 1); + // Fixed 256x256 reference controls independently observed in + // the successful 25G76 draw; these replace inherited Neo values. + ta[0x789] = 8; + ta[0x85f] = 7; + put32(ta, 0x892, 0); + ta[0x932] = 0; + put32(fragment, 0x1fb0, 8); + fragment[0x21cc] = 0; + fragment[0x222d] = 0; + Ok(()) + } + + pub(super) fn parameter_descriptor(&self) -> Result<[u32; 4]> { + render::initial_parameter_buffer_descriptor(self.parameter_buffer.page_list(), PAGE_COUNT) + .map_err(|_| EINVAL) + } + + pub(super) fn uma_descriptor(&self) -> [u64; 4] { self.program.usc().descriptor() } + + pub(super) fn roots(&self) -> [u64; 2] { self.program.roots() } + + pub(super) fn context_id(&self) -> u8 { self.program.context_id() } + + pub(super) fn owns_vm(&self, vm: &mmu::Vm) -> bool { + self.program.vm().id() == vm.id() + } + + /// Hand this fully retired job to another submitting VM. The hardware + /// context slot, descriptor graph, signals, queues and the PB/USC pools + /// all keep their firmware-visible addresses: the context is re-leased + /// for the new VM and the pools are aliased into it. The previous VM's + /// render targets are not used in the new VM: they stay cached under the + /// previous VM for its next turn, and the next application frame takes or + /// allocates targets in the new VM. The caller publishes the new roots + /// before the next draw. + pub(super) fn rebind_vm(&mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + if self.owns_vm(vm) { return Ok(()); } + if self.scheduler_status()? != [2, 0] { return Err(EBUSY); } + let progress = self.progress()?; + if progress[0] != 2 || progress[1] != self.generation * 0x100 + || progress[3] != self.generation * 0x100 { return Err(EBUSY); } + // Release an earlier handoff's views before reserving the same + // addresses in the incoming VM. + self.vm_aliases = KVec::new(); + let mut aliases = self.program.rebind(dev, vm)?; + for alias in self.parameter_buffer.alias_into(vm)? { + aliases.push(alias, GFP_KERNEL)?; + } + let storage = self.compute_storage.iova(); + if let Some(alias) = self.compute_storage.map_alias_at_with_guards( + vm, storage, mmu::PROT_GPU_SHARED_RW, 2)? { + aliases.push(alias, GFP_KERNEL)?; + } + self.vm_aliases = aliases; + dev_dbg!(dev.as_ref(), "M5_RENDER_VM_HANDOFF context={} generation={} vm={}\n", + self.program.context_id(), self.generation, vm.id()); + Ok(()) + } + + /// Compute shares this application's published roots and USC owner. Only + /// its operand/preemption storage is its own: the GPU address of that storage. + pub(super) fn compute_storage(&self) -> u64 { + self.compute_storage.iova() + } + + pub(super) fn descriptor_addresses(&self, stage: usize) -> Result<[u64; 2]> { + let offset = match stage { 0 => TA, 1 => FRAGMENT, _ => return Err(EINVAL) }; + Ok([HIGH + offset as u64, LOW + offset as u64]) + } + + pub(super) fn usc_control(&self) -> u64 { HIGH + USC as u64 } + + pub(super) fn parameter_state(&self) -> u64 { HIGH + PM as u64 } + + pub(super) fn publish_context(&mut self) -> Result { self.program.publish() } + + pub(super) fn quarantine(&self) { self.program.quarantine(); } + + pub(super) fn progress(&mut self) -> Result<[u32; 5]> { + Ok([self.signals.read_u32(SIGNALS_HIGH + SUBMITTED as u64)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[0] as u64 * 4)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[0] as u64 * 4)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[1] as u64 * 4)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[1] as u64 * 4)?]) + } + + pub(super) fn scheduler_status(&mut self) -> Result<[u32; 2]> { + // GFX cleans this 0xc8-byte object after unlinking the finished pair. + Ok([self.graph.read_u32(HIGH + SCHEDULER as u64 + 0xc)?, + self.graph.read_u32(HIGH + SCHEDULER as u64 + 0x94)?]) + } + + fn rebuild_retired_frame(&mut self, generation: u32) -> Result { + if generation != self.generation + 1 || generation > MAX_GRAPHICS_GENERATION + || self.scheduler_status()? != [2, 0] { + return Err(EBUSY); + } + let progress = self.progress()?; + if progress[0] != 2 || progress[1] != self.generation * 0x100 + || progress[3] != self.generation * 0x100 { return Err(EBUSY); } + self.graph.with_bytes_mut(|out| { + out[..SCHEDULER].copy_from_slice(&self.descriptor_template); + out[SCHEDULER..SCHEDULER + 0xc8].fill(0); + put64(out, SCHEDULER, SIGNALS_HIGH + SUBMITTED as u64); + put32(out, SCHEDULER + 0x10, 0x50); + out[SCENES + 0x80..SCENES + 0x100].copy_from_slice(&self.scene_template); + out[TIMESTAMPS..TIMESTAMPS + 16].fill(0); + let stamp = generation * 0x100; + put64(out, TA + 0x7a0, stamp as u64); + put64(out, TA + 0x7a8, generation as u64); + put32(out, TA + 0x8b6, stamp); + put64(out, TA + 0x8c6, (1u64 << 40) | generation as u64); + put32(out, FRAGMENT + 0x2150, stamp); + put64(out, FRAGMENT + 0x2160, (1u64 << 40) | generation as u64); + Ok(()) + })?; + // Leave previous stamps intact: they must be superseded by firmware, + // never host-written to the expected new completion value. + self.generation = generation; + Ok(()) + } + + /// Re-encode a retired application's descriptors without replaying its old + /// command pointers. The current bounded queue retains one VM. Target + /// storage can change only after complete retirement; the job + /// stamp keeps increasing and context replacement remains unsupported. + #[allow(clippy::too_many_arguments)] + pub(super) fn prepare_application_frame( + &mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm, mut p: render_desc::G17pRenderParameters, + controls: render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], generation: u32, + ) -> Result { + if vm.id() != self.program.vm().id() || clusters != 1 + || !(1..=MAX_APPLICATION_DIMENSION).contains(&p.width) + || !(1..=MAX_APPLICATION_DIMENSION).contains(&p.height) || p.layers != 1 + || generation != self.generation + 1 || generation > MAX_GRAPHICS_GENERATION { + return Err(ENOTSUPP); + } + // Validate retirement before allocating or releasing any target pages. + // The caller additionally checks both queues, stamps and event bits. + let progress = self.progress()?; + if self.scheduler_status()? != [2, 0] || progress[0] != 2 + || progress[1] != self.generation * 0x100 + || progress[3] != self.generation * 0x100 { return Err(EBUSY); } + let shape = [p.width, p.height, u32::from(p.layers), + u32::from(p.utile_width), u32::from(p.utile_height)]; + let vm_id = self.program.vm().id(); + let replacement = if (vm_id, shape) != (self.target_vm, self.target_shape) { + let cached = self.target_cache.iter() + .position(|(cached_vm, cached_shape, _)| (*cached_vm, *cached_shape) == (vm_id, shape)); + Some(match cached { + Some(index) => self.target_cache.remove(index).map_err(|_| EINVAL)?.2, + None => M5TargetMemory::new(dev, &self.program, &p, clusters).inspect_err(|e| pr_err!("M5: target memory failed: {:?}\n", e))?, + }) + } else { None }; + replacement.as_ref().unwrap_or(&self.target_memory).apply(&mut p); + p.cycle = self.parameter_buffer.scene_cycle(); + p.record_index = render::parameter_buffer_metrics_record( + self.parameter_buffer.metrics() + 0x40).map_err(|_| EINVAL)?; + p.ta_hardware_buffer_id = PB_HARDWARE_ID; + p.usc_flist_hardware_buffer_id = graphics_usc::HARDWARE_ID; + p.ta_timestamp_start = HIGH + TIMESTAMPS as u64; + p.ta_timestamp_end = p.ta_timestamp_start + 8; + p.fragment_timestamp_start = p.ta_timestamp_start; + p.fragment_timestamp_end = p.ta_timestamp_end; + let mut descriptors = KVec::new(); + descriptors.extend_from_slice(&self.descriptor_template, GFP_KERNEL)?; + Self::encode_descriptors(&mut descriptors, &p, &controls, + self.program.context_id(), usc_required).inspect_err(|e| pr_err!("M5: encode_descriptors failed: {:?}\n", e))?; + self.descriptor_template = descriptors; + // The application owns its output. It must never be erased by the + // native diagnostic's sentinel/reset logic. + self.rebuild_retired_frame(generation).inspect_err(|e| pr_err!("M5: rebuild_retired_frame failed: {:?}\n", e))?; + if let Some(target) = replacement { + dev_dbg!(dev.as_ref(), "M5_RENDER_TARGET_RESIZE generation={} old={:?} new={:?}\n", + generation, self.target_shape, shape); + let retired = core::mem::replace(&mut self.target_memory, target); + if self.target_cache.len() == TARGET_CACHE_ENTRIES { + self.target_cache.remove(0).map_err(|_| EINVAL)?; + } + self.target_cache.push((self.target_vm, self.target_shape, retired), GFP_KERNEL)?; + self.target_vm = vm_id; + self.target_shape = shape; + } + Ok(()) + } + +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +pub(super) fn triangle_parameters() -> (render_desc::G17pRenderParameters, render::M5RenderControls) { + let mut p = render_desc::G17pRenderParameters::source_rsrc8(); + p.aux_fb = 0x10000288000; + p.aux_fb_flags = 0x10c001; + p.aux_fb_page_count = 0x100000; + p.context_base = USER_BASE; + p.cycle = 0x220040; + p.deflake_1 = 0x1000068340; + p.deflake_2 = 0x1000068040; + p.deflake_3 = 0x1000068000; + p.depth_aux_buffer = 0; + p.depth_aux_stride = 0; + p.depth_buffer = 0; + p.depth_clear_value_bits = 0x3f800000; + p.depth_dimensions = 0; + p.depth_flags = 0; + p.depth_stride = 0; + p.encoder = 0x1000018000; + p.fragment_lifecycle = 0x4089f4c0508e74c; + p.fragment_status = 0x1000258000; + p.heapmeta = 0x1000261000; + p.width = 256; + p.height = 256; + p.layers = 1; + p.lifecycle = 0x4089f4c0508e74d; + p.load_pipeline = 0x10000170200; + p.load_pipeline_bind = 0xffff800000000040; + p.merge_upper_x_bits = 0x3bddb3d9; + p.merge_upper_y_bits = 0x3bddb3d9; + p.multisample_control = 0x88; + p.occlusion_query_base = 0; + p.ppp_control = 0x202; + p.record_index = 0x80041; + p.stencil_aux_buffer = 0; + p.stencil_aux_stride = 0; + p.stencil_buffer = 0; + p.stencil_clear_value = 0; + p.stencil_stride = 0; + p.store_pipeline = 0x10000170400; + p.store_pipeline_bind = 0; + p.ta_hardware_buffer_id = PB_HARDWARE_ID; + p.ta_status = 0x1000078000; + p.tib_blocks = 0; + p.tile_config = 0x10280; + p.tilemap = 0x1000260000; + p.tpc = 0x1000288000; + p.usc_flist_hardware_buffer_id = graphics_usc::HARDWARE_ID; + p.utile_config = 0xa000; + p.utile_height = 32; + p.utile_width = 32; + p.absolute_pointers = false; + p.partial_store_pipeline_bind = 0; + p.partial_store_pipeline = 0x10000170400; + p.partial_load_pipeline_bind = 0xffff800000000040; + p.partial_load_pipeline = 0x10000170000; + p.depth_bias_array = 0x100002d8000; + p.scissor_array = 0x10000180000; + p.ta_timestamp_start = HIGH + TIMESTAMPS as u64; + p.ta_timestamp_end = p.ta_timestamp_start + 8; + p.fragment_timestamp_start = p.ta_timestamp_start; + p.fragment_timestamp_end = p.ta_timestamp_end; + let controls = render::M5RenderControls { + ta: render::M5RenderStageControls { + resource_flags: 0x4054b, resource_control: 0x200000001, ptc_max: 0xc6, work_key: 0x300, + }, + fragment: render::M5RenderStageControls { + resource_flags: 0x104054b, resource_control: 0x200000001, ptc_max: 0xce, work_key: 0x300, + }, + ta_tile_control: 0x10c08860, ta_dispatch_control: 0x700, + fragment_raster_control: 0x200000ff0200, fragment_tile_flags: 0x200003717f, + store_pipeline_2_bind: 0, store_pipeline_2: 0x10000170400, + partial_store_pipeline_2_bind: 0, partial_store_pipeline_2: 0x10000170400, + partial_tile_flags: 0x200000, + }; + (p, controls) +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_memory.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_memory.rs new file mode 100644 index 0000000000000..cf9a2f93efb03 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_memory.rs @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained M5 render VM and private storage, with an optional native test program. +//! +//! Allocation is separate from root publication and queue submission. A future +//! job owner keeps this graph alive through queue completion. After a fault, +//! ownership remains retained until both firmware CPUs stop. Captured physical +//! backing is never installed. + +use super::MappedObject; +use super::graphics_usc::M5GraphicsUsc; +use crate::{driver::AsahiDrmDevice, mmu}; + +/// Pages of a client VM aliased into the context, at most this many. +const PAGE_BYTES: usize = 0x4000; +const MAX_PAGES: usize = 384; +use kernel::prelude::*; + +pub(super) struct M5GraphicsMemory { + // Retract any published roots before dropping their mapping owners. + context: mmu::M5ExecutionContext, + // Keep user mappings and GEM backing across handle close/VM teardown. + // This guard's bookkeeping is generation-neutral despite its old name. + user_mappings: Option, + pages: KVec, + usc: M5GraphicsUsc, + output_first: Option, +} + +impl M5GraphicsMemory { + + /// Keep the actual submitting VM and allocate only driver-owned storage. + /// Application shader/input/output mappings remain in that VM; no native + /// program firmware or captured pages are requested by this constructor. + pub(super) fn from_vm(dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm) -> Result { + crate::m5::vm_layout::compact_range(&vm.kernel_range()).ok_or(EINVAL)?; + let user_mappings = vm.retain_job()?; + let context = uat.new_m5_execution_context(vm)?; + let usc = M5GraphicsUsc::new_application(dev, context.vm())?; + Ok(Self { context, user_mappings: Some(user_mappings), pages: KVec::new(), usc, output_first: None }) + } + + /// Hand the retained USC pool and this context slot to another + /// application VM. The previous roots are retracted first (with ASID + /// invalidation), the same hardware context is re-leased for `vm`, and + /// the pool is aliased at unchanged addresses, so every firmware record + /// that names the context or the pool keeps its value. The caller has + /// verified that the previous owner's work is completely retired and + /// publishes the new roots explicitly afterwards. + pub(super) fn rebind(&mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm) + -> Result> { + crate::m5::vm_layout::compact_range(&vm.kernel_range()).ok_or(EINVAL)?; + // Only application owners hand over; the native diagnostic program + // keeps captured pages in its private VM. + if !self.pages.is_empty() || self.output_first.is_some() { return Err(ENOTSUPP); } + let user_mappings = vm.retain_job()?; + self.context.rebind(vm)?; + // Release the previous VM's mapping hold only after its roots are gone. + self.user_mappings = Some(user_mappings); + let aliases = self.usc.alias_into(vm)?; + dev_dbg!(dev.as_ref(), "M5_GRAPHICS_CONTEXT id={} rebound vm={} roots={:x?}; USC pool aliased\n", + self.context.context_id(), vm.id(), self.context.roots()); + Ok(aliases) + } + + /// Return an owning allocation in the compact TA resource arena. The + /// render job retains it until completion or confirmed processor stop. + pub(super) fn allocate_compact( + &self, dev: &AsahiDrmDevice, size: usize, alignment: u64, prot: mmu::Prot, + ) -> Result { + let range = crate::m5::vm_layout::compact_range(&self.vm().kernel_range()).ok_or(EINVAL)?; + MappedObject::new_wc_in_range_with_guards( + dev, self.vm(), range, size, alignment, prot, 2, + ) + } + + /// Job construction can add its own objects before root publication. + pub(super) fn vm(&self) -> &mmu::Vm { + self.context.vm() + } + + pub(super) fn context_id(&self) -> u8 { self.context.context_id() } + + pub(super) fn roots(&self) -> [u64; 2] { + self.context.roots() + } + + pub(super) fn usc(&self) -> &M5GraphicsUsc { + &self.usc + } + + /// A stop request is not proof that hardware has released these mappings. + /// Close VM_BIND admission before any future error fence can wake userspace. + pub(super) fn quarantine(&self) { + if let Some(guard) = &self.user_mappings { + guard.quarantine(); + } + } + + /// Alias existing owned user pages into the firmware VM. PM page lists + /// and scene counters must share backing with their user GPU views. + /// The returned mappings belong to the eventual retained job owner. + pub(super) fn alias_kernel_pages( + &mut self, + uat: &mmu::Uat, + user_address: u64, + kernel_address: u64, + bytes: usize, + ) -> Result> { + if self.context.is_current() { + return Err(EBUSY); + } + if bytes == 0 + || bytes > MAX_PAGES * PAGE_BYTES + || bytes % PAGE_BYTES != 0 + || (user_address | kernel_address) & (PAGE_BYTES as u64 - 1) != 0 + { + return Err(EINVAL); + } + user_address.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + kernel_address.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + let mut indices = KVec::new(); + for offset in (0..bytes).step_by(PAGE_BYTES) { + let address = user_address + offset as u64; + let index = self + .pages + .iter() + .position(|page| page.iova() == address) + .ok_or(EFAULT)?; + indices.push(index, GFP_KERNEL)?; + } + let mut aliases = KVec::new(); + for (ordinal, index) in indices.iter().enumerate() { + let high = kernel_address + (ordinal * PAGE_BYTES) as u64; + let alias = + self.pages[*index].map_alias_at(uat.kernel_vm(), high, mmu::PROT_FW_SHARED_RW)?; + if uat.kernel_vm().translate_iova(high)? + != self + .context + .vm() + .translate_iova(self.pages[*index].iova())? + { + return Err(EFAULT); + } + aliases.push(alias, GFP_KERNEL)?; + } + Ok(aliases) + } + + /// Read a word from a retained reference page while preparing its job. + pub(super) fn read_word(&mut self, address: u64) -> Result { + if address & 3 != 0 { + return Err(EINVAL); + } + let base = address & !(PAGE_BYTES as u64 - 1); + let page = self + .pages + .iter_mut() + .find(|page| page.iova() == base) + .ok_or(EFAULT)?; + page.read_u32(address) + } + + /// Reset owned PM outputs before exposing the context to either firmware. + pub(super) fn clear_pages(&mut self, address: u64, bytes: usize) -> Result { + if self.context.is_current() || bytes == 0 || bytes % PAGE_BYTES != 0 + || address & (PAGE_BYTES as u64 - 1) != 0 { + return Err(EINVAL); + } + let end = address.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + let count = self.pages.iter().filter(|p| p.iova() >= address && p.iova() < end).count(); + if count != bytes / PAGE_BYTES { return Err(EFAULT); } + for page in self.pages.iter_mut().filter(|p| p.iova() >= address && p.iova() < end) { + page.with_bytes_mut(|out| { out.fill(0); Ok(()) })?; + } + Ok(()) + } + + pub(super) fn write_word(&mut self, address: u64, value: u32) -> Result { + if self.context.is_current() || address & 3 != 0 { return Err(EINVAL); } + let base = address & !(PAGE_BYTES as u64 - 1); + let offset = (address - base) as usize; + let page = self.pages.iter_mut().find(|p| p.iova() == base).ok_or(EFAULT)?; + page.with_bytes_mut(|out| { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + Ok(()) + }) + } + + /// Call only after the complete TA/3D resource graph is prepared. + pub(super) fn publish(&mut self) -> Result { + self.context.publish() + } + + pub(super) fn is_current(&self) -> bool { + self.context.is_current() + } + +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_queues.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_queues.rs new file mode 100644 index 0000000000000..02850fbe073df --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_queues.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Paired first-submission queues for the retained M5 graphics experiment. + +use super::{graphics_job::{M5GraphicsJob, PB_HARDWARE_ID, QUEUE_IDS}, graphics_usc, + firmware_view::FirmwareDataView, MappedObject}; +use crate::{driver::AsahiDrmDevice, m5::submission, m5::channel, m5::sksm, mmu}; +use crate::m5::completion::{graphics_pointer_head, GRAPHICS_POINTER_COUNT, + MAX_GRAPHICS_GENERATION}; +use kernel::prelude::*; + +const HIGH: u64 = 0xffff_fc20_c000_0000; +const ENTRY_HIGH: u64 = 0xffff_fc20_0039_0000; +const ENTRY_LOW: u64 = 0x70_005f_0000; +const ENTRY_BYTES: usize = 0x20000; +const CONTROL: usize = 0x18000; +// Native TA and fragment channel+0x10 both point to fffffc2000000048. +// One scheduler belongs to one intrusive list even when two SKSM queues run it. +pub(super) const JOB_LIST: u64 = HIGH + 0x20000; +const POINTERS: [usize; 2] = [0x4000, 0xc000]; +const ITEMS: [usize; 2] = [0x8000, 0x10000]; +const CONFIG: [usize; 2] = [0xa800, 0x12800]; + +pub(super) struct M5GraphicsQueues { + aliases: KVec, + entries: KVec, + graph: MappedObject, + job_list: MappedObject, + firmware: FirmwareDataView, +} + +impl M5GraphicsQueues { + pub(super) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, job: &M5GraphicsJob) -> Result { + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), HIGH, + 0x20000, mmu::PROT_FW_PRIV_RW)?; + let mut job_list = MappedObject::new_wc_at(dev, uat.kernel_vm(), JOB_LIST, + 0x4000, mmu::PROT_FW_SHARED_RW)?; + job_list.with_bytes_mut(|out| { put64(out, 8, JOB_LIST); Ok(()) })?; + let mut entries = KVec::new(); + let mut aliases = KVec::new(); + for stage in 0..2 { + let id = QUEUE_IDS[stage]; + let high = ENTRY_HIGH + (stage * ENTRY_BYTES) as u64; + let low = ENTRY_LOW + (stage * ENTRY_BYTES) as u64; + let mut entry = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, + ENTRY_BYTES, mmu::PROT_FW_PRIV_RW)?; + let alias = entry.map_alias_at(uat.kernel_lower_vm(), low, mmu::PROT_GPU_SHARED_RW)?; + for offset in (0..ENTRY_BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset as u64)? + != uat.kernel_lower_vm().translate_iova(low + offset as u64)? { + return Err(EFAULT); + } + } + let [descriptor, _] = job.descriptor_addresses(stage)?; + let dm = if stage == 0 { sksm::M5DataMaster::Tiling } + else { sksm::M5DataMaster::Fragment }; + let prepared = Self::prepare_entry(job, stage, 1)?; + entry.with_bytes_mut(|out| { + out[prepared.offset as usize..prepared.offset as usize + prepared.bytes.len()] + .copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let install = channel::encode_m5_channel_config_for(channel::M5ChannelConfig { + ring_low: low, ring_high: high, queue_id: id, priority: 2, + install_queue: true, gpu_context: job.context_id(), event: None, + channel_control: HIGH + CONTROL as u64, update_scheduler: true, + scheduler_index: 0, channel_stamp_index: stage as u32, + update_context: false, owner_context: 0, extended_command: false, + }, stage as u32, 2, None, None).map_err(|_| EINVAL)?; + let config = channel::encode_m5_channel_config_for(channel::M5ChannelConfig { + ring_low: low, ring_high: high, queue_id: id, priority: 2, + install_queue: false, gpu_context: job.context_id(), + event: Some(channel::M5ChannelEvent { address: job.usc_control(), + timestamp: 1, index: graphics_usc::HARDWARE_ID }), + channel_control: HIGH + CONTROL as u64, update_scheduler: false, + scheduler_index: 0, channel_stamp_index: stage as u32, + update_context: true, owner_context: 1, extended_command: false, + }, stage as u32, 2, Some(0), if stage == 0 { + Some(channel::M5ParameterBufferEvent { address: job.parameter_state(), + timestamp: 1, hardware_buffer_id: PB_HARDWARE_ID, manager_id: 0 }) + } else { None }).map_err(|_| EINVAL)?; + let kick = sksm::encode_m5_sksm_add_kicks_for(dm, id, 2, 1, 1) + .map_err(|_| EINVAL)?; + graph.with_bytes_mut(|out| { + let pointers = POINTERS[stage]; + let queue = id as usize * 0xc0; + let reset = submission::encode_m5_25g76_channel_state_reset( + submission::M5ChannelStateResetValues { + uncached_state: HIGH + pointers as u64, + command_pointer_ring: HIGH + ITEMS[stage] as u64, + gpu_va_10: JOB_LIST, word_48: 0, + gpu_va_9c: HIGH + CONTROL as u64, + }); + out[queue..queue + 0xc0].copy_from_slice(&reset); + for offset in [0x28, 0x2c, 0x40] { put32(out, queue + offset, 2); } + out[queue + 0x36..queue + 0x38].fill(0xff); + put32(out, pointers + 0x50, u32::MAX); + put32(out, pointers + 0x60, 0x500); + out[CONFIG[stage]..CONFIG[stage] + install.len()].copy_from_slice(&install); + out[CONFIG[stage] + 0x100..CONFIG[stage] + 0x100 + config.len()].copy_from_slice(&config); + out[CONFIG[stage] + 0x200..CONFIG[stage] + 0x200 + kick.len()].copy_from_slice(&kick); + for (index, address) in [HIGH + CONFIG[stage] as u64, descriptor, + HIGH + CONFIG[stage] as u64 + 0x100, + HIGH + CONFIG[stage] as u64 + 0x200].into_iter().enumerate() { + put64(out, ITEMS[stage] + index * 8, address); + } + // Publish only installation. The draw stays beyond the producer + // until both firmware queue slots and scheduler bindings verify. + put32(out, pointers + 0x40, 1); + Ok(()) + })?; + aliases.push(alias, GFP_KERNEL)?; + entries.push(entry, GFP_KERNEL)?; + } + graph.with_bytes_mut(|out| { + out[CONTROL..CONTROL + 2].fill(0xff); + out[CONTROL + 5] = 1; + out[CONTROL + 0x26] = 2; + out[CONTROL + 0x33] = 0xff; + Ok(()) + })?; + Ok(Self { aliases, entries, graph, job_list, firmware: FirmwareDataView::new(dev)? }) + } + + pub(super) fn queue_address(stage: usize) -> Result { + Ok(HIGH + *QUEUE_IDS.get(stage).ok_or(EINVAL)? as u64 * 0xc0) + } + + fn prepare_entry(job: &M5GraphicsJob, stage: usize, generation: u32) + -> Result { + if !(1..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + let id = *QUEUE_IDS.get(stage).ok_or(EINVAL)?; + let [descriptor, descriptor_low] = job.descriptor_addresses(stage)?; + let stamp = generation as u64; + let barrier = [sksm::M5SksmDependency { queue_id: QUEUE_IDS[0], stamp, kind: 1 }]; + let mut rce = [sksm::M5SksmRce { address: 0, tag: 0 }; 4]; + if stage == 0 { + rce[0] = sksm::M5SksmRce { address: descriptor_low + 0x60, tag: 79 }; + } else { + for (index, (offset, count)) in [(0xa0, 87), (0x7c0, 18), + (0xee0, 25), (0x1600, 12)].into_iter().enumerate() { + rce[index] = sksm::M5SksmRce { address: descriptor_low + offset, tag: count }; + } + } + sksm::prepare_m5_sksm_entry(sksm::M5SksmQueue { + id, priority: 2, current_stamp: stamp, submitted: 0, + previous_stamp: if generation == 1 { None } else { Some(stamp - 1) }, + }, sksm::M5SksmOperands { + barriers: if stage == 0 { &[] } else { &barrier }, first_predecessor: 0, + implicit_dependency_type: 1, mcache: None, flag_bit1: false, + flag_bit61: false, flag_bit63: false, + payload: [descriptor, Self::queue_address(stage)?], + qos_byte_32: 3, qos_class_40: if stage == 0 { 12 } else { 24 }, + qos_word_48: 0xffff, event_mask: [0; 4], rce_kind: 0, rce, + auxiliary: 0x3f_ffff_ffff_ffff, + }).map_err(|_| EINVAL) + } + + pub(super) fn publish_next_draw(&mut self, job: &M5GraphicsJob, generation: u32) -> Result { + if !(2..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + let previous_head = graphics_pointer_head(generation - 1); + if self.counters()? != [[previous_head; 3]; 2] || !self.installed()? + || self.completion_list()? != [0, JOB_LIST] { return Err(EBUSY); } + for stage in 0..2 { + let prepared = Self::prepare_entry(job, stage, generation)?; + self.entries[stage].with_bytes_mut(|out| { + let offset = prepared.offset as usize; + let slot = &mut out[offset..offset + prepared.bytes.len()]; + // Every preceding job has retired, including the prior user + // of this 256-entry SKSM slot. Preserve the first-use check. + if generation < 256 && slot.iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + slot.copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let dm = if stage == 0 { sksm::M5DataMaster::Tiling } + else { sksm::M5DataMaster::Fragment }; + let kick = sksm::encode_m5_sksm_add_kicks_for(dm, QUEUE_IDS[stage], + 2, generation as u64, 1).map_err(|_| EINVAL)?; + self.graph.with_bytes_mut(|out| { + out[CONFIG[stage] + 0x200..CONFIG[stage] + 0x200 + kick.len()] + .copy_from_slice(&kick); + let addresses = [job.descriptor_addresses(stage)?[0], + HIGH + CONFIG[stage] as u64 + 0x100, + HIGH + CONFIG[stage] as u64 + 0x200]; + for (index, address) in addresses.into_iter().enumerate() { + let absolute = 1 + (generation as usize - 1) * 3 + index; + let offset = ITEMS[stage] + (absolute % GRAPHICS_POINTER_COUNT as usize) * 8; + if absolute < GRAPHICS_POINTER_COUNT as usize + && out[offset..offset + 8].iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + put64(out, offset, address); + } + Ok(()) + })?; + } + for pointers in POINTERS { + self.graph.write_u32(HIGH + pointers as u64 + 0x40, + graphics_pointer_head(generation))?; + } + Ok(()) + } + + pub(super) fn counters(&mut self) -> Result<[[u32; 3]; 2]> { + let mut result = [[0; 3]; 2]; + for stage in 0..2 { + for (index, offset) in [0, 0x30, 0x40].into_iter().enumerate() { + result[stage][index] = self.graph.read_u32(HIGH + (POINTERS[stage] + offset) as u64)?; + } + } + Ok(result) + } + + pub(super) fn completion_list(&mut self) -> Result<[u64; 2]> { + Ok([self.job_list.read_u64(JOB_LIST)?, self.job_list.read_u64(JOB_LIST + 8)?]) + } + + pub(super) fn installation_status(&mut self) -> Result<([[u64; 5]; 2], [u64; 2], [u64; 2])> { + let mut slots = [[0; 5]; 2]; + let mut pointers = [0; 2]; + for stage in 0..2 { + (slots[stage], pointers[stage]) = self.firmware.queue_registration(QUEUE_IDS[stage])?; + } + Ok((slots, pointers, [u64::from(self.graph.read_u32(HIGH + CONTROL as u64)?), + self.graph.read_u64(HIGH + CONTROL as u64 + 0x12)?])) + } + + pub(super) fn installed(&mut self) -> Result { + let (slots, pointers, control) = self.installation_status()?; + for stage in 0..2 { + let slot = slots[stage]; + let offset = (stage * ENTRY_BYTES) as u64; + // GFX 0x12240/0x1241c copies tag15 +0x22 to slot +0x20: + // this word is the TA/fragment stage, not the outer channel. + if slot[0] & 1 == 0 || slot[1] != ENTRY_HIGH + offset + || slot[2] != ENTRY_LOW + offset || slot[3] != 4u64 << 32 + || slot[4] as u32 != stage as u32 || pointers[stage] != HIGH + CONTROL as u64 { + return Ok(false); + } + } + Ok(control[0] & 0xff != 0xff && (control[0] >> 8) & 0xff != 0xff + && (control[0] >> 16) & 0xff == 4 + && control[1] == ((1u64 << QUEUE_IDS[0]) | (1u64 << QUEUE_IDS[1]))) + } + + pub(super) fn publish_draw(&mut self) -> Result { + if self.counters()? != [[1; 3]; 2] || !self.installed()? { return Err(EBUSY); } + for pointers in POINTERS { + self.graph.write_u32(HIGH + pointers as u64 + 0x40, 4)?; + } + Ok(()) + } +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_usc.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_usc.rs new file mode 100644 index 0000000000000..1d5bd727ee719 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_usc.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained USC backing in an M5 application's reserved driver VA region. +//! +//! Firmware records are encoded from the owned mappings, never fixed test +//! addresses. Guard pages belong to each mapping's allocator reservation. +//! No root or firmware table is published here. + +use super::MappedObject; +use crate::{driver::AsahiDrmDevice, mmu}; +use kernel::prelude::*; + +pub(super) const HARDWARE_ID: u32 = 1; +const DIRECTORY_ENTRIES: usize = 0x80000; +const BLOCK_BYTES: usize = 0x20_0000; +const BLOCK_COUNT: usize = 19; +const UNIT_BYTES: usize = 0x1000; +const UNIT_COUNT: usize = BLOCK_COUNT * BLOCK_BYTES / UNIT_BYTES; +pub(super) const CAPACITY_BYTES: u64 = (BLOCK_COUNT * BLOCK_BYTES) as u64; + +pub(super) struct M5GraphicsUsc { + directory: MappedObject, + freelist: MappedObject, + blocks: KVec, +} + +impl M5GraphicsUsc { + pub(super) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + Self::new_in_range(dev, vm, vm.kernel_range()) + } + + pub(super) fn new_application(dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let range = crate::m5::vm_layout::auxiliary_range(&vm.kernel_range()).ok_or(EINVAL)?; + Self::new_in_range(dev, vm, range) + } + + fn new_in_range( + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: core::ops::Range, + ) -> Result { + let allocate = |size, guard_pages| MappedObject::new_wc_in_range_with_guards( + dev, vm, range.clone(), size, 0x8000, mmu::PROT_GPU_SHARED_RW, guard_pages, + ); + let mut directory = allocate(DIRECTORY_ENTRIES * 8, 2)?; + let mut freelist = allocate(mmu::UAT_PGSZ, 1)?; + let mut blocks = KVec::new(); + for _ in 0..BLOCK_COUNT { + blocks.push(allocate(BLOCK_BYTES, 2)?, GFP_KERNEL)?; + } + directory.with_bytes_mut(|out| { + for unit in 0..UNIT_COUNT { + let address = Self::unit_address(&blocks, unit).ok_or(EINVAL)?; + out[unit * 8..unit * 8 + 8].copy_from_slice(&address.to_le_bytes()); + } + Ok(()) + })?; + freelist.with_bytes_mut(|out| { + // Same initial free-block format as the qualified M5 compute + // pool. Every allocation here is free; no native growth history + // or in-use allocator contents are imported. + for block in 0..BLOCK_COUNT { + let address = blocks[block].iova(); + out[block * 0x40..block * 0x40 + 8] + .copy_from_slice(&(address | (1u64 << 61)).to_le_bytes()); + } + Ok(()) + })?; + let mut result = Self { directory, freelist, blocks }; + result.validate(vm)?; + dev_dbg!(dev.as_ref(), + "M5_GRAPHICS_USC reserved={:#x}..{:#x} directory={:#x} freelist={:#x} blocks={} capacity={:#x}; guard pages reserved\n", + range.start, range.end, result.directory.iova(), result.freelist.iova(), + BLOCK_COUNT, CAPACITY_BYTES); + Ok(result) + } + + fn unit_address(blocks: &[MappedObject], unit: usize) -> Option { + if unit >= UNIT_COUNT { return None; } + let units_per_block = BLOCK_BYTES / UNIT_BYTES; + Some(blocks.get(unit / units_per_block)?.iova() + + (unit % units_per_block * UNIT_BYTES) as u64) + } + + pub(super) fn descriptor(&self) -> [u64; 4] { + // M5 halLoadUMADesc: 4-KiB population, not 16-KiB CPU page count. + [self.directory.iova() >> 7, (UNIT_COUNT as u64) << 32, + (DIRECTORY_ENTRIES as u64) << 32 | UNIT_COUNT as u64, 0] + } + + /// Initialize a fresh owner using the qualified M5 initial-state recipe. + /// Native graphics snapshots include later growth generations; copying + /// their seeds/disabled flag would disagree with these all-free blocks. + pub(super) fn encode_initial_owner(&self, inner: u64, out: &mut [u8]) -> Result { + if out.len() < 0x70 || inner < 0xffff_fc00_0000_0000 || inner & 7 != 0 { + return Err(EINVAL); + } + out[..0x70].fill(0); + for (offset, value) in [(0, 1u32), (8, HARDWARE_ID), (0x10, 2), + (0x1c, DIRECTORY_ENTRIES as u32), (0x24, UNIT_COUNT as u32), + (0x2c, UNIT_COUNT as u32), (0x40, 4), (0x48, (BLOCK_COUNT * 8) as u32), + (0x5c, 1), (0x60, 2)] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, address) in [(0x14, self.directory.iova()), + (0x30, self.freelist.iova()), (0x4c, inner)] { + out[offset..offset + 8].copy_from_slice(&address.to_le_bytes()); + } + Ok(()) + } + + /// Map the retained pool into another application VM at the same + /// addresses, reproducing each allocation's guard reservation, so every + /// firmware record naming the directory, free list or blocks stays valid + /// across a render client handoff. The returned aliases must be kept for + /// as long as that VM may submit. + pub(super) fn alias_into(&mut self, vm: &mmu::Vm) -> Result> { + let range = crate::m5::vm_layout::auxiliary_range(&vm.kernel_range()).ok_or(EINVAL)?; + let mut aliases = KVec::new(); + let alias = |object: &mut MappedObject, size: usize, guards: usize, + aliases: &mut KVec| -> Result { + let end = object.iova().checked_add((size + guards * mmu::UAT_PGSZ) as u64) + .ok_or(EOVERFLOW)?; + if object.iova() < range.start || end > range.end { return Err(ERANGE); } + if let Some(mapping) = object.map_alias_at_with_guards( + vm, object.iova(), mmu::PROT_GPU_SHARED_RW, guards)? { + aliases.push(mapping, GFP_KERNEL)?; + } + Ok(()) + }; + alias(&mut self.directory, DIRECTORY_ENTRIES * 8, 2, &mut aliases)?; + alias(&mut self.freelist, mmu::UAT_PGSZ, 1, &mut aliases)?; + for block in self.blocks.iter_mut() { + alias(block, BLOCK_BYTES, 2, &mut aliases)?; + } + // Only the mappings are checked here: the firmware has been + // advancing the free-ring cursors and returning directory units + // while the pool was in use, so the construction-time contents no + // longer hold and must not be re-asserted on handoff. + self.validate_mappings(vm)?; + Ok(aliases) + } + + /// Verify every CPU page named by the populated 4-KiB directory and the + /// free-block heads. The allocator keeps the underlying mappings alive. + pub(super) fn validate(&mut self, vm: &mmu::Vm) -> Result { + self.validate_mappings(vm)?; + self.validate_contents() + } + + /// Range, alignment, guard-gap and translation checks for every pool + /// object in `vm`; valid before and after firmware use of the pool. + fn validate_mappings(&self, vm: &mmu::Vm) -> Result { + if self.blocks.len() != BLOCK_COUNT { return Err(EFAULT); } + let range = vm.kernel_range(); + let validate_guards = |object: &MappedObject, size: usize, guards: usize| -> Result { + let end = object.iova().checked_add((size + guards * mmu::UAT_PGSZ) as u64) + .ok_or(EOVERFLOW)?; + if object.iova() < range.start || end > range.end + || object.iova() & 0x7fff != 0 { return Err(EFAULT); } + // Check after every allocation has completed: a later neighbor + // must not have consumed either reserved 16-KiB guard page. + for guard in 0..guards { + if vm.translate_iova(object.iova() + (size + guard * mmu::UAT_PGSZ) as u64).is_ok() { + return Err(EFAULT); + } + } + Ok(()) + }; + validate_guards(&self.directory, DIRECTORY_ENTRIES * 8, 2)?; + validate_guards(&self.freelist, mmu::UAT_PGSZ, 1)?; + for block in &self.blocks { validate_guards(block, BLOCK_BYTES, 2)?; } + let translate_all = |object: &MappedObject| -> Result { + for offset in (0..object.logical_size).step_by(mmu::UAT_PGSZ) { + vm.translate_iova(object.iova() + offset as u64)?; + } + Ok(()) + }; + translate_all(&self.directory)?; + translate_all(&self.freelist)?; + for block in &self.blocks { translate_all(block)?; } + Ok(()) + } + + /// Construction-time contents: an all-free block list and a directory + /// that names every unit in order. Firmware use changes both. + fn validate_contents(&mut self) -> Result { + for block in 0..BLOCK_COUNT { + let base = self.blocks[block].iova(); + if self.freelist.read_u64(self.freelist.iova() + (block * 0x40) as u64)? + != (base | (1u64 << 61)) { + return Err(EFAULT); + } + } + for unit in 0..UNIT_COUNT { + if self.directory.read_u64(self.directory.iova() + (unit * 8) as u64)? + != Self::unit_address(&self.blocks, unit).ok_or(EFAULT)? { + return Err(EFAULT); + } + } + if self.directory.read_u64(self.directory.iova() + (UNIT_COUNT * 8) as u64)? != 0 { + return Err(EFAULT); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m5/resources/parameter_buffer.rs b/drivers/gpu/drm/asahi/m5/resources/parameter_buffer.rs new file mode 100644 index 0000000000000..3c6b955b6c799 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/parameter_buffer.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! VM-owned parameter buffer for the qualified 256x256 M5 render profile. +//! +//! The allocation addresses are dynamic. Capacity and scene layout remain +//! the witnessed native profile; larger renders and pool growth are separate. + +use super::{graphics_memory::M5GraphicsMemory, MappedObject}; +use crate::{driver::AsahiDrmDevice, mmu, m5::vm_layout}; +use kernel::prelude::*; + +pub(super) const BLOCK_BYTES: usize = 0x80000; +pub(super) const PAGE_LIST_BYTES: usize = 0x1c000; +pub(super) const PAGE_COUNT: u32 = 48; + +pub(super) struct M5ParameterBuffer { + blocks: [MappedObject; 3], + page_list: MappedObject, + metrics: MappedObject, + scene_scratch: MappedObject, + discard: MappedObject, +} + +impl M5ParameterBuffer { + pub(super) fn new_reference(dev: &AsahiDrmDevice, memory: &M5GraphicsMemory) -> Result { + let allocate = |size| memory.allocate_compact( + dev, size, 0x8000, mmu::PROT_GPU_SHARED_RW, + ); + let mut owner = Self { + blocks: [allocate(BLOCK_BYTES)?, allocate(BLOCK_BYTES)?, allocate(BLOCK_BYTES)?], + page_list: allocate(PAGE_LIST_BYTES)?, + metrics: allocate(0x4000)?, + scene_scratch: allocate(0x4000)?, + discard: allocate(0x8000)?, + }; + let block_bases = owner.block_page_bases()?; + owner.page_list.with_bytes_mut(|out| { + for (block, first) in block_bases.into_iter().enumerate() { + for page in 0..16 { + let offset = (block * 16 + page) * 4; + out[offset..offset + 4].copy_from_slice(&(first + page as u32).to_le_bytes()); + } + } + Ok(()) + })?; + owner.validate(memory.vm())?; + dev_dbg!(dev.as_ref(), + "M5_PARAMETER_BUFFER blocks={:x?} page_list={:#x} metrics={:#x} scene={:#x} discard={:#x}; compact mappings owned\n", + owner.blocks.each_ref().map(|b| b.iova()), owner.page_list(), owner.metrics(), + owner.scene_scratch.iova(), owner.discard.iova()); + Ok(owner) + } + + pub(super) fn block_page_bases(&self) -> Result<[u32; 3]> { + let mut bases = [0; 3]; + for (base, block) in bases.iter_mut().zip(self.blocks.iter()) { + let offset = block.iova().checked_sub(vm_layout::DRIVER_BASE).ok_or(ERANGE)?; + if offset & 0x7fff != 0 { return Err(EINVAL); } + *base = u32::try_from(offset >> 15).map_err(|_| ERANGE)?; + } + Ok(bases) + } + + pub(super) fn page_list(&self) -> u64 { self.page_list.iova() } + pub(super) fn metrics(&self) -> u64 { self.metrics.iova() } + pub(super) fn scene_cycle(&self) -> u64 { + self.scene_scratch.iova() - vm_layout::DRIVER_BASE + 0x40 + } + pub(super) fn discard_offset(&self) -> u64 { + self.discard.iova() - vm_layout::DRIVER_BASE + } + + /// Firmware and application views must reference the same backing. + pub(super) fn firmware_aliases( + &mut self, uat: &mmu::Uat, vm: &mmu::Vm, page_list_high: u64, metrics_high: u64, + ) -> Result> { + let mut aliases = KVec::new(); + for (object, high) in [(&mut self.page_list, page_list_high), (&mut self.metrics, metrics_high)] { + let alias = object.map_alias_at(uat.kernel_vm(), high, mmu::PROT_FW_SHARED_RW)?; + for offset in (0..object.logical_size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset as u64)? + != vm.translate_iova(object.iova() + offset as u64)? { + return Err(EFAULT); + } + } + aliases.push(alias, GFP_KERNEL)?; + } + Ok(aliases) + } + + /// Map the retained pool into another application VM at the same compact + /// addresses with the original two-page guard reservations, so the + /// firmware's parameter-manager records keep naming valid pages after a + /// render client handoff. + pub(super) fn alias_into(&mut self, vm: &mmu::Vm) -> Result> { + let range = vm_layout::compact_range(&vm.kernel_range()).ok_or(EINVAL)?; + let mut aliases = KVec::new(); + let alias = |object: &mut MappedObject, aliases: &mut KVec| -> Result { + let end = object.iova() + .checked_add((object.logical_size + 2 * mmu::UAT_PGSZ) as u64) + .ok_or(EOVERFLOW)?; + if object.iova() < range.start || end > range.end { return Err(ERANGE); } + if let Some(mapping) = object.map_alias_at_with_guards( + vm, object.iova(), mmu::PROT_GPU_SHARED_RW, 2)? { + aliases.push(mapping, GFP_KERNEL)?; + } + Ok(()) + }; + for block in self.blocks.iter_mut() { alias(block, &mut aliases)?; } + alias(&mut self.page_list, &mut aliases)?; + alias(&mut self.metrics, &mut aliases)?; + alias(&mut self.scene_scratch, &mut aliases)?; + alias(&mut self.discard, &mut aliases)?; + self.validate(vm)?; + Ok(aliases) + } + + fn validate(&self, vm: &mmu::Vm) -> Result { + let check = |object: &MappedObject| -> Result { + let start = object.iova(); + let end = start.checked_add(object.logical_size as u64).ok_or(EOVERFLOW)?; + if start < vm_layout::DRIVER_BASE || end > vm_layout::COMPACT_END + || start & 0x7fff != 0 { return Err(ERANGE); } + for offset in (0..object.logical_size).step_by(mmu::UAT_PGSZ) { + vm.translate_iova(start + offset as u64)?; + } + for page in 0..2 { + if vm.translate_iova(end + page * mmu::UAT_PGSZ as u64).is_ok() { + return Err(EFAULT); + } + } + Ok(()) + }; + for block in &self.blocks { check(block)?; } + for object in [&self.page_list, &self.metrics, &self.scene_scratch, &self.discard] { + check(object)?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m5/resources/target_memory.rs b/drivers/gpu/drm/asahi/m5/resources/target_memory.rs new file mode 100644 index 0000000000000..f0f2c5176018d --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/target_memory.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Driver-owned render targets in the actual submitting VM. + +use super::{graphics_memory::M5GraphicsMemory, MappedObject}; +use crate::{driver::AsahiDrmDevice, m5::render_desc::G17pRenderParameters, mmu, m5::vm_layout, + m5::render_target::{self, M5RenderTargetLayout, PAGE_BYTES, AUX_BYTES}}; +use kernel::prelude::*; + +pub(super) struct M5TargetMemory { + target: MappedObject, + tpc: MappedObject, + deflake: MappedObject, + ta_status: MappedObject, + fragment_status: MappedObject, + auxiliary: MappedObject, + layout: M5RenderTargetLayout, +} + +impl M5TargetMemory { + pub(super) fn new( + dev: &AsahiDrmDevice, memory: &M5GraphicsMemory, p: &G17pRenderParameters, + clusters: u32, + ) -> Result { + let layout = M5RenderTargetLayout::new(p.width, p.height, p.layers, + p.utile_width, p.utile_height, clusters).ok_or(EINVAL)?; + let allocate = |size| memory.allocate_compact( + dev, size, 0x8000, mmu::PROT_GPU_SHARED_RW, + ); + let mut owner = Self { + target: allocate(layout.target_bytes)?, tpc: allocate(layout.tpc_bytes)?, + deflake: allocate(PAGE_BYTES)?, ta_status: allocate(PAGE_BYTES)?, + fragment_status: allocate(PAGE_BYTES)?, auxiliary: allocate(AUX_BYTES)?, layout, + }; + owner.deflake.with_bytes_mut(|bytes| { + render_target::initialize_deflake(bytes, p.width, p.height).ok_or(EINVAL) + })?; + owner.validate(memory.vm())?; + dev_dbg!(dev.as_ref(), + "M5_RENDER_TARGET target={:#x}/{:#x} heapmeta={:#x} tpc={:#x}/{:#x} deflake={:#x} status={:x?} auxiliary={:#x}; VM-owned\n", + owner.target.iova(), layout.target_bytes, owner.target.iova() + layout.heapmeta_offset as u64, + owner.tpc.iova(), layout.tpc_bytes, owner.deflake.iova(), + [owner.ta_status.iova(), owner.fragment_status.iova()], owner.auxiliary.iova()); + Ok(owner) + } + + pub(super) fn apply(&self, p: &mut G17pRenderParameters) { + p.context_base = vm_layout::DRIVER_BASE; + p.tilemap = self.target.iova(); + p.heapmeta = self.target.iova() + self.layout.heapmeta_offset as u64; + p.tpc = self.tpc.iova(); + p.deflake_3 = self.deflake.iova(); + p.deflake_2 = self.deflake.iova() + 0x40; + p.deflake_1 = self.deflake.iova() + 0x340; + p.ta_status = self.ta_status.iova(); + p.fragment_status = self.fragment_status.iova(); + p.aux_fb = self.auxiliary.iova(); + } + + fn validate(&self, vm: &mmu::Vm) -> Result { + for object in [&self.target, &self.tpc, &self.deflake, &self.ta_status, + &self.fragment_status, &self.auxiliary] { + let start = object.iova(); + let end = start.checked_add(object.logical_size as u64).ok_or(EOVERFLOW)?; + if start < vm_layout::DRIVER_BASE || end > vm_layout::COMPACT_END + || start & 0x7fff != 0 { return Err(ERANGE); } + for offset in (0..object.logical_size).step_by(PAGE_BYTES) { + vm.translate_iova(start + offset as u64)?; + } + for page in 0..2 { + if vm.translate_iova(end + page * PAGE_BYTES as u64).is_ok() { return Err(EFAULT); } + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m5/sksm.rs b/drivers/gpu/drm/asahi/m5/sksm.rs new file mode 100644 index 0000000000000..826f1def2c9b4 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/sksm.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8142/25G76 HAL300 SKSM entry and tag-14 serialization. +//! +//! The M5 host clears 0x200 bytes and writes two-bit dependency types at +//! +0x180. Neo's 0x180-byte entry cannot represent this data. These pure +//! builders do not register queues, publish producers or ring a doorbell. +//! Callers must retain every referenced object and implement the M5 lifetime +//! and completion protocol before any generated bytes reach firmware. + +pub(crate) const M5_SKSM_ENTRY_BYTES: usize = 0x200; +pub(crate) const M5_SKSM_ENTRY_COUNT: u32 = 256; + +/// Logical channel selector measured in native TA, 3D and CL tag-14 records. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum M5DataMaster { + Tiling = 0, + Fragment = 1, + Compute = 2, +} +const STAMP_MASK: u64 = (1 << 40) - 1; +const PACKED_ADDRESS_MASK: u64 = (1 << 38) - 1; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5SksmError { + QueueId, + Priority, + Stamp, + QueueFull, + Dependencies, + DependencyType, + Address, + Mcache, + Rce, + Qos, + AddCount, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmDependency { + pub(crate) queue_id: u8, + /// Already converted to the hardware's 32-bit generation / 8-bit index. + pub(crate) stamp: u64, + pub(crate) kind: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmRce { + pub(crate) address: u64, + pub(crate) tag: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmMcache { + pub(crate) address: u64, + pub(crate) index: u8, + pub(crate) count: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmQueue { + pub(crate) id: u8, + /// Host queue +0x1c, not the outer firmware channel's priority. + pub(crate) priority: u8, + pub(crate) current_stamp: u64, + pub(crate) submitted: u32, + pub(crate) previous_stamp: Option, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmOperands<'a> { + /// Explicit dependencies already resolved by their owning event machine. + pub(crate) barriers: &'a [M5SksmDependency], + /// Native convertEventMachineTSToHWTS(command) - 1, for first publication. + /// This must be grounded in the queue's initialized completion state. + pub(crate) first_predecessor: u64, + pub(crate) implicit_dependency_type: u8, + pub(crate) mcache: Option, + pub(crate) flag_bit1: bool, + pub(crate) flag_bit61: bool, + pub(crate) flag_bit63: bool, + pub(crate) payload: [u64; 2], + pub(crate) qos_byte_32: u8, + pub(crate) qos_class_40: u8, + pub(crate) qos_word_48: u16, + pub(crate) event_mask: [u64; 4], + pub(crate) rce_kind: u8, + pub(crate) rce: [M5SksmRce; 4], + /// Native encoder's scalar x5 argument, copied unmodified to +0x178. + pub(crate) auxiliary: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PreparedM5SksmEntry { + pub(crate) bytes: [u8; M5_SKSM_ENTRY_BYTES], + pub(crate) offset: u32, + /// May only replace host state after entry storage and a dmb ish. + pub(crate) queue_after: M5SksmQueue, +} + +fn check_queue(id: u8, priority: u8, stamp: u64) -> Result<(), M5SksmError> { + if id > 0x7f { + return Err(M5SksmError::QueueId); + } + if priority > 3 { + return Err(M5SksmError::Priority); + } + if stamp & !STAMP_MASK != 0 { + return Err(M5SksmError::Stamp); + } + Ok(()) +} + +fn packed_address(address: u64) -> Result { + if address & 31 != 0 || address >> 5 > PACKED_ADDRESS_MASK { + return Err(M5SksmError::Address); + } + Ok(address >> 5) +} + +fn put(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Recovered from M5 encodeKickEntry at 25G76 0xfffffe00089e750c and its +/// encodeDependencyList at 0xfffffe00089e781c. Geometry is deliberately fixed +/// to the independently captured 256 x 512-byte M5 queues. +pub(crate) fn prepare_m5_sksm_entry( + queue: M5SksmQueue, + operands: M5SksmOperands<'_>, +) -> Result { + check_queue(queue.id, queue.priority, queue.current_stamp)?; + if queue.submitted >= M5_SKSM_ENTRY_COUNT { + return Err(M5SksmError::QueueFull); + } + // Native always appends one implicit predecessor after explicit barriers. + if operands.barriers.len() >= 32 { + return Err(M5SksmError::Dependencies); + } + if operands.qos_class_40 > 31 { + return Err(M5SksmError::Qos); + } + if operands.rce_kind > 3 || operands.rce.iter().any(|r| r.tag > 0x7f) { + return Err(M5SksmError::Rce); + } + let mut bytes = [0; M5_SKSM_ENTRY_BYTES]; + let mut header = (queue.current_stamp << 2) + | ((queue.id as u64) << 42) + | ((queue.priority as u64) << 59) + | ((operands.flag_bit1 as u64) << 1) + | ((operands.flag_bit61 as u64) << 61) + | ((operands.flag_bit63 as u64) << 63); + if let Some(mcache) = operands.mcache { + if mcache.index > 63 || mcache.count == 0 || mcache.count > 64 { + return Err(M5SksmError::Mcache); + } + let packed = packed_address(mcache.address)?; + put( + &mut bytes, + 8, + (mcache.index as u64) | (packed << 6) | ((mcache.count as u64 - 1) << 54), + ); + header |= 1 << 58; + } + put(&mut bytes, 0, header); + put(&mut bytes, 0x10, operands.payload[0]); + put(&mut bytes, 0x18, operands.payload[1]); + let predecessor = M5SksmDependency { + queue_id: queue.id, + stamp: queue.previous_stamp.unwrap_or(operands.first_predecessor), + kind: operands.implicit_dependency_type, + }; + let mut kinds = 0; + for (i, dependency) in operands + .barriers + .iter() + .chain(core::iter::once(&predecessor)) + .enumerate() + { + check_queue(dependency.queue_id, 0, dependency.stamp)?; + if dependency.kind > 3 { + return Err(M5SksmError::DependencyType); + } + put( + &mut bytes, + 0x28 + i * 8, + dependency.stamp | ((dependency.queue_id as u64) << 40), + ); + kinds |= (dependency.kind as u64) << (i * 2); + } + let count = operands.barriers.len() + 1; + let valid = if count == 32 { + u32::MAX + } else { + (1u32 << count) - 1 + }; + put( + &mut bytes, + 0x20, + valid as u64 + | ((operands.qos_byte_32 as u64) << 32) + | ((operands.qos_class_40 as u64) << 40) + | ((operands.qos_word_48 as u64) << 48), + ); + put(&mut bytes, 0x180, kinds); + for (i, mask) in operands.event_mask.into_iter().enumerate() { + put(&mut bytes, 0x128 + i * 8, mask); + } + bytes[0x148] = operands.rce_kind; + for (i, rce) in operands.rce.into_iter().enumerate() { + put( + &mut bytes, + 0x150 + i * 8, + packed_address(rce.address)? | ((rce.tag as u64) << 43), + ); + } + put(&mut bytes, 0x178, operands.auxiliary); + Ok(PreparedM5SksmEntry { + bytes, + offset: (queue.current_stamp as u32 & 255) * M5_SKSM_ENTRY_BYTES as u32, + queue_after: M5SksmQueue { + current_stamp: (queue.current_stamp + 1) & STAMP_MASK, + submitted: queue.submitted + 1, + previous_stamp: Some(queue.current_stamp), + ..queue + }, + }) +} + +/// Native CL add-kicks record at the captured submitBuffer publication site. +/// This is a packed 21-byte command, unrelated to the outer 24-byte entry. +pub(crate) fn encode_m5_sksm_add_kicks( + queue_id: u8, + priority: u8, + first_stamp: u64, + count: u16, +) -> Result<[u8; 21], M5SksmError> { + encode_m5_sksm_add_kicks_for( + M5DataMaster::Compute, + queue_id, + priority, + first_stamp, + count, + ) +} + +/// Encode a retained queue's stage explicitly; graphics never uses CL's DM2. +pub(crate) fn encode_m5_sksm_add_kicks_for( + data_master: M5DataMaster, + queue_id: u8, + priority: u8, + first_stamp: u64, + count: u16, +) -> Result<[u8; 21], M5SksmError> { + check_queue(queue_id, priority, first_stamp)?; + if count == 0 || count > 256 { + return Err(M5SksmError::AddCount); + } + let mut bytes = [0; 21]; + bytes[..4].copy_from_slice(&14u32.to_le_bytes()); + bytes[4..6].copy_from_slice(&(queue_id as u16).to_le_bytes()); + bytes[6..8].copy_from_slice(&count.to_le_bytes()); + bytes[8] = priority; + bytes[9..17].copy_from_slice(&first_stamp.to_le_bytes()); + bytes[17..21].copy_from_slice(&(data_master as u32).to_le_bytes()); + Ok(bytes) +} + diff --git a/drivers/gpu/drm/asahi/m5/submission.rs b/drivers/gpu/drm/asahi/m5/submission.rs new file mode 100644 index 0000000000000..c5414cea1ae2d --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/submission.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +fn put_u32_le(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64_le(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// M5 G17G / macOS 25G76 resets six pairs of 128-bit vectors, covering 0xc0 +/// bytes. Its firmware consumes through +0xa7. The common 0x80-byte reset +/// would omit the moved fields and the firmware-visible tail. +pub(crate) const M5_25G76_CHANNEL_STATE_RESET_SIZE: usize = 0xc0; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ChannelStateResetValues { + pub(crate) uncached_state: u64, + pub(crate) command_pointer_ring: u64, + pub(crate) gpu_va_10: u64, + pub(crate) word_48: u32, + /// Opaque host field +0xf8 copied to the unaligned firmware field +0x9c. + pub(crate) gpu_va_9c: u64, +} + +/// Exact reset stores from M5 AGXChannel::resetChannelState at +/// 0xfffffe000897d36c..0xfffffe000897d410 (AGXG17G 353.14, 25G76). +/// Uncached stores are separately shared with apply_g17_uncached_channel_reset. +/// This codec alone does not admit M5 submission or a Linux GPU runtime. +pub(crate) fn encode_m5_25g76_channel_state_reset( + values: M5ChannelStateResetValues, +) -> [u8; M5_25G76_CHANNEL_STATE_RESET_SIZE] { + let mut raw = [0u8; M5_25G76_CHANNEL_STATE_RESET_SIZE]; + put_u64_le(&mut raw, 0x00, values.uncached_state); + put_u64_le(&mut raw, 0x08, values.command_pointer_ring); + put_u64_le(&mut raw, 0x10, values.gpu_va_10); + put_u32_le(&mut raw, 0x24, u32::MAX); + put_u32_le(&mut raw, 0x28, 4); + put_u32_le(&mut raw, 0x44, u32::MAX); + put_u32_le(&mut raw, 0x48, values.word_48); + put_u64_le(&mut raw, 0x9c, values.gpu_va_9c); + raw +} + +/// Logical render prefix, resolved at ioctl admission before asynchronous work. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] +pub(crate) struct G17PRenderOrder { + pub(crate) sequence: u64, + pub(crate) wait_through: Option, +} + diff --git a/drivers/gpu/drm/asahi/m5/vm_layout.rs b/drivers/gpu/drm/asahi/m5/vm_layout.rs new file mode 100644 index 0000000000000..19a6593736112 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/vm_layout.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Application VA contract shared with Mesa's agx_m5_va.h. + +use core::ops::Range; + +pub(crate) const DRIVER_BASE: u64 = 0x10_0000_0000; +pub(crate) const COMPACT_END: u64 = DRIVER_BASE + (1u64 << 32); +pub(crate) const DATA_START: u64 = 0x40_0000_0000; + +/// TA deflake fields encode an unsigned 32-bit offset from DRIVER_BASE. +/// Keep the private render arena inside that aperture and the USC backing +/// after it, within the same VM_CREATE reservation. A client may leave a +/// prefix of the compact aperture for its own command data. Mesa's default +/// reservation still begins at DRIVER_BASE and data begins at DATA_START. +pub(crate) fn compact_range(reserved: &Range) -> Option> { + if reserved.start < DRIVER_BASE || reserved.start >= COMPACT_END + || reserved.start & 0x3fff != 0 || reserved.end <= COMPACT_END + || reserved.end > DATA_START || reserved.end & 0x3fff != 0 { + return None; + } + Some(reserved.start..COMPACT_END) +} + +pub(crate) fn auxiliary_range(reserved: &Range) -> Option> { + compact_range(reserved)?; + Some(COMPACT_END..reserved.end) +} + +/// Command pointers must not reach driver-owned mappings, including when +/// the client leaves application data below its private arena reservation. +pub(crate) fn user_range_contains( + reserved: &Range, address: u64, size: u64, user_end: u64, +) -> bool { + let Some(end) = address.checked_add(size) else { return false; }; + size != 0 && address >= DRIVER_BASE && end <= user_end + && (end <= reserved.start || address >= reserved.end) +} + diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 122b42ccd1b6d..ad44eba42fb32 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -113,11 +113,22 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; mod lifetime; +#[cfg(CONFIG_DRM_ASAHI_M5)] +mod m5_context; mod ppl; mod residency; mod shared; +#[cfg(CONFIG_DRM_ASAHI_G18)] +mod t8152; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; +#[cfg(CONFIG_DRM_ASAHI_G18)] +pub(crate) use t8152::G18ExecutionContext; +/// The module-1134 name of the job guard G18 holds while a job uses a VM. +#[cfg(CONFIG_DRM_ASAHI_G18)] +pub(crate) type T8140VmJobGuard = VmJobGuard; +#[cfg(CONFIG_DRM_ASAHI_M5)] +pub(crate) use m5_context::M5ExecutionContext; pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region @@ -164,6 +175,28 @@ pub(crate) enum UatFirmware { /// Offset of the secondary instance's kernel root in the `pagetables` region. secondary_root_offset: Option, }, + /// Two coprocessor instances that never touch the handoff region and publish their roots like + /// [`UatFirmware::PublishedRoots`], without the context-1 kernel alias (T8152/G18G, 26A428). + /// + /// A live firmware peer in the handoff region is refused, and the high roots are prepared, + /// confirmed and mirrored by `mmu::t8152`. + #[cfg(CONFIG_DRM_ASAHI_G18)] + NoPeerRoots { + /// Offset of the secondary instance's kernel root in the `pagetables` region. + secondary_root_offset: u64, + }, +} + +impl UatFirmware { + /// Whether this is the T8152 no-peer firmware. + #[inline(always)] + const fn is_no_peer(&self) -> bool { + #[cfg(CONFIG_DRM_ASAHI_G18)] + if let UatFirmware::NoPeerRoots { .. } = self { + return true; + } + false + } } /// UAT policy of a SoC where it differs from the A18 Pro's. @@ -581,6 +614,14 @@ impl gpuvm::DriverGpuVm for VmInner { if ctx.dev.is_none() { return Err(ENODEV); } + // T8152 (module 1134): GPUVM cannot see driver-owned KernelMappings, and G18 places + // driver aliases in user VMs outside the VM_CREATE kernel range. Preflight the full + // object range in the authoritative page table (EBUSY on any populated leaf) and + // allocate every intermediate table before the first scatterlist leaf is written. + if self.uat_inner.firmware.is_no_peer() { + self.page_table + .prepare_map(map_addr..map_addr.checked_add(map_length as u64).ok_or(EOVERFLOW)?)?; + } // Bind preparation retained the table's backing before taking the VM reservation. // Never acquire the object reservation from inside the GPUVM map callback. let segments = op.vm_bo().data().segments.lock(); @@ -887,6 +928,9 @@ impl VmInner { UatFirmware::PublishedRoots { .. } => { mem::tlbi_range_or_asid(asid, iova as usize, size) } + // T8152: the per-page range encoding module 1134 issues. + #[cfg(CONFIG_DRM_ASAHI_G18)] + UatFirmware::NoPeerRoots { .. } => mem::tlbi_range(asid, iova as usize, size), } } @@ -919,9 +963,14 @@ impl VmInner { if contexts == 0 { return; } + let no_peer = self.uat_inner.firmware.is_no_peer(); for ctx in 0..UAT_NUM_CTX { if contexts & (1 << ctx) != 0 { - mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); + if no_peer { + mem::tlbi_range(ctx as u8, iova as usize, size); + } else { + mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); + } } } mem::sync(); @@ -1046,6 +1095,9 @@ pub(crate) struct Vm { device_binding: Arc, dev: ARef, id: u64, + /// The range reserved for kernel-managed objects at creation. + #[cfg_attr(not(any(CONFIG_DRM_ASAHI_M5, CONFIG_DRM_ASAHI_G18)), allow(dead_code))] + kernel_range: Range, mapping_epoch: Option>, bind_sequence: Arc, lifetime: Option>, @@ -1311,13 +1363,13 @@ impl TimestampVmap { /// Prepared CPU timestamp access with its own vmap; destruction happens after completion /// signalling. -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] pub(crate) struct TimestampMapping { mapping: TimestampVmap, offset: usize, } -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] impl TimestampMapping { /// Prepare one bounded CPU timestamp destination on retained GEM backing. pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { @@ -1368,7 +1420,7 @@ impl KernelMapping { /// use a complete atomic store; byte stores support unaligned destinations without /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { let gem = self.0._gem.as_ref().ok_or(EINVAL)?; TimestampMapping::new(gem, self.timestamp_offset(offset)?) @@ -1412,6 +1464,7 @@ impl KernelMapping { let mut mutated = false; let result = (|| { if self.0.uat_inner.firmware == UatFirmware::Handoff + || self.0.uat_inner.firmware.is_no_peer() || prefix == 0 || prefix > self.size() || (prefix | self.size()) & UAT_PGMSK != 0 @@ -1462,6 +1515,27 @@ impl KernelMapping { (result, mutated) } + /// Downgrade this mapping's PTEs to uncached, without a firmware cache flush (T8152). + #[cfg(CONFIG_DRM_ASAHI_G18)] + fn remap_uncached_only(&mut self) { + let mut owner = self.0.owner.lock_inner(); + let prot = self.0.prot.as_uncached(); + let _mutation = owner.mapping_mutation(); + if owner + .page_table + .reprot_pages(self.iova_range(), prot) + .is_err() + { + dev_err!( + owner.dev.as_ref(), + "MMU: remap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + mem::sync(); + } + /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the /// coprocessor cache. This is required to safely unmap cached/private mappings. fn remap_uncached_and_flush(&mut self) { @@ -1644,6 +1718,12 @@ impl Drop for KernelMapping { ); self.remap_uncached_and_flush(); } + // T8152 (module 1134): downgrade the PTEs to uncached before the unmap. The firmware + // cache flush is never armed for this firmware, so no flush request follows. + #[cfg(CONFIG_DRM_ASAHI_G18)] + if self.0.prot.is_cached_noncoherent() && self.0.uat_inner.firmware.is_no_peer() { + self.remap_uncached_only(); + } // Only VM page tables/binding state are touched here. The shared // reservation is sufficient and avoids fallible exec allocation in Drop. @@ -1985,6 +2065,14 @@ impl Handoff { /// Initialize the handoff region fn init(&self, firmware: UatFirmware, slot: u32) -> Result { + #[cfg(CONFIG_DRM_ASAHI_G18)] + if firmware.is_no_peer() + && (self.magic_fw.load(Ordering::Acquire) != 0 + || self.lock_fw.load(Ordering::Acquire) != 0) + { + pr_err!("T8152 handoff: unexpected firmware peer; refusing no-peer mode\n"); + return Err(ENODEV); + } self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); self.cur_slot.store(slot, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); @@ -2172,7 +2260,7 @@ impl Vm { &dev, &*dummy_obj.gem, gpuvm_range, - kernel_range, + kernel_range.clone(), VmInner { dev: dev.clone(), is_kernel, @@ -2194,6 +2282,7 @@ impl Vm { device_binding, dev: dev.as_ref().as_ref().into(), id, + kernel_range, gpuvm, mapping_epoch, bind_sequence, @@ -2207,6 +2296,12 @@ impl Vm { }) } + /// The range reserved for kernel-managed objects when this VM was created. + #[cfg_attr(not(any(CONFIG_DRM_ASAHI_M5, CONFIG_DRM_ASAHI_G18)), allow(dead_code))] + pub(crate) fn kernel_range(&self) -> Range { + self.kernel_range.clone() + } + /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { self.status = Some(Arc::pin_init( @@ -2551,9 +2646,24 @@ impl Vm { prot: Prot, guard: bool, ) -> Result { - let reserved_size = size - .checked_add(if guard { UAT_PGSZ } else { 0 }) - .ok_or(EOVERFLOW)?; + self.map_at_with_guard_size(addr, size, gem, prot, if guard { UAT_PGSZ } else { 0 }) + } + + /// Map a GEM object into this Vm at a specific address, followed by an unmapped guard of + /// `guard_size` bytes (a multiple of the page size). + #[inline(never)] + pub(crate) fn map_at_with_guard_size( + &self, + addr: u64, + size: usize, + gem: &gem::Object, + prot: Prot, + guard_size: usize, + ) -> Result { + if guard_size & UAT_PGMSK != 0 { + return Err(EINVAL); + } + let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; // End the bound-device lease before entering firmware/GPUVM waits. let (_bind_lease, vm_bo) = { @@ -3354,6 +3464,11 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + // T8152 uses the module-1134 M5 execution-context leases (`mmu::t8152`); the slot + // allocator would claim contexts the leases own. + if self.inner.firmware.is_no_peer() { + return Err(ENOTSUPP); + } self.bind_inner(vm, true) } @@ -3471,7 +3586,9 @@ impl Uat { /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { - if self.inner.firmware == UatFirmware::Handoff { + // NoPeerRoots: T8152 publishes `t8152::G18ExecutionContext` leases (private empty upper + // root) instead of kernel-root upper halves. + if self.inner.firmware == UatFirmware::Handoff || self.inner.firmware.is_no_peer() { return Err(EINVAL); } { @@ -3702,7 +3819,13 @@ impl Uat { shared <- new_mutex!( UatShared { kernel_ttb1: 0, - execution_contexts: !EXECUTION_CONTEXT_MASK, + // NoPeerRoots: contexts 3..63 are the M5 execution-context leases of + // `mmu::t8152`; 0..2 belong to the driver and the firmware. + execution_contexts: if firmware.is_no_peer() { + 0b111 + } else { + !EXECUTION_CONTEXT_MASK + }, context_generations: [0; UAT_NUM_CTX], binding_generations: [0; UAT_NUM_CTX], binding_published: [false; UAT_NUM_CTX], @@ -3760,6 +3883,13 @@ impl Uat { UatFirmware::PublishedRoots { secondary_root_offset: None, } => (UAT_PGSZ, None), + #[cfg(CONFIG_DRM_ASAHI_G18)] + UatFirmware::NoPeerRoots { + secondary_root_offset, + } => ( + secondary_root_offset as usize + UAT_PGSZ, + Some(ttb1 + secondary_root_offset), + ), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); diff --git a/drivers/gpu/drm/asahi/mmu/m5_context.rs b/drivers/gpu/drm/asahi/mmu/m5_context.rs new file mode 100644 index 0000000000000..5548d5d0dab6c --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/m5_context.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 (T8142) application contexts. +//! +//! The 25G76 firmware runs application work in a context whose lower half is the client VM's +//! root and whose upper half is a private, empty root: the firmware's own upper mappings are +//! not shared into application contexts. The context is claimed when a queue is set up and its +//! roots are published explicitly, once the work that names it has been prepared. Mapping +//! changes reach a published context through [`VmInner::tlbi_contexts_naming_root`]. + +use super::*; + +/// A claimed M5 application context and the VMs it names. +pub(crate) struct M5ExecutionContext { + inner: Arc, + id: u8, + low: Vm, + high: Vm, + published: bool, +} + +impl Uat { + /// Claims an application context for `vm`, a client VM of this UAT. Nothing is published. + pub(crate) fn new_m5_execution_context(&self, vm: &Vm) -> Result { + self.check_client_vm(vm)?; + let id = { + let mut inner = self.inner.lock(); + let free = !inner.execution_contexts & EXECUTION_CONTEXT_MASK; + if free == 0 { + return Err(ENOSPC); + } + let id = free.trailing_zeros() as usize; + inner.execution_contexts |= 1u64 << id; + id + }; + let high = match self.new_vm(0x8142_1000 + id as u64, self.geometry.user_range()) { + Ok(high) => high, + Err(error) => { + self.inner.lock().execution_contexts &= !(1u64 << id); + return Err(error); + } + }; + Ok(M5ExecutionContext { + inner: self.inner.clone(), + id: id as u8, + low: vm.clone(), + high, + published: false, + }) + } + + fn check_client_vm(&self, vm: &Vm) -> Result { + let owner = vm.gpuvm.lock_inner(); + if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) + { + return Err(EINVAL); + } + Ok(()) + } +} + +impl M5ExecutionContext { + /// The client VM named as the lower half. + pub(crate) fn vm(&self) -> &Vm { + &self.low + } + + /// The hardware context number, as the firmware records name it. + pub(crate) fn context_id(&self) -> u8 { + self.id + } + + /// The TTBAT words of this context. + pub(crate) fn roots(&self) -> [u64; 2] { + let id = usize::from(self.id); + [tagged_root(self.low.ttb(), id), tagged_root(self.high.ttb(), id)] + } + + /// Publishes the roots. A slot that is not empty is never replaced. + pub(crate) fn publish(&mut self) -> Result { + if self.published { + return Err(EBUSY); + } + let [low, high] = self.roots(); + let id = usize::from(self.id); + { + let inner = self.inner.lock(); + inner.handoff_lock(); + let slot = &inner.ttbs()[id]; + let empty = + slot.ttb0.load(Ordering::Acquire) == 0 && slot.ttb1.load(Ordering::Acquire) == 0; + if empty { + slot.ttb0.store(low, Ordering::Release); + slot.ttb1.store(high, Ordering::Release); + inner.publish_slot(id); + } + inner.handoff_unlock(); + if !empty { + return Err(EBUSY); + } + } + self.published = true; + mem::sync(); + mem::tlbi_asid(self.id); + mem::sync(); + Ok(()) + } + + /// Whether this context's roots are published and still in place. + pub(crate) fn is_current(&self) -> bool { + if !self.published { + return false; + } + let [low, high] = self.roots(); + let inner = self.inner.lock(); + let slot = &inner.ttbs()[usize::from(self.id)]; + slot.ttb0.load(Ordering::Acquire) == low && slot.ttb1.load(Ordering::Acquire) == high + } + + /// Names another client VM of the same UAT, keeping the context number. Published roots + /// are retracted first; the caller has made sure no work of the previous VM is outstanding. + pub(crate) fn rebind(&mut self, vm: &Vm) -> Result { + { + let owner = vm.gpuvm.lock_inner(); + if owner.is_kernel + || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) + { + return Err(EINVAL); + } + } + self.retract(); + self.low = vm.clone(); + Ok(()) + } + + fn retract(&mut self) { + if !self.published { + return; + } + let [low, high] = self.roots(); + let id = usize::from(self.id); + { + let inner = self.inner.lock(); + inner.handoff_lock(); + let slot = &inner.ttbs()[id]; + // Only our own pair is retracted. + if slot.ttb0.load(Ordering::Acquire) == low && slot.ttb1.load(Ordering::Acquire) == high + { + slot.ttb0.store(0, Ordering::Release); + slot.ttb1.store(0, Ordering::Release); + inner.publish_slot(id); + } + inner.handoff_unlock(); + } + mem::sync(); + mem::tlbi_asid(self.id); + mem::sync(); + self.published = false; + } +} + +impl Drop for M5ExecutionContext { + fn drop(&mut self) { + self.retract(); + let id = usize::from(self.id); + let mut inner = self.inner.lock(); + let slot = &inner.ttbs()[id]; + // A slot someone else filled is not ours to hand out again: keep it claimed. + if slot.ttb0.load(Ordering::Acquire) == 0 && slot.ttb1.load(Ordering::Acquire) == 0 { + inner.execution_contexts &= !(1u64 << id); + } + } +} diff --git a/drivers/gpu/drm/asahi/mmu/t8152.rs b/drivers/gpu/drm/asahi/mmu/t8152.rs new file mode 100644 index 0000000000000..5f5906e131127 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/t8152.rs @@ -0,0 +1,641 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! T8152 (G18G; Mac mini M6, J873) UAT support on the Direct backend. +//! +//! Module 1134 ran T8152 on its own MMU (`Uat::new_t8152`, handoff mode +//! `NoPeerT8152`, `m5_context.rs`). This file carries those pieces onto this +//! driver's UAT with [`UatFirmware::NoPeerRoots`]: +//! - the constructor and its firmware-reservation checks; +//! - the preparation, confirmation and mirroring of the firmware-published +//! high roots, with the shared-L2 topology proof, the DMB barrier and the +//! log lines of module 1134; +//! - the M5 execution-context leases (contexts 3..63, private empty upper +//! root, explicit publication), used instead of [`ExecutionContext`]; +//! - guard-page mappings and guard-tail aliases; +//! - the module-1134 names G18 calls (`retain_t8140_job`, `id`, `kernel_range`). +//! +//! Bodies are module 1134's (mmu.rs, m5_context.rs) unless a comment says otherwise. + +use super::*; +use kernel::drm::gem::BaseObject; +use kernel::page::Page; + +/// J873 (T8152) boot-local firmware reservations from the ADT /arm-io/sgx +/// node, as a loader copies them into /reserved-memory. +const T8152_DRAM_BASE: u64 = 0x100_0000_0000; +/// J873 DRAM end: /chosen dram-base 0x100_0000_0000 + dram-size 24 GiB. +const T8152_DRAM_END: u64 = 0x106_0000_0000; +/// (memory-region name, exact size): 26A428 GFX/GFX1 __DATA, then the +/// __DATA_SHARED_RO segments (firmware VA +0x10c000 / +0x108000). +const T8152_FIRMWARE_DATA: [(&CStr, u64); 2] = [ + (c_str!("gfx-data"), 0xbc000), + (c_str!("gfx1-data"), 0xb4000), +]; +const T8152_FIRMWARE_SHARED_RO: [(&CStr, u64); 2] = [ + (c_str!("gfx-data-shared-ro"), 0x8000), + (c_str!("gfx1-data-shared-ro"), 0x8000), +]; + +/// The GFX1 root follows the GFX root in the `pagetables` region (module 1134 +/// `t8140_secondary_ttb1 = pagetables + 0x40000`). +const T8152_SECONDARY_ROOT_OFFSET: u64 = 0x40000; +/// 42-bit input and output address space (module 1134 `UatConfig::T8152`). +const T8152_GEOMETRY: UatGeometry = UatGeometry { ias: 42, oas: 42 }; + +/// Entries 0 and 1 of each firmware root are private to that instance. +const T8140_FIRMWARE_PRIVATE_ROOT_ENTRIES: usize = 2; +/// Entries 0..2, including the shared-L2 kernel window, are published by firmware. +const T8140_FIRMWARE_SHARED_ROOT_ENTRIES: usize = 3; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct T8140PrimaryRootSnapshot { + entries: [u64; T8140_FIRMWARE_SHARED_ROOT_ENTRIES], +} + +fn clear_stale_host_root_entries(root: &mut [u64]) { + if root.len() > T8140_FIRMWARE_SHARED_ROOT_ENTRIES { + root[T8140_FIRMWARE_SHARED_ROOT_ENTRIES..].fill(0); + } +} + +fn mirror_host_root_entries(primary: &[u64], secondary: &mut [u64]) -> usize { + let mut mirrored = 0; + for (source, destination) in primary + .iter() + .zip(secondary.iter_mut()) + .skip(T8140_FIRMWARE_PRIVATE_ROOT_ENTRIES) + { + if *source != 0 && *destination != *source { + *destination = *source; + mirrored += 1; + } + } + mirrored +} + +impl Uat { + /// Create the T8152 UAT. The firmware never writes its half of the + /// handoff page, so the page is run in [`UatFirmware::NoPeerRoots`] and a + /// live peer is refused. + /// + /// `crash_views` gives the DATA-relative offsets of the GFX and GFX1 + /// RTKit crash views. The G18G offset is not yet known; G18 passes `None` + /// and that is the only supported value here. + pub(crate) fn new_t8152( + dev: &driver::AsahiDevice, + crash_views: Option<[u64; 2]>, + ) -> Result { + let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let mut regions = [(0u64, 0u64); 4]; + for (index, (name, size)) in Iterator::chain( + T8152_FIRMWARE_DATA.into_iter(), + T8152_FIRMWARE_SHARED_RO, + ) + .enumerate() + { + let region = node.reserved_mem_region_to_resource_byname(name)?; + let start = region.start(); + let end = start.checked_add(region.size()).ok_or(EOVERFLOW)?; + if start < T8152_DRAM_BASE + || region.size() != size + || start & UAT_PGMSK as u64 != 0 + || end > T8152_DRAM_END + { + dev_err!( + dev.as_ref(), + "MMU: T8152 {} reservation {:#x}+{:#x} does not match the 26A428 layout ({:#x} bytes in J873 DRAM)\n", + name, + start, + region.size(), + size + ); + return Err(EINVAL); + } + if regions[..index].iter().any(|&(s, e)| start < e && s < end) { + dev_err!(dev.as_ref(), "MMU: T8152 {} overlaps another firmware reservation\n", name); + return Err(EINVAL); + } + regions[index] = (start, end); + } + // Module 1134 maps optional crash views here; G18 never requests them. + if crash_views.is_some() { + return Err(ENOTSUPP); + } + // The A18 Pro policy is the one module 1134 ran this UAT with: no + // table cleaning, range TLBIs, the handoff lock and slot 0. + Self::new( + dev.clone(), + T8152_GEOMETRY, + false, + UatFirmware::NoPeerRoots { + secondary_root_offset: T8152_SECONDARY_ROOT_OFFSET, + }, + UatQuirks::DEFAULT, + ) + } + + /// Whether this is the T8152 UAT; identity only. + pub(crate) fn is_t8152_geometry(&self) -> bool { + matches!(self.inner.firmware, UatFirmware::NoPeerRoots { .. }) + && self.geometry.ias == T8152_GEOMETRY.ias + && self.geometry.oas == T8152_GEOMETRY.oas + } + + /// Prepare the firmware-published upper roots for host mappings. + /// + /// Each firmware instance publishes three entries during RTKit boot: two + /// private code/data entries and the shared L2 entry used by the kernel VA + /// window. Host mappings must be added beneath that live entry, not beneath + /// a root image constructed before the processor starts. + pub(crate) fn prepare_t8140_firmware_high_roots( + &self, + ) -> Result { + if !self.is_t8152_geometry() { + return Err(EINVAL); + } + let primary_phys = self.kernel_vm.ttb(); + let secondary_phys = self.inner.lock().secondary_ttb1.ok_or(EIO)?; + // SAFETY: both roots are pages of the bootloader-reserved `pagetables` + // region, which outlives this Uat. + let primary = unsafe { Page::borrow_phys(&primary_phys) }.ok_or(EIO)?; + // SAFETY: as above. + let secondary = unsafe { Page::borrow_phys(&secondary_phys) }.ok_or(EIO)?; + let mut primary_entries = [0u64; T8140_FIRMWARE_SHARED_ROOT_ENTRIES]; + let mut secondary_entries = [0u64; T8140_FIRMWARE_SHARED_ROOT_ENTRIES]; + + primary.with_pointer_into_page(0, UAT_PGSZ, |pointer| { + // SAFETY: the pointer covers the whole root page. + let entries = unsafe { + core::slice::from_raw_parts_mut( + pointer as *mut u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + primary_entries.copy_from_slice(&entries[..T8140_FIRMWARE_SHARED_ROOT_ENTRIES]); + Ok(()) + })?; + secondary.with_pointer_into_page(0, UAT_PGSZ, |pointer| { + // SAFETY: the pointer covers the whole root page. + let entries = unsafe { + core::slice::from_raw_parts( + pointer as *const u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + secondary_entries.copy_from_slice(&entries[..T8140_FIRMWARE_SHARED_ROOT_ENTRIES]); + Ok(()) + })?; + { + let node = self.dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let shared_l2 = node.reserved_mem_region_to_resource_byname(c_str!("shared-l2"))?; + let mask = ((1u64 << self.geometry.oas) - 1) & !(UAT_PGMSK as u64); + if shared_l2.size() != UAT_PGSZ as u64 + || Iterator::chain(primary_entries.iter(), secondary_entries.iter()).any(|pte| pte & 3 != 3) + || primary_entries[2] & mask != shared_l2.start() + || secondary_entries[2] & mask != shared_l2.start() + || primary_entries[0] & mask == secondary_entries[0] & mask + { + dev_err!( + self.dev.as_ref(), + "{} firmware-private roots/shared L2 do not match the native topology\n", + "T8152" + ); + return Err(EIO); + } + } + primary.with_pointer_into_page(0, UAT_PGSZ, |pointer| { + // SAFETY: the pointer covers the whole root page. + let entries = unsafe { core::slice::from_raw_parts_mut( + pointer as *mut u64, UAT_PGSZ / core::mem::size_of::()) }; + clear_stale_host_root_entries(entries); + Ok(()) + })?; + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + dev_info!( + self.dev.as_ref(), + "MMU: GFX high root {:#x} entries [{:#x}, {:#x}, {:#x}]; GFX1 root {:#x} entries [{:#x}, {:#x}, {:#x}]\n", + primary_phys, + primary_entries[0], + primary_entries[1], + primary_entries[2], + secondary_phys, + secondary_entries[0], + secondary_entries[1], + secondary_entries[2] + ); + Ok(T8140PrimaryRootSnapshot { + entries: primary_entries, + }) + } + + /// Confirm construction retained the three firmware-published entries. + pub(crate) fn confirm_t8140_primary_high_root( + &self, + snapshot: T8140PrimaryRootSnapshot, + ) -> Result { + if !self.is_t8152_geometry() { + return Err(EINVAL); + } + let primary_phys = self.kernel_vm.ttb(); + // SAFETY: the root is a page of the bootloader-reserved `pagetables` region. + let primary = unsafe { Page::borrow_phys(&primary_phys) }.ok_or(EIO)?; + let mut current = [0u64; T8140_FIRMWARE_SHARED_ROOT_ENTRIES]; + primary.with_pointer_into_page(0, UAT_PGSZ, |pointer| { + // SAFETY: the pointer covers the whole root page. + let entries = unsafe { + core::slice::from_raw_parts( + pointer as *const u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + current.copy_from_slice(&entries[..T8140_FIRMWARE_SHARED_ROOT_ENTRIES]); + Ok(()) + })?; + if current != snapshot.entries { + dev_err!( + self.dev.as_ref(), + "MMU: GFX high-root firmware entries changed during graph construction: before {:?}, after {:?}\n", + snapshot.entries, + current + ); + return Err(EIO); + } + dev_info!( + self.dev.as_ref(), + "MMU: GFX high-root firmware entries preserved across graph construction\n" + ); + Ok(()) + } + + /// Mirror host-visible upper-root entries into GFX1's private top table. + /// Entries 0 and 1 remain owned by that firmware instance. + pub(crate) fn mirror_t8140_secondary_high_root(&self) -> Result { + if !self.is_t8152_geometry() { + return Err(EINVAL); + } + let primary_phys = self.kernel_vm.ttb(); + let secondary_phys = self.inner.lock().secondary_ttb1.ok_or(EIO)?; + // SAFETY: both roots are pages of the bootloader-reserved `pagetables` region. + let primary = unsafe { Page::borrow_phys(&primary_phys) }.ok_or(EIO)?; + // SAFETY: as above. + let secondary = unsafe { Page::borrow_phys(&secondary_phys) }.ok_or(EIO)?; + let mirrored = primary.with_pointer_into_page(0, UAT_PGSZ, |primary_pointer| { + secondary.with_pointer_into_page(0, UAT_PGSZ, |secondary_pointer| { + // SAFETY: each pointer covers its whole root page; the pages differ. + let primary_entries = unsafe { + core::slice::from_raw_parts( + primary_pointer as *const u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + // SAFETY: as above. + let secondary_entries = unsafe { + core::slice::from_raw_parts_mut( + secondary_pointer as *mut u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + Ok(mirror_host_root_entries(primary_entries, secondary_entries)) + }) + })?; + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + dev_info!( + self.dev.as_ref(), + "MMU: mirrored {} host high-root entries into GFX1 at {:#x}\n", + mirrored, + secondary_phys + ); + Ok(mirrored) + } + + /// Unpublished, driver-private VM for the qualified G18 compute fixture. + /// Its VA-zero allocation is required by the saved native user payload. + /// This allocates page tables only; no context slot or TTB root is written. + pub(crate) fn new_t8152_compute_fixture_vm(&self) -> Result { + if !self.is_t8152_geometry() { return Err(ENODEV); } + // The user-VM allocator of a non-Handoff UAT starts at VA 0, as the + // module-1134 internal fixture VM's did. + self.new_vm(0x8152_0001, IOVA_USER_BASE..T8152_GEOMETRY.user_range().end)? + .with_t8152_lifetime() + } + + /// G18's 26A428 HAL400 exposes 64 context IDs (8c44c50..8c44c58). + /// The T8152 UAT already reserves the native firmware roots in 0..2; + /// use a separate, retained lower/empty-upper pair. This claims a slot + /// only. Root publication remains explicit and bounded by the owning G18 + /// firmware session's lifetime. + pub(crate) fn prepare_t8152_execution_context( + &self, vm: &Vm, + ) -> Result { + if !self.is_t8152_geometry() { + return Err(ENODEV); + } + { + let owner = vm.gpuvm.lock_inner(); + if owner.is_kernel || !core::ptr::eq( + Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner), + ) { + return Err(EINVAL); + } + } + // Module 1134 claims the lowest free context in 3..63 with a CAS on + // its own bitmap; here the same bitmap is the shared execution-context + // set, seeded with 0..2 for NoPeerRoots and taken under the UAT lock. + let context = { + let mut inner = self.inner.lock(); + let free = !inner.execution_contexts & !0b111u64; + if free == 0 { + return Err(ENOSPC); + } + let index = free.trailing_zeros() as usize; + inner.execution_contexts |= 1u64 << index; + index + }; + // Construct the slot guard first so a failed private-root allocation + // returns its claim. This high VM is never exposed for mappings. + let slot = M5ContextSlot { inner: self.inner.clone(), context }; + let range = IOVA_USER_BASE..T8152_GEOMETRY.user_range().end; + let high = self.new_vm((0x8152u64 << 16) + 0x1000 + context as u64, range)? + .with_t8152_lifetime()?; + let low = vm.clone(); + let roots = context_roots(context, &low, &high)?; + Ok(G18ExecutionContext { slot, low, high, roots, published: false }) + } +} + +struct M5ContextSlot { + inner: Arc, + context: usize, +} + +impl Drop for M5ContextSlot { + fn drop(&mut self) { + // An unexpected published root is not ours to retract or recycle. + // Keep its bit quarantined rather than hand it to another client. + let mut inner = self.inner.lock(); + let slot = &inner.ttbs()[self.context]; + let empty = slot.ttb0.load(Ordering::Acquire) == 0 + && slot.ttb1.load(Ordering::Acquire) == 0; + if empty { + inner.execution_contexts &= !(1u64 << self.context); + } + } +} + +/// Retained M5 application root lease, shared by render and compute queues. +/// +/// Native M5/G18 compute has a private lower root and a distinct empty upper +/// root. Its firmware-global root is not copied into the application slot. +/// Firmware owns contexts 0..2. Application contexts are claimed before any +/// root becomes visible, and the lease retains both VM roots. The submission +/// owner must retain the lease until completion or confirmed processor stop. +pub(crate) struct G18ExecutionContext { + slot: M5ContextSlot, + low: Vm, + high: Vm, + /// The tagged roots of `low` and `high`, computed when `low` was set + /// (`Vm::ttb` is fallible on this driver; module 1134 recomputed them). + roots: [u64; 2], + published: bool, +} + +fn context_roots(context: usize, low: &Vm, high: &Vm) -> Result<[u64; 2]> { + let tag = (context as u64) << TTBR_ASID_SHIFT | TTBR_VALID; + Ok([low.ttb() | tag, high.ttb() | tag]) +} + +impl G18ExecutionContext { + pub(crate) fn vm(&self) -> &Vm { &self.low } + + pub(crate) fn context_id(&self) -> u8 { self.slot.context as u8 } + + pub(crate) fn matches(&self, vm: &Vm) -> bool { + core::ptr::eq(&*self.low.gpuvm, &*vm.gpuvm) + } + + pub(crate) fn roots(&self) -> [u64; 2] { + self.roots + } + + /// Publish only after all command-owned mappings have been prepared. + /// An occupied slot is never replaced, even by a second M5 probe owner. + pub(crate) fn publish(&mut self) -> Result { + if self.published { return Err(EBUSY); } + let roots = self.roots(); + { + let inner = self.slot.inner.lock(); + let slot = &inner.ttbs()[self.slot.context]; + if slot.ttb0.load(Ordering::Acquire) != 0 + || slot.ttb1.load(Ordering::Acquire) != 0 { + return Err(EBUSY); + } + slot.ttb0.store(roots[0], Ordering::Release); + slot.ttb1.store(roots[1], Ordering::Release); + self.published = true; + } + fence(Ordering::SeqCst); + mem::tlbi_asid(self.context_id()); + mem::sync(); + Ok(()) + } + + pub(crate) fn is_current(&self) -> bool { + if !self.published { return false; } + let roots = self.roots(); + let inner = self.slot.inner.lock(); + let slot = &inner.ttbs()[self.slot.context]; + slot.ttb0.load(Ordering::Acquire) == roots[0] + && slot.ttb1.load(Ordering::Acquire) == roots[1] + } + + /// Retarget this lease at another application VM without giving up the + /// hardware context slot. Any published roots are retracted and the ASID + /// invalidated first, exactly as on drop; the caller has already verified + /// that no work of the previous VM is outstanding. Publication of the new + /// lower root is explicit again, once the new owner's graph is prepared. + pub(crate) fn rebind(&mut self, vm: &Vm) -> Result { + { + let owner = vm.gpuvm.lock_inner(); + if owner.is_kernel || !core::ptr::eq( + Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.slot.inner), + ) { + return Err(EINVAL); + } + } + let roots = context_roots(self.slot.context, vm, &self.high)?; + self.retract(); + self.low = vm.clone(); + self.roots = roots; + Ok(()) + } + + fn retract(&mut self) { + if !self.published { return; } + let roots = self.roots(); + { + let inner = self.slot.inner.lock(); + let slot = &inner.ttbs()[self.slot.context]; + // Only our exact pair may be retracted. VM owners remain alive + // through root invalidation and the final ASID synchronization. + if slot.ttb0.load(Ordering::Acquire) == roots[0] + && slot.ttb1.load(Ordering::Acquire) == roots[1] { + slot.ttb0.store(0, Ordering::Release); + slot.ttb1.store(0, Ordering::Release); + } + } + fence(Ordering::SeqCst); + mem::tlbi_asid(self.context_id()); + mem::sync(); + self.published = false; + } +} + +impl Drop for G18ExecutionContext { + fn drop(&mut self) { + self.retract(); + } +} + +impl Vm { + /// Attach the sticky status and job lifetime that every module-1134 VM + /// had, and nothing else: no residency gate (no idle reclaim or shrinker), + /// no T8140 context-view bindings and no shared-footprint tracking. Use + /// this instead of [`Vm::with_status`] for every VM of the T8152 UAT. + pub(crate) fn with_t8152_lifetime(mut self) -> Result { + if self.status.is_none() { + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + } + self.lifetime = Some(lifetime::VmLifetime::new()?); + Ok(self) + } + + /// The module-1134 name of [`Vm::retain_job`]. + pub(crate) fn retain_t8140_job(&self) -> Result { + self.retain_job() + } + + /// Reserve a checked number of trailing unmapped pages for M5 resources. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_in_range_with_guards( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_pages: usize, + ) -> Result { + let guard_size = guard_pages.checked_mul(UAT_PGSZ).ok_or(EOVERFLOW)?; + self.map_in_range_with_guard_size( + gem, object_range, alignment, range, prot, guard_size, + ) + } + + /// Map a GEM object at a specific address and reserve `guard_pages` + /// trailing unmapped pages in the same allocator node, so an alias of a + /// guarded allocation keeps its hardware guard gap in this Vm too. + pub(crate) fn map_at_with_guards( + &self, + addr: u64, + size: usize, + gem: &gem::Object, + prot: Prot, + guard_pages: usize, + ) -> Result { + if size != gem.size() { + return Err(EINVAL); + } + let guard = guard_pages.checked_mul(UAT_PGSZ).ok_or(EOVERFLOW)?; + let reserved = size.checked_add(guard).ok_or(EOVERFLOW)?; + let end = addr.checked_add(reserved as u64).ok_or(EOVERFLOW)?; + // The range holds exactly one placement, so this is a fixed-address + // mapping of the whole object followed by its guard reservation. + self.map_in_range_lowest(gem, 0..size, UAT_PGSZ as u64, addr..end, prot, guard) + } +} + +impl KernelMapping { + /// Map this shared GEM subrange into the end of another mapping's guards. + /// The caller must keep GPU work quiescent until the complete resource + /// update is published. Existing mappings are never removed or relocated. + pub(crate) fn map_alias_into_guard( + &self, anchor: &mut KernelMapping, prot: Prot, + ) -> Result { + let size = self.size(); + let gem: &gem::Object = self.0._gem.as_deref().ok_or(EINVAL)?; + if size == 0 || size & UAT_PGMSK != 0 || prot.is_cached_noncoherent() + || !gem.is_kernel() + || self.0.offset.checked_add(size).ok_or(EOVERFLOW)? > gem.size() + { + return Err(EINVAL); + } + let available = anchor.0.size().checked_sub(anchor.size() as u64).ok_or(EINVAL)?; + if size as u64 > available { return Err(ENOSPC); } + let owner_ref = anchor.0.owner.clone(); + // As in Vm::map_at_reserving: hold the backing and prepare the VM BO + // outside the VM reservation, and end the bound-device lease first. + let (_bind_lease, vm_bo) = { + let gate = crate::binding::lookup(&owner_ref.lock_inner().dev)?; + let binding = gate.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = owner_ref.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; + // SAFETY: no other UniqRefGpuVm exists, as in Vm::map_at_reserving. + let mut vm_exec = unsafe { owner_ref.exec_lock_vm(None, false)? }; + let inner = vm_exec.unique_gpuvm.data(); + let mut vm_bo = Some(MappingBo::new(vm_bo, owner_ref.clone())); + // As in map_at_with_guards, the fallible allocator operation must not + // own DMA-reservation references whose Drop would reacquire this lock. + let node = anchor.0.as_mut().split_tail(size as u64, KernelMappingInner { + owner: owner_ref.clone(), uat_inner: inner.uat_inner.clone(), prot, + bo: None, _gem: None, offset: self.0.offset, mapped_size: size, + }); + let mut node = match node { + Ok(node) => node, + Err(error) => { drop(vm_exec); return Err(error); } + }; + { + let payload = node.as_mut().inner_mut(); + payload.bo = vm_bo.take(); + payload._gem = Some(gem::KernelMappingPin::new(gem)); + } + let mapped = inner.map_node(&node, prot); + if let Err(error) = mapped { + // Restore reservation coverage before the temporary mapping is + // dropped. Its normal Drop unmaps partial PTEs and invalidates + // every live context; the detached node cannot free the guard. + let rollback = anchor.0.as_mut().rejoin_tail(node.as_mut()); + drop(vm_exec); + drop(KernelMapping(ManuallyDrop::new(node))); + rollback?; + return Err(error); + } + drop(vm_exec); + Ok(KernelMapping(ManuallyDrop::new(node))) + } + + /// Undo a staged guard-tail alias before publication. Consume tails in + /// reverse growth order so each remains adjacent to its original anchor. + /// The caller owns the tail and must ensure no GPU work can reference it. + pub(crate) fn rollback_guard_tail(&mut self, mut tail: KernelMapping) -> Result { + if tail.0.prot.is_cached_noncoherent() { return Err(EINVAL); } + self.0.as_mut().rejoin_tail(tail.0.as_mut())?; + // KernelMapping::drop removes the PTEs and flushes live contexts. Its + // now-invalid MM node leaves the restored guard reservation intact. + drop(tail); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 2b04726e43878..7ed0d552a4879 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -770,6 +770,26 @@ impl G17Resources { self.sgx_read32::() } + /// M5: set the low bit of the PMGR boot control word at SGX+0xe90000, as the native + /// 25G76 driver does before either firmware role starts. The word reads 0x30000 (or + /// 0x30001 when already set) on J813; anything else is refused. + #[cfg(CONFIG_DRM_ASAHI_M5)] + pub(crate) fn m5_match_pmgr_boot_control(&self) -> Result { + const OFFSET: usize = 0xe90000; + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let before = sgx.try_read32(OFFSET)?; + if before != 0x30000 && before != 0x30001 { + dev_err!(self.dev.as_ref(), "M5: unexpected PMGR boot control {:#x}\n", before); + return Err(EIO); + } + sgx.try_write32(before | 1, OFFSET)?; + if sgx.try_read32(OFFSET)? != 0x30001 { + return Err(EIO); + } + Ok(()) + } + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. pub(crate) fn enable_g17_gating(&self) -> Result { diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index 9f13436b4caf5..fd3de19a96d29 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -3,14 +3,14 @@ //! Prepared timestamp destinations shared by native command completion paths. use crate::mmu; -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] use crate::{file, gem}; -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] use core::ops::Range; -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] use core::sync::atomic::{fence as barrier, Ordering}; use kernel::{prelude::*, sync::Arc}; -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] use kernel::{uapi, xarray}; /// A firmware-mapped timestamp buffer with a kernel CPU mapping of its backing, kept for the @@ -27,7 +27,7 @@ pub(crate) enum Buffer { /// releasing the CPU mapping takes the reservation lock: the deferral runs the release /// on the cleanup queue. Firmware(crate::cleanup::Deferred), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Cpu { // The GEM pin keeps the object's held SG table alive through CPU publication. gem: gem::KernelMappingPin, @@ -45,7 +45,7 @@ impl Buffer { } /// Retain a CPU-written byte range without publishing any firmware address. - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] pub(crate) fn cpu(bo: gem::ObjectRef, range: Range) -> Result { let size = range.end.checked_sub(range.start).ok_or(EINVAL)?; if size == 0 || (range.start | size) & mmu::UAT_PGMSK != 0 || range.end > bo.size() { @@ -61,16 +61,16 @@ impl Buffer { pub(crate) fn size(&self) -> usize { match self { Self::Firmware(buffer) => buffer.mapping.size(), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Self::Cpu { range, .. } => range.end - range.start, } } - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { match self { Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Self::Cpu { gem, range, .. } => { if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { return Err(ERANGE); @@ -85,7 +85,7 @@ impl Buffer { pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { match self { Self::Firmware(buffer) => buffer.mapping.timestamp_offset(offset), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Self::Cpu { .. } => Err(EINVAL), } } @@ -95,7 +95,7 @@ impl Buffer { pub(crate) fn write_timestamp(&self, offset: usize, value: u64) { match self { Self::Firmware(buffer) => buffer.cpu.write(offset, value), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Self::Cpu { .. } => {} } } @@ -104,13 +104,13 @@ impl Buffer { pub(crate) fn firmware_mapping(&self) -> Result> { match self { Self::Firmware(buffer) => Ok(buffer.mapping.clone()), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Self::Cpu { .. } => Err(EINVAL), } } } -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] fn checked_offset(offset: u64, size: usize, aligned: bool) -> Result { let offset = usize::try_from(offset).map_err(|_| EINVAL)?; let end = offset.checked_add(size_of::()).ok_or(EINVAL)?; @@ -123,14 +123,14 @@ fn checked_offset(offset: u64, size: usize, aligned: bool) -> Result { Ok(offset) } -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] struct Destination { // Retain the bound range and backing along with its prepared CPU view. _mapping: Arc, cpu: mmu::TimestampMapping, } -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] impl Destination { fn resolve( objects: Pin<&xarray::XArray>>, @@ -165,12 +165,13 @@ impl Destination { } /// Timestamp mappings resolved and retained before any command is enqueued. -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] pub(crate) struct Destinations(Option; 4]>>); -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] impl Destinations { /// G16 retains its eight-byte destination alignment and EINVAL contract. + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn resolve_aligned( objects: Pin<&xarray::XArray>>, timestamps: &[uapi::drm_asahi_timestamps], @@ -178,6 +179,15 @@ impl Destinations { Self::resolve_inner(objects, timestamps, true) } + /// G18 keeps module 1134's rule: any in-range offset, ERANGE beyond the buffer. + #[cfg(CONFIG_DRM_ASAHI_G18)] + pub(crate) fn resolve_unaligned( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + ) -> Result { + Self::resolve_inner(objects, timestamps, false) + } + fn resolve_inner( objects: Pin<&xarray::XArray>>, timestamps: &[uapi::drm_asahi_timestamps], diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 7d2e408e34486..2cba04cd6f443 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -551,6 +551,38 @@ static const struct avd_variant avd_t6041_variant = { .quirks = AVD_QUIRK_HEVC_V5_HEADER, }; +/* + * M5 (Hibiscus, macOS CAvdMcpuHibiscus): T8140's VP block layout and HEVC + * header; avd-fw's v5 tier 2 tunables match macOS's Hibiscus table. + */ +static const struct avd_variant avd_t8142_variant = { + .vp_slots = { + [AVD_CODEC_HEVC] = 2, + [AVD_CODEC_H264] = 1, + [AVD_CODEC_VP9] = 1, + [AVD_CODEC_AV1] = 1, + }, + .vp_slot_base = { + [AVD_CODEC_HEVC] = 0, + [AVD_CODEC_H264] = 4, + [AVD_CODEC_VP9] = 8, + [AVD_CODEC_AV1] = 9, + }, + .fifo_slots = 7, + .configure_stream = t8122_configure_stream, + .capabilities = AVD_CAPABILITY_HEVC | + AVD_CAPABILITY_H264 | + AVD_CAPABILITY_VP9 | + AVD_CAPABILITY_AV1, + .fw_name = "apple/avd-fw-v5-t2.bin", + .revision = 4, + .vp_slot_offset = 0xc, + .submit_offset = 0x40, + .submit_queue_max_offset = 0x44, + .submit_queue_status_offset = 0x7c, + .quirks = AVD_QUIRK_HEVC_V5_HEADER | AVD_QUIRK_H264_MV_128, +}; + /* can also be derived from a version register */ static const struct of_device_id avd_of_match[] = { { @@ -589,6 +621,10 @@ static const struct of_device_id avd_of_match[] = { .compatible = "apple,t8140-avd", .data = &avd_t8140_variant }, + { + .compatible = "apple,t8142-avd", + .data = &avd_t8142_variant + }, {}, }; @@ -770,3 +806,4 @@ MODULE_FIRMWARE("apple/avd-fw-v3-t1.bin"); MODULE_FIRMWARE("apple/avd-fw-v4-t0.bin"); MODULE_FIRMWARE("apple/avd-fw-v5-t0.bin"); MODULE_FIRMWARE("apple/avd-fw-v5-t1.bin"); +MODULE_FIRMWARE("apple/avd-fw-v5-t2.bin"); diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index fd1d4d53025ae..df80d61acdef6 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -120,9 +120,11 @@ static const u32 default_8x8_inter[] = { 0x191b1c1e, 0x1b1c1e20, 0x1c1e2021, 0x1e202123, }; -static inline u32 mv_color_size(u32 w, u32 h) +static inline u32 mv_color_size(struct avd_ctx *ctx, u32 w, u32 h) { - return (DIV_ROUND_UP(w, 16) + 1) * (DIV_ROUND_UP(h, 16) + 1) * 64; + u32 per_mb = ctx->dev->variant->quirks & AVD_QUIRK_H264_MV_128 ? 128 : 64; + + return (DIV_ROUND_UP(w, 16) + 1) * (DIV_ROUND_UP(h, 16) + 1) * per_mb; } /* sorry for the formatting */ @@ -562,7 +564,7 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) dma_addr_t mv_color_addr = vb ? vb2_dma_contig_plane_dma_addr(vb, 0) + (vb->planes[0].length - - mv_color_size(fmt_width(ctx), + mv_color_size(ctx, fmt_width(ctx), fmt_height(ctx))) : run->addresses.mv_color; @@ -782,7 +784,7 @@ static void avd_h264_run_preamble(struct avd_ctx *ctx, struct avd_h264_run *run) dst_len = run->base.bufs.dst->vb2_buf.planes[0].length; - mv_color_len = mv_color_size(fmt_width(ctx), fmt_height(ctx)); + mv_color_len = mv_color_size(ctx, fmt_width(ctx), fmt_height(ctx)); run->addresses.mv_color = run->base.y_out + (dst_len - mv_color_len); } @@ -1157,7 +1159,7 @@ static void avd_h264_adjust_decoded_fmt(struct avd_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp) { pix_mp->plane_fmt[0].sizeimage += - mv_color_size(pix_mp->width, pix_mp->height); + mv_color_size(ctx, pix_mp->width, pix_mp->height); } static int avd_h264_try_ctrl(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl) diff --git a/drivers/media/platform/apple/avd/avd-hw.c b/drivers/media/platform/apple/avd/avd-hw.c index b64c0bf4d78ea..e973ba9d25b0f 100644 --- a/drivers/media/platform/apple/avd/avd-hw.c +++ b/drivers/media/platform/apple/avd/avd-hw.c @@ -61,8 +61,10 @@ int avd_boot(struct avd_dev *avd) /* wait for cm3 to boot */ ret = readl_poll_timeout(avd->mbox + AVD_REG_FLAG0_SET, val, val == 1, 10, 10000); - if (ret) + if (ret) { + dev_err(avd->dev, "cm3 did not start: flag0 %#x\n", val); return ret; + } return 0; } diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index a1825a1029ec5..1313a07e00389 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -41,6 +41,8 @@ #define AVD_QUIRK_HEVC_V5_HEADER BIT(2) /* T8140: pace only HEVC submissions, with a shorter back-off. */ #define AVD_QUIRK_V5_QUEUE_PACING BIT(3) +/* H.264 co-located motion vectors take 128 bytes per macroblock, not 64. */ +#define AVD_QUIRK_H264_MV_128 BIT(4) #define VP_SLOT_NONE 255 #define INST_FIFO_SLOT_NONE 255 diff --git a/drivers/media/platform/apple/avd/tests/program-test.py b/drivers/media/platform/apple/avd/tests/program-test.py index 2778a416d3a93..ab84291ea3c7c 100755 --- a/drivers/media/platform/apple/avd/tests/program-test.py +++ b/drivers/media/platform/apple/avd/tests/program-test.py @@ -47,6 +47,7 @@ #define VISIBLE_IF_KUNIT #define AVD_QUIRK_LSR 1 #define AVD_QUIRK_NO_PIPE_STATE 2 +#define AVD_QUIRK_H264_MV_128 (1U << 4) #define AVD_CODEC_HEVC 0 #define AVD_CODEC_H264 1 #define AVD_CODEC_VP9 2 diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 15b1ca2816fbc..4b096aa29dddf 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -696,7 +696,7 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) memset(isp->data_surf->virt + len, 0, setfile->size - len); release_firmware(fw); /* Seen on T6040 only; T8140 runs its setfiles as they are. */ - if (isp->hw->gen == ISP_GEN_T6040) + if (isp->hw->gen == ISP_GEN_T6040 || isp->hw->setfile_stride_fixup) isp_setfile_fixup_h17(isp->data_surf->virt, len); err = isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index a2390964a13d8..627a9bdb9e94d 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -1083,6 +1083,64 @@ static const struct apple_isp_hw apple_isp_hw_t8132 = { .deferred_start = true, }; +/* + * T8142 (M5, J813): the Neo's ISP17a. The J813 ADT's dart-isp + * dapf-instance-0 ranges, rounded out to 16 KiB pages; mostly the T8140 + * windows at +0x80000000, with one DSID window and two new ranges. + */ +static const struct isp_mmio_window apple_isp_fw_mmio_t8142[] = { + { 0x220104000, 0x14000 }, /* DSID broadcast-clear window */ + { 0x3803c0000, 0x28000 }, /* PMGR, with the PMP clock scratch */ + { 0x380704000, 0x4000 }, /* PMGR, ISP power states */ + { 0x380730000, 0x4000 }, + { 0x380e3c000, 0x4000 }, + { 0x382724000, 0x4000 }, /* PMP bandwidth scratch */ + { 0x39062c000, 0x30000 }, + { 0x3a1340000, 0x4000 }, + { 0x3a1358000, 0xc000 }, + { 0x421660000, 0x4000 }, +}; + +static const struct apple_isp_hw apple_isp_hw_t8142 = { + .gen = ISP_GEN_T8140, + .fw_abi = ISP_FW_ABI_H17, + .pmu_base = 0x0, + + .dsid_count = 1, + .dsid_clr_base0 = 0x220116000, + .dsid_clr_range0 = 0x2fc, + + .clock_scratch = 0x3803d0ca0, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x382724000, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + .mbox_irq_enable = ISP_MBOX_IRQ_ENABLE_T6031, + + /* + * No BES scaler: the firmware asserts on CH_CROP_SET ("BesProc is + * removed/not present. Must config AuxSclPr") and takes the SCL1 set. + */ + .scl1 = true, + .lpdp = true, + .meta_size = ISP_META_SIZE_T8140, + + .fw_iova_mask = GENMASK_ULL(35, 0), + .mbox2_offset = 0x410, + .mbox_irq_route = true, + .coproc_control = ISP_COPROC_CONTROL_T8140, + .fw_mmio = apple_isp_fw_mmio_t8142, + .num_fw_mmio = ARRAY_SIZE(apple_isp_fw_mmio_t8142), + .boot_mode = 1, + .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, + .resident_fw = true, + /* The J813 IMX958 runs the xx24 setfile 2224_01, as T6040 does. */ + .setfile_stride_fixup = true, +}; + static const struct of_device_id apple_isp_of_match[] = { { .compatible = "apple,t8103-isp", .data = &apple_isp_hw_t8103 }, { .compatible = "apple,t8112-isp", .data = &apple_isp_hw_t8112 }, @@ -1092,6 +1150,7 @@ static const struct of_device_id apple_isp_of_match[] = { { .compatible = "apple,t6030-isp", .data = &apple_isp_hw_t6030 }, { .compatible = "apple,t6031-isp", .data = &apple_isp_hw_t6031 }, { .compatible = "apple,t8140-isp", .data = &apple_isp_hw_t8140 }, + { .compatible = "apple,t8142-isp", .data = &apple_isp_hw_t8142 }, { .compatible = "apple,t6041-isp", .data = &apple_isp_hw_t6040 }, { .compatible = "apple,t8132-isp", .data = &apple_isp_hw_t8132 }, {}, diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 738590fec3f35..9056c3d7af143 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -251,6 +251,9 @@ struct apple_isp_hw { * detected only once. */ bool deferred_start; + + /* The xx24 setfiles' lens shading strides need zeroing (T8142). */ + bool setfile_stride_fixup; }; enum isp_sensor_id { diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index e88af94489c05..546e5cdbb72ad 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -63,6 +63,11 @@ static const struct mfd_cell apple_smc_devs[] = { static const struct mfd_cell apple_smc_low_gpio_dev = MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"); +/* T8152 currently exposes read-only telemetry; firmware owns fan control. */ +static const struct mfd_cell apple_smc_telemetry_devs[] = { + MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), +}; + static void apple_smc_read_sram(struct apple_smc *smc, void *buf, size_t size) { if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) @@ -541,6 +546,13 @@ static int apple_smc_probe(struct platform_device *pdev) return dev_err_probe(smc->dev, ret, "Failed to get key count"); smc->key_count = be32_to_cpu(count); + /* Notifications, GPIO and power management are not qualified on T8152. */ + if (of_device_is_compatible(dev->of_node, "apple,t8152-smc")) + return devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, + apple_smc_telemetry_devs, + ARRAY_SIZE(apple_smc_telemetry_devs), + NULL, 0, NULL); + /* Enable notifications */ apple_smc_write_flag(smc, SMC_KEY(NTAP), true); ret = devm_add_action_or_reset(dev, apple_smc_disable_notifications, smc); @@ -572,6 +584,7 @@ static int apple_smc_probe(struct platform_device *pdev) } static const struct of_device_id apple_smc_of_match[] = { + { .compatible = "apple,t8152-smc" }, { .compatible = "apple,t8103-smc" }, { .compatible = "apple,smc" }, {}, diff --git a/drivers/net/wireless/apple/Kconfig b/drivers/net/wireless/apple/Kconfig index e40d5d9c5fd53..d3ead4d9259e7 100644 --- a/drivers/net/wireless/apple/Kconfig +++ b/drivers/net/wireless/apple/Kconfig @@ -2,18 +2,24 @@ config WLAN_VENDOR_APPLE bool "Apple wireless devices" default y + help + Support for Apple's wireless network devices. if WLAN_VENDOR_APPLE config APPLE_CENTAURI - tristate "Apple N1 Centauri firmware and IPC transport" + tristate "Apple N1 Centauri experimental J813 Wi-Fi" depends on ARCH_APPLE || COMPILE_TEST - depends on PCI && CFG80211 && PCIE_APPLE_APIF + depends on PCI && CFG80211 && DEBUG_FS select FW_LOADER help - Boot the shared N1 firmware and establish its PCI DMA contexts. - Requires a preinitialized APIF host with the supported reset layout - and DT-supplied personalized firmware name and platform identifier. - The endpoint must start in the stopped ROM personality. Firmware - and per-device calibration are external inputs and are not included. + Experimental Apple N1 PCIe and ACIPC Wi-Fi support for the J813 + MacBook Air. Firmware boot, ring setup and connection setup currently + use debugfs and the tools/aurora/j813 workbench. The tested single-link + WPA2 setup completes DHCP and bidirectional traffic. Suspend/resume, + roaming and automatic startup are not yet qualified. + + The firmware and per-device calibration are private external inputs. + This driver does not include Bluetooth HCI support. + To build as a module, choose M; the module is apple_centauri. endif diff --git a/drivers/net/wireless/apple/centauri/Makefile b/drivers/net/wireless/apple/centauri/Makefile index ddd2501998887..1a17e5283e227 100644 --- a/drivers/net/wireless/apple/centauri/Makefile +++ b/drivers/net/wireless/apple/centauri/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_CENTAURI) += apple_centauri.o -apple_centauri-y := centauri_pci.o centauri_ipc.o -apple_centauri-$(CONFIG_APPLE_CENTAURI_WIFI) += centauri_wifi.o +apple_centauri-y := centauri_pci.o centauri_port.o centauri_ipc.o centauri_wifi.o diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h index 55106d4ef0794..330b9e1a207d9 100644 --- a/drivers/net/wireless/apple/centauri/centauri.h +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -1,87 +1,96 @@ - +/* SPDX-License-Identifier: GPL-2.0-only */ #ifndef AURORA_CENTAURI_H #define AURORA_CENTAURI_H #include -#include #include +#include #include #include -#include -#define CEN_ALPHA_WINDOW_SIZE SZ_32M #define CEN_ALPHA_SCRATCH_OFFSET SZ_8M #define CEN_ALPHA_SCRATCH_SIZE (5 * SZ_1M) struct centauri_ipc; struct centauri_wifi; -struct centauri_irqs { - struct pci_dev *pdev; - void *data; - unsigned int count; -}; - struct cen_alpha_resources { - struct pci_dev *pdev; void *context, *messages, *window; dma_addr_t context_dma, messages_dma, window_dma; void __iomem *bar; u32 index[4], mtr, mcr; }; -enum centauri_boot_phase { - CEN_BOOT_ROM, - CEN_BOOT_CYCLE, - CEN_BOOT_IPC, - CEN_BOOT_READY, - CEN_BOOT_FAILED, - CEN_BOOT_STOPPED, -}; - struct centauri { struct pci_dev *pdev; void __iomem *bar; resource_size_t size; - u32 platform_id, protocol; - const char *firmware_name; + u32 stage_offset; struct dentry *debug; struct mutex boot_lock; - struct delayed_work boot_work; - enum centauri_boot_phase boot_phase; - bool removing, ready; - void *firmware, *descriptor; - struct centauri_irqs irqs, alpha_irqs; - dma_addr_t firmware_dma, descriptor_dma; + void *firmware, *descriptor, *msi_page; + dma_addr_t firmware_dma, descriptor_dma, msi_dma; size_t firmware_size; u32 image_response; + u16 msi_original; + int msi_cap; bool boot_attempted; bool link_ready; struct centauri_ipc *ipc; struct centauri_wifi *wifi; }; -int centauri_request_irqs(struct centauri_irqs *irqs, struct pci_dev *pdev, - unsigned int count, irq_handler_t handler, void *data); -void centauri_free_irqs(struct centauri_irqs *irqs); +/* One row per machine with an N1: the FTAB file name, the word written to BAR+0x807c, + * (board-id << 16) | chip-id from the machine's ADT /chosen, and the physical addresses and + * the one differing register value of the port cycle (centauri_port.c). + * J813: board 0x24, chip 0x8142 (measured); port, PHY lane and PERST pad from the live ADT. + * J873 (Mac mini M6): board 0x24, chip 0x8152 from its ADT, the composition taken from the J813 + * word; port 0 config 0x388028000, lane 0x38f010000 and PERST# gpio0 pin 3 (0x31a000000 + 4 * 3) + * from its ADT; config +0x130 is 0x208 as in the m1n1 T8152 bring-up that links the port. + * keep_port_tables: the J813 sequence zeroes the 16 words at port+0x3000 and the 256 words at + * port+0x3800. On the J873 those are the RID2SID and MSIMAP tables that pcie-apple programmed + * (PORT_T602X_RID2SID, PORT_T602X_MSIMAP); the cycle then drops function 0's stream entry and + * its DMA arrives as SID 0 (DART "BPF reject"). Such a board gets its RID2SID table written + * back; the MSI map and the MSI address/config are restored on every board. + * link_tlv, scan_tlvs: the macOS 27.0 WLAN adapter differs from the 26.6.1 one the J813 recipe + * came from. startInterface carries the three link addresses in TLV 386 (was 385), and the scan + * request (command 0x1000f) carries a channel-list TLV 0x104 and an OCE TLV 0x165 after the fixed + * 14 bytes. The J873 scan crash in SCHED_Req_Add came from a zero passive dwell in those 14 bytes + * (see build_scan), not from the missing TLVs; whether the TLVs are required is untested. + */ +struct centauri_board { + const char *compatible; + const char *firmware; + u32 board_word; + u64 port, phy, pad; + u32 tune_130; + u16 bridge_id; /* root port device ID, 0 = any Apple root port */ + bool keep_port_tables; + u16 link_tlv; + bool scan_tlvs; +}; + +static inline const struct centauri_board *centauri_board(void) +{ + static const struct centauri_board boards[] = { + { "apple,j813", "apple/centauri/j813-ftab.bin", 0x00248142, + 0x488028000ULL, 0x48f010000ULL, 0x39a000300ULL, 0x03020000, 0x100c, false, + 385, false }, + { "apple,j873g", "apple/centauri/j873-ftab.bin", 0x00248152, + 0x388028000ULL, 0x38f010000ULL, 0x31a00000cULL, 0x208, 0, true, + 386, true }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(boards); i++) + if (of_machine_is_compatible(boards[i].compatible)) + return &boards[i]; + return NULL; +} +int centauri_port_cycle(struct pci_dev *pdev); int centauri_start_ipc(struct centauri *c); void centauri_free_ipc(struct centauri *c); int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *resources); -bool centauri_stop_dma(struct pci_dev *pdev); - -#if IS_ENABLED(CONFIG_APPLE_CENTAURI_WIFI) -irqreturn_t centauri_alpha_irq(int irq, void *data); +int centauri_alpha_log_buffer(struct centauri *c, void **cpu, dma_addr_t *dma, size_t *size); int centauri_wifi_register(struct centauri *c); void centauri_wifi_unregister(struct centauri *c); -#else -static inline irqreturn_t centauri_alpha_irq(int irq, void *data) -{ - return IRQ_HANDLED; -} -static inline int centauri_wifi_register(struct centauri *c) -{ - return 0; -} -static inline void centauri_wifi_unregister(struct centauri *c) -{ -} -#endif +void centauri_wifi_kick(struct centauri *c); #endif diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c new file mode 100644 index 0000000000000..4c26c70853815 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -0,0 +1,588 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Native DMA implementation of the measured J813 Control/Alpha bootstrap. + * Wire layout source: AppleCentauriCommon/scripts/alpha_wifi_scan.py v259. + * Polling and bounded bootstrap rings are deliberate bring-up restrictions. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "centauri.h" + +struct cen_buffer { + struct list_head list; + struct device *dev; + void *cpu; + dma_addr_t dma; + size_t size; +}; + +struct cen_context { + struct cen_buffer *buffer, *messages; + u32 index[4], mtr, mcr; + u16 tr_count, cr_count, message_head; +}; + +struct centauri_ipc { + struct list_head buffers; + struct pci_dev *function[3]; + void __iomem *bar[3]; + struct cen_context control, alpha; + struct cen_buffer *secondary, *window, *cchi_rx, *cchi_tx, *debug_rx; + struct cen_buffer *alpha_log; + struct cen_buffer *msi[3]; + unsigned int vectors[3]; + u16 cchi_head, cchi_tail; +}; + +/* Each function has its own IOMMU stream on the J873 (RID2SID entries 1, 2, 3 and separate DART + * domains), so a buffer a function's firmware reads or writes has to be mapped through that + * function's own struct device. Function 0 is the Control firmware, 1 Alpha (Wi-Fi), 2 Beta. + */ +static struct cen_buffer *buffer_alloc(struct centauri *c, unsigned int function, size_t size) +{ + struct cen_buffer *buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + struct pci_dev *owner = function ? c->ipc->function[function] : c->pdev; + + if (!buffer) + return NULL; + if (!owner) { + kfree(buffer); + return NULL; + } + buffer->dev = &owner->dev; + buffer->size = PAGE_ALIGN(size); + buffer->cpu = dma_alloc_coherent(buffer->dev, buffer->size, &buffer->dma, GFP_KERNEL); + if (!buffer->cpu) { + kfree(buffer); + return NULL; + } + memset(buffer->cpu, 0, buffer->size); + dev_dbg(&c->pdev->dev, "dma buffer fn%u %#llx..%#llx (%zu bytes)\n", function, + (u64)buffer->dma, (u64)buffer->dma + buffer->size, buffer->size); + list_add_tail(&buffer->list, &c->ipc->buffers); + return buffer; +} + +static int ftab_entry(const void *image, size_t size, const char tag[4], + u32 *offset, u32 *length, bool sparse) +{ + u32 count, i, off, len; + const u8 *table = image; + + if (size < 0x30 || memcmp(table + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(table + 0x28); + if (!count || count > 256 || 0x30 + count * 16 > size) + return -EINVAL; + for (i = 0; i < count; i++) { + const u8 *entry = table + 0x30 + 16 * i; + + if (memcmp(entry, tag, 4)) + continue; + off = get_unaligned_le32(entry + 4); + len = get_unaligned_le32(entry + 8); + if (off < 0x30 + count * 16 || !len || (u64)off + len > (sparse ? SZ_64M : size)) + return -EINVAL; + *offset = off; + *length = len; + return 0; + } + return -ENOENT; +} + +/* The native secondary FTAB reserves a bounded "cras" region. Capture it + * after a firmware abort; these private diagnostics may contain device data. + */ +static ssize_t firmware_crash_read(struct file *file, char __user *buffer, + size_t count, loff_t *pos) +{ + struct centauri *c = file->private_data; + struct cen_buffer *secondary; + u32 offset, length; + ssize_t ret; + + mutex_lock(&c->boot_lock); + if (!c->ipc || !c->ipc->secondary) { + ret = -ENODEV; + goto unlock; + } + secondary = c->ipc->secondary; + ret = ftab_entry(secondary->cpu, secondary->size, "cras", &offset, &length, false); + if (ret) + goto unlock; + dma_rmb(); + ret = simple_read_from_buffer(buffer, count, pos, secondary->cpu + offset, length); +unlock: + mutex_unlock(&c->boot_lock); + return ret; +} + +static const struct file_operations firmware_crash_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = firmware_crash_read, + .llseek = default_llseek, +}; + +static int secondary_init(struct centauri *c) +{ + u32 offset, length, count, i, required; + const u8 *seed; + int ret; + + ret = ftab_entry(c->firmware, c->firmware_size, "2ftb", &offset, &length, false); + if (ret) + return ret; + seed = c->firmware + offset; + if (length < 0x30 || memcmp(seed + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(seed + 0x28); + if (!count || count > 256 || 0x30 + 16 * count != length) + return -EINVAL; + required = length; + for (i = 0; i < count; i++) { + u64 end = (u64)get_unaligned_le32(seed + 0x34 + i * 16) + + get_unaligned_le32(seed + 0x38 + i * 16); + + if (end > SZ_64M) + return -EINVAL; + required = max_t(u32, required, end); + } + c->ipc->secondary = buffer_alloc(c, 0, required); + if (!c->ipc->secondary) + return -ENOMEM; + memcpy(c->ipc->secondary->cpu, seed, length); + dma_wmb(); + pci_write_config_dword(c->pdev, 0xf88, lower_32_bits(c->ipc->secondary->dma)); + pci_write_config_dword(c->pdev, 0xf8c, upper_32_bits(c->ipc->secondary->dma)); + pci_write_config_dword(c->pdev, 0xf90, required); + dev_info(&c->pdev->dev, "secondary memory: seed=%u required=%u\n", length, required); + debugfs_create_file("firmware_crash", 0400, c->debug, c, &firmware_crash_fops); + return 0; +} + +static int context_init(struct centauri *c, unsigned int function, struct cen_context *ctx, + u16 tr, u16 cr, dma_addr_t scratch, size_t scratch_size) +{ + u8 *header; + dma_addr_t dma; + int i; + + ctx->buffer = buffer_alloc(c, function, SZ_32K); + ctx->messages = buffer_alloc(c, function, SZ_16K); + if (!ctx->buffer || !ctx->messages) + return -ENOMEM; + ctx->tr_count = tr; + ctx->cr_count = cr; + ctx->index[0] = 0x78; + ctx->index[1] = ctx->index[0] + cr * 2; + ctx->index[2] = ctx->index[1] + tr * 2; + ctx->index[3] = ctx->index[2] + cr * 2; + ctx->mtr = ALIGN(ctx->index[3] + tr * 2, tr == 192 ? 16 : 2); + ctx->mcr = ALIGN(ctx->mtr + 256, 4); + header = ctx->buffer->cpu; + dma = ctx->buffer->dma; + put_unaligned_le16(768, header); + put_unaligned_le16(0x68, header + 2); + put_unaligned_le64(dma + 0x68, header + 8); + for (i = 0; i < 4; i++) + put_unaligned_le64(dma + ctx->index[i], header + 0x10 + 8 * i); + put_unaligned_le16(cr, header + 0x30); + put_unaligned_le16(tr, header + 0x32); + put_unaligned_le64(dma + ctx->mcr, header + 0x34); + put_unaligned_le64(dma + ctx->mtr, header + 0x3c); + put_unaligned_le16(16, header + 0x44); + put_unaligned_le16(16, header + 0x46); + put_unaligned_le64(scratch ?: dma + ALIGN(ctx->mcr + 256, 8), header + 0x58); + put_unaligned_le64(scratch_size, header + 0x60); + dma_wmb(); + return 0; +} + +static int handshake(void __iomem *bar, struct cen_context *ctx, + u32 status, u32 address, u32 control) +{ + u32 value; + int ret; + + writel(1, bar + control); + ret = readl_poll_timeout(bar + status, value, value == 1, 100, 2000000); + if (ret) + return ret; + writel(lower_32_bits(ctx->buffer->dma), bar + address); + writel(upper_32_bits(ctx->buffer->dma), bar + address + 4); + dma_wmb(); + writel(2, bar + control); + return readl_poll_timeout(bar + status, value, value == 2, 100, 2000000); +} + +static int message(struct cen_context *ctx, void __iomem *doorbell, + const void *payload, size_t length) +{ + u16 head = ctx->message_head; + u8 *descriptor; + u16 tail; + int ret; + + if (head >= 15 || !length || length > 512) + return -ENOSPC; + memcpy(ctx->messages->cpu + 512 * (head + 1), payload, length); + descriptor = ctx->buffer->cpu + ctx->mtr + head * 16; + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(ctx->messages->dma + 512 * (head + 1), descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + dma_wmb(); + writel(head + 1, doorbell); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) == head + 1, + 100, 1200000, false, *(__le16 *)(ctx->buffer->cpu + ctx->index[1])); + dma_rmb(); + ctx->message_head++; + return ret; +} + +static int open_transfer(struct centauri *c, u16 id, struct cen_buffer *buffer, + u16 footer) +{ + u8 msg[0x34] = { 1 }; + u32 words = footer >> 2, shift = min_t(u32, __ffs(words), 4); + + msg[2] = words >> shift; + msg[3] = shift << 4; + put_unaligned_le16(id, msg + 4); + put_unaligned_le64(buffer->dma, msg + 8); + put_unaligned_le64(~0ULL, msg + 0x10); + put_unaligned_le16(16, msg + 0x18); + put_unaligned_le16(0xffff, msg + 0x1a); + put_unaligned_le16(id, msg + 0x1c); + put_unaligned_le16(0x40, msg + 0x1e); + return message(&c->ipc->control, c->bar + 0x9000, msg, sizeof(msg)); +} + +static int cchi_init(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + u8 config[4] = {9, 0, 0xff, 0xff}, *entry; + u16 tail; + int i, ret; + + ipc->cchi_rx = buffer_alloc(c, 0, SZ_32K); + ipc->cchi_tx = buffer_alloc(c, 0, SZ_32K); + ipc->debug_rx = buffer_alloc(c, 0, SZ_16K); + if (!ipc->cchi_rx || !ipc->cchi_tx || !ipc->debug_rx) + return -ENOMEM; + ret = message(&ipc->control, c->bar + 0x9000, config, sizeof(config)); + if (ret) + return ret; + ret = open_transfer(c, 4, ipc->cchi_rx, 1024); + if (!ret) + ret = open_transfer(c, 3, ipc->cchi_tx, 1024); + if (!ret) + ret = open_transfer(c, 5, ipc->debug_rx, 256); + if (ret) + return ret; + for (i = 0; i < 15; i++) { + entry = ipc->cchi_rx->cpu + i * 1040; + put_unaligned_le32(1 | (1024 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + entry = ipc->debug_rx->cpu + i * 272; + put_unaligned_le32(1 | (256 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + } + dma_wmb(); + writel(15, c->bar + 0x9010); + writel(15, c->bar + 0x9014); + entry = ipc->cchi_tx->cpu; + put_unaligned_le32(1 | (64 << 8), entry); + put_unaligned_le32(1, entry + 12); + put_unaligned_le16(64, entry + 16); /* CCHI gid 0, oid 0, seq 0 hello. */ + dma_wmb(); + writel(1, c->bar + 0x900c); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) != 0, + 100, 2000000, false, + *(__le16 *)(ipc->control.buffer->cpu + ipc->control.index[1] + 4 * 2)); + dma_rmb(); + if (ret) + return ret; + entry = ipc->cchi_rx->cpu; + if ((get_unaligned_le32(entry) >> 8) < 8 || entry[18] || entry[19]) + return -EPROTO; + ipc->cchi_head = ipc->cchi_tail = 1; + dev_info(&c->pdev->dev, "Control CCHI hello response received\n"); + return 0; +} + +static int memswap_handoff(struct centauri *c, unsigned int function, const char tag[4]) +{ + struct cen_buffer *secondary = c->ipc->secondary, *working; + u32 offset, size; + int ret; + + ret = ftab_entry(secondary->cpu, secondary->size, tag, &offset, &size, false); + if (ret) + return ret; + working = buffer_alloc(c, function, size); + if (!working) + return -ENOMEM; + dma_rmb(); + memcpy(working->cpu, secondary->cpu + offset, size); + dma_wmb(); + pci_write_config_dword(c->ipc->function[function], 0xf88, lower_32_bits(working->dma)); + pci_write_config_dword(c->ipc->function[function], 0xf8c, upper_32_bits(working->dma)); + pci_write_config_dword(c->ipc->function[function], 0xf90, size); + dev_info(&c->pdev->dev, "memswap function %u: %.4s %u bytes\n", function, tag, size); + return 0; +} + +static irqreturn_t function_irq(int irq, void *data) +{ + centauri_wifi_kick(data); + return IRQ_HANDLED; +} + +/* Real MSIs need an MSI controller on the port (pcie-apple, pcie-apple-msi). The data rings + * name Alpha vectors 3 and 8, so a function gets all 16 vectors or none; one that gets none + * keeps the DMA mailbox below (polled). Alpha asks first: on the J873 the 10GbE NIC shares + * the controller's 32 vectors and only one block of 16 is left. + */ +static int request_vectors(struct centauri *c, unsigned int function) +{ + struct pci_dev *pdev = c->ipc->function[function]; + int count, i, ret; + + count = pci_alloc_irq_vectors(pdev, 16, 16, PCI_IRQ_MSI); + if (count < 0) + return count; + for (i = 0; i < count; i++) { + ret = request_irq(pci_irq_vector(pdev, i), function_irq, 0, "centauri", c); + if (ret) { + while (i--) + free_irq(pci_irq_vector(pdev, i), c); + pci_free_irq_vectors(pdev); + return ret; + } + } + c->ipc->vectors[function] = count; + return 0; +} + +static void free_vectors(struct centauri *c, unsigned int function) +{ + struct pci_dev *pdev = c->ipc->function[function]; + + while (c->ipc->vectors[function]) + free_irq(pci_irq_vector(pdev, --c->ipc->vectors[function]), c); + pci_free_irq_vectors(pdev); +} + +static int enumerate_functions(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + u16 control; + int i, cap; + dma_addr_t msi; + + pci_lock_rescan_remove(); + /* The ROM personality was single-function; boot exposes Alpha and Beta. */ + c->pdev->multifunction = 1; + pci_rescan_bus(c->pdev->bus); + pci_unlock_rescan_remove(); + for (i = 2; i >= 0; i--) { + ipc->function[i] = pci_get_domain_bus_and_slot(pci_domain_nr(c->pdev->bus), + c->pdev->bus->number, PCI_DEVFN(0, i)); + if (!ipc->function[i] || ipc->function[i]->vendor != PCI_VENDOR_ID_APPLE || + (i && ipc->function[i]->device != 0x1901 + i)) + return -ENODEV; + if (!(pci_resource_flags(ipc->function[i], 0) & IORESOURCE_MEM)) + return -ENODEV; + ipc->bar[i] = pci_iomap(ipc->function[i], 0, 0); + if (!ipc->bar[i]) + return -ENOMEM; + cap = pci_find_capability(ipc->function[i], PCI_CAP_ID_MSI); + if (!cap) + return -ENODEV; + pci_read_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, &control); + if (!(control & PCI_MSI_FLAGS_64BIT)) + return -EOPNOTSUPP; + if (i < 2 && !request_vectors(c, i)) { + dev_info(&c->pdev->dev, "function %d: %u MSIs\n", i, ipc->vectors[i]); + pci_set_master(ipc->function[i]); + continue; + } + if (i) { + ipc->msi[i] = buffer_alloc(c, i, SZ_16K); + if (!ipc->msi[i]) + return -ENOMEM; + } + msi = i ? ipc->msi[i]->dma : c->msi_dma; + pci_write_config_dword(ipc->function[i], cap + PCI_MSI_ADDRESS_LO, + lower_32_bits(msi)); + pci_write_config_dword(ipc->function[i], cap + PCI_MSI_ADDRESS_HI, + upper_32_bits(msi)); + pci_write_config_word(ipc->function[i], cap + PCI_MSI_DATA_64, 0xa5a0 + 0x100 * i); + pci_write_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, + (control & ~PCI_MSI_FLAGS_QSIZE) | (5 << 4) | PCI_MSI_FLAGS_ENABLE); + pci_set_master(ipc->function[i]); + } + return 0; +} + +int centauri_start_ipc(struct centauri *c) +{ + struct centauri_ipc *ipc; + u32 stage; + int ret; + + if (!c->firmware || !c->link_ready || c->ipc || readl(c->bar + 0x8000) != 1) + return -EBUSY; + ipc = kzalloc(sizeof(*ipc), GFP_KERNEL); + if (!ipc) + return -ENOMEM; + INIT_LIST_HEAD(&ipc->buffers); + c->ipc = ipc; + ret = enumerate_functions(c); + if (ret) + goto failed; + ret = secondary_init(c); + if (ret) + goto failed; + ret = context_init(c, 0, &ipc->control, 13, 1, 0, 0); + if (ret) + goto failed; + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + writel(0x62746632, c->bar + 0x8074); /* 2ftb */ + dma_wmb(); + writel(1, c->bar + 0x9070); + ret = readl_poll_timeout(c->bar + 0x8000, stage, stage == 2 || stage == 3, 100, 3000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + dev_info(&c->pdev->dev, "Control firmware reached OS stage\n"); + ret = handshake(c->bar, &ipc->control, 0x8050, 0x8054, 0x907c); + if (ret) + goto failed; + dev_info(&c->pdev->dev, "Control IPC handshake complete\n"); + ret = memswap_handoff(c, 0, "mswc"); + if (!ret) + ret = memswap_handoff(c, 2, "mswb"); + if (!ret) + ret = memswap_handoff(c, 1, "msww"); + if (ret) + goto failed; + ret = cchi_init(c); + if (ret) + goto failed; + ret = readl_poll_timeout(ipc->bar[1] + 8, stage, stage == 2 || stage == 3, 100, 2000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + ipc->window = buffer_alloc(c, 1, SZ_16M); + if (!ipc->window) { + ret = -ENOMEM; + goto failed; + } + ret = context_init(c, 1, &ipc->alpha, 192, 234, + ipc->window->dma + CEN_ALPHA_SCRATCH_OFFSET, + CEN_ALPHA_SCRATCH_SIZE); + if (ret) + goto failed; + writel(lower_32_bits(ipc->window->dma), ipc->bar[1] + 0x18); + writel(upper_32_bits(ipc->window->dma), ipc->bar[1] + 0x1c); + writel(SZ_16M, ipc->bar[1] + 0x20); + ret = handshake(ipc->bar[1], &ipc->alpha, 0xc, 0x10, 0x1028); + if (ret) + goto failed; + dev_info(&c->pdev->dev, "Alpha Wi-Fi firmware running; IPC handshake complete\n"); + return centauri_wifi_register(c); +failed: + dev_err(&c->pdev->dev, "IPC bootstrap failed: %d (Control stage=%#x)\n", + ret, readl(c->bar + 0x8000)); + /* Published DMA stays pinned until every function is stopped at removal. */ + return ret; +} + +void centauri_free_ipc(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + struct cen_buffer *buffer, *next; + int i, cap; + u16 control; + + if (!ipc) + return; + for (i = 0; i < 3; i++) + if (ipc->vectors[i]) + free_vectors(c, i); + centauri_wifi_unregister(c); + for (i = 0; i < 3; i++) { + if (!ipc->function[i]) + continue; + cap = pci_find_capability(ipc->function[i], PCI_CAP_ID_MSI); + if (cap) { + pci_read_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, &control); + pci_write_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, + control & ~PCI_MSI_FLAGS_ENABLE); + } + pci_clear_master(ipc->function[i]); + } + list_for_each_entry_safe(buffer, next, &ipc->buffers, list) { + dma_free_coherent(buffer->dev, buffer->size, buffer->cpu, buffer->dma); + list_del(&buffer->list); + kfree(buffer); + } + for (i = 0; i < 3; i++) { + if (ipc->bar[i]) + pci_iounmap(ipc->function[i], ipc->bar[i]); + pci_dev_put(ipc->function[i]); + } + kfree(ipc); + c->ipc = NULL; +} + +int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *resources) +{ + struct centauri_ipc *ipc = c->ipc; + + if (!ipc || !ipc->window || !ipc->alpha.buffer || !ipc->alpha.messages) + return -ENODEV; + resources->context = ipc->alpha.buffer->cpu; + resources->context_dma = ipc->alpha.buffer->dma; + resources->messages = ipc->alpha.messages->cpu; + resources->messages_dma = ipc->alpha.messages->dma; + resources->window = ipc->window->cpu; + resources->window_dma = ipc->window->dma; + resources->bar = ipc->bar[1]; + memcpy(resources->index, ipc->alpha.index, sizeof(resources->index)); + resources->mtr = ipc->alpha.mtr; + resources->mcr = ipc->alpha.mcr; + return 0; +} + +/* Called once under the Wi-Fi lock. Published logging memory stays owned by + * IPC until every function has bus mastering disabled in centauri_free_ipc. + */ +int centauri_alpha_log_buffer(struct centauri *c, void **cpu, dma_addr_t *dma, size_t *size) +{ + struct centauri_ipc *ipc = c->ipc; + + if (!ipc || !ipc->window) + return -ENODEV; + if (!ipc->alpha_log) + ipc->alpha_log = buffer_alloc(c, 1, SZ_4M); + if (!ipc->alpha_log) + return -ENOMEM; + *cpu = ipc->alpha_log->cpu; + *dma = ipc->alpha_log->dma; + *size = ipc->alpha_log->size; + return 0; +} diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c new file mode 100644 index 0000000000000..86ad99c48db2e --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Initial real-hardware N1 PCI/ACIPC backend for J813. + * Register offsets are from AppleCentauriCommon/PROTOCOL.md and its measured + * J813 prototype. Firmware execution and radio operations are separate gates. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "centauri.h" + +static struct dentry *centauri_debug; + +static unsigned int board_word; +module_param(board_word, uint, 0644); +MODULE_PARM_DESC(board_word, "Diagnostic: word written to BAR0+0x807c before the ROM boot, 0 = per-machine table value"); + +static int centauri_status_show(struct seq_file *s, void *unused) +{ + struct centauri *c = s->private; + + seq_printf(s, "pci=%s vendor=%04x device=%04x\n", + pci_name(c->pdev), c->pdev->vendor, c->pdev->device); + if (c->link_ready) + seq_printf(s, "bar0_size=%pa exec_stage=%#x\n", &c->size, + readl(c->bar + c->stage_offset)); + else + seq_puts(s, "link unavailable; endpoint reads disabled\n"); + seq_printf(s, "transport=hardware-pci firmware_boot=%s image_response=%#x wifi=%s\n", + c->boot_attempted ? "attempted" : "not-started", c->image_response, + READ_ONCE(c->wifi) ? "registered" : "not-owned-by-this-function"); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(centauri_status); + +static void centauri_free_boot(struct centauri *c) +{ + struct device *dev = &c->pdev->dev; + + if (!c->link_ready && c->boot_attempted) { + dev_warn(dev, "retaining DMA buffers after failed link transition; reboot required\n"); + return; + } + /* Stop device writes before any DMA storage is released. */ + centauri_free_ipc(c); + if (c->msi_cap) + pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, + c->msi_original & ~PCI_MSI_FLAGS_ENABLE); + pci_clear_master(c->pdev); + if (c->firmware) + dma_free_coherent(dev, c->firmware_size, c->firmware, c->firmware_dma); + if (c->descriptor) + dma_free_coherent(dev, SZ_16K, c->descriptor, c->descriptor_dma); + if (c->msi_page) + dma_free_coherent(dev, SZ_16K, c->msi_page, c->msi_dma); + c->firmware = c->descriptor = c->msi_page = NULL; +} + +static int centauri_rom_boot(struct centauri *c) +{ + const struct firmware *fw; + const struct centauri_board *board = centauri_board(); + struct device *dev = &c->pdev->dev; + u32 stage; + int ret; + + if ((c->pdev->device != 0x1900 && c->pdev->device != 0x1901) || !board) + return -ENODEV; + if (!c->link_ready || c->boot_attempted) + return -EBUSY; + stage = readl(c->bar + 0x8000); + if (stage != 0 && stage != 4 && stage != 1) + return -EBUSY; + ret = request_firmware(&fw, board->firmware, dev); + if (ret) + return ret; + if (fw->size < 0x30 || fw->size > SZ_64M || + memcmp(fw->data + 0x20, "rkosftab", 8)) { + ret = -EINVAL; + goto release; + } + c->firmware_size = PAGE_ALIGN(fw->size); + c->firmware = dma_alloc_coherent(dev, c->firmware_size, &c->firmware_dma, GFP_KERNEL); + c->descriptor = dma_alloc_coherent(dev, SZ_16K, &c->descriptor_dma, GFP_KERNEL); + c->msi_page = dma_alloc_coherent(dev, SZ_16K, &c->msi_dma, GFP_KERNEL); + if (!c->firmware || !c->descriptor || !c->msi_page) { + ret = -ENOMEM; + goto free_buffers; + } + memset(c->firmware, 0, c->firmware_size); + memcpy(c->firmware, fw->data, fw->size); + memset(c->descriptor, 0, SZ_16K); + memset(c->msi_page, 0, SZ_16K); + /* Measured ROM scatter entry: u64 DMA, u32 length, u32 flags, u64 x2 zero. */ + put_unaligned_le64(c->firmware_dma, c->descriptor); + put_unaligned_le32(ALIGN(fw->size, SZ_4K), c->descriptor + 8); + c->msi_cap = pci_find_capability(c->pdev, PCI_CAP_ID_MSI); + if (!c->msi_cap) { + ret = -ENODEV; + goto free_buffers; + } + pci_read_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, &c->msi_original); + if (!(c->msi_original & PCI_MSI_FLAGS_64BIT)) { + ret = -EOPNOTSUPP; + goto free_buffers; + } + /* Poll the measured DMA-backed MSI mailbox until native IRQ support exists. */ + pci_write_config_dword(c->pdev, c->msi_cap + PCI_MSI_ADDRESS_LO, lower_32_bits(c->msi_dma)); + pci_write_config_dword(c->pdev, c->msi_cap + PCI_MSI_ADDRESS_HI, upper_32_bits(c->msi_dma)); + pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_DATA_64, 0xa5a0); + pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, + (c->msi_original & ~PCI_MSI_FLAGS_QSIZE) | PCI_MSI_FLAGS_ENABLE); + pci_set_master(c->pdev); + dev_info(dev, "ROM boot: %s, %zu bytes, board word %#x\n", board->firmware, fw->size, + board_word ?: board->board_word); + writel(board_word ?: board->board_word, c->bar + 0x807c); + writel(0, c->bar + 0x803c); + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + dma_wmb(); + c->boot_attempted = true; + if (stage == 4 || stage == 1) { + c->image_response = readl(c->bar + 0x8040); + dev_info(dev, "adopted firmware stage %u; native DMA buffers ready\n", stage); + ret = 0; + goto release; + } + writel(1, c->bar + 0x9000); + ret = readl_poll_timeout(c->bar + 0x8040, c->image_response, + c->image_response != 0, 20000, 20000000); + if (!ret) + ret = readl_poll_timeout(c->bar + 0x8000, stage, stage != 0, 20000, 20000000); + else + stage = readl(c->bar + 0x8000); + dma_rmb(); + dev_info(dev, "ROM firmware result: response=%#x stage=%#x mailbox=%#x poll=%d\n", + c->image_response, stage, READ_ONCE(*(u32 *)c->msi_page), ret); + /* Keep DMA buffers owned until device removal, even on timeout. */ + if (!ret && stage != 4) + ret = -EIO; + goto release; +free_buffers: + centauri_free_boot(c); +release: + release_firmware(fw); + return ret; +} + +static ssize_t centauri_boot_write(struct file *file, const char __user *buffer, + size_t count, loff_t *pos) +{ + struct centauri *c = file->private_data; + u32 value; + int ret = kstrtou32_from_user(buffer, count, 0, &value); + + if (ret || value < 1 || value > 3) + return ret ?: -EINVAL; + mutex_lock(&c->boot_lock); + if (value == 1) { + ret = centauri_rom_boot(c); + } else if (value == 3) { + ret = centauri_start_ipc(c); + } else if (!c->firmware || !c->link_ready || readl(c->bar + 0x8000) != 4) { + ret = -EBUSY; + } else { + c->link_ready = false; + ret = centauri_port_cycle(c->pdev); + if (!ret) { + c->link_ready = true; + dev_info(&c->pdev->dev, "post-reset execution stage=%#x\n", + readl(c->bar + 0x8000)); + } + } + mutex_unlock(&c->boot_lock); + return ret ?: count; +} + +static const struct file_operations centauri_boot_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = centauri_boot_write, +}; + +static int centauri_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct centauri *c; + dma_addr_t dma; + void *page; + int ret; + + c = devm_kzalloc(&pdev->dev, sizeof(*c), GFP_KERNEL); + if (!c) + return -ENOMEM; + c->pdev = pdev; + c->link_ready = true; + mutex_init(&c->boot_lock); + c->stage_offset = id->driver_data; + ret = pcim_enable_device(pdev); + if (ret) + return ret; + c->size = pci_resource_len(pdev, 0); + if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || + c->size < c->stage_offset + sizeof(u32)) + return dev_err_probe(&pdev->dev, -ENODEV, "invalid ACIPC BAR0\n"); + ret = pcim_iomap_regions(pdev, BIT(0), "apple_centauri"); + if (ret) + return ret; + c->bar = pcim_iomap_table(pdev)[0]; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return dev_err_probe(&pdev->dev, ret, "40-bit DMA unavailable\n"); + /* Test Linux's DART allocation without publishing it to the device. */ + page = dma_alloc_coherent(&pdev->dev, SZ_16K, &dma, GFP_KERNEL); + if (!page) + return dev_err_probe(&pdev->dev, -ENOMEM, "DART DMA allocation failed\n"); + memset(page, 0, SZ_16K); + dev_info(&pdev->dev, "ACIPC endpoint reachable: stage=%#x, 40-bit DMA mapping=%pad\n", + readl(c->bar + c->stage_offset), &dma); + dma_free_coherent(&pdev->dev, SZ_16K, page, dma); + /* Bus mastering remains disabled until the firmware transport owns buffers. */ + pci_clear_master(pdev); + pci_set_drvdata(pdev, c); + c->debug = debugfs_create_dir(pci_name(pdev), centauri_debug); + debugfs_create_file("status", 0400, c->debug, c, ¢auri_status_fops); + debugfs_create_file("rom_boot", 0200, c->debug, c, ¢auri_boot_fops); + return 0; +} + +static void centauri_remove(struct pci_dev *pdev) +{ + struct centauri *c = pci_get_drvdata(pdev); + + centauri_wifi_unregister(c); + debugfs_remove_recursive(c->debug); + centauri_free_boot(c); +} + +/* + * The N1 firmware keeps running across s2idle: it cannot be restarted without + * the bootloader's link training. Without PM callbacks the PCI core disables a + * function at suspend, which turns bus mastering off under the running + * firmware and leaves every ring dead (each scan then fails with -EIO). Save + * the state ourselves so the core keeps the function enabled and in D0, and + * keep the PCIe host's power domains up through the noirq phase. + */ +static int centauri_suspend(struct device *dev) +{ + device_set_awake_path(dev); + return pci_save_state(to_pci_dev(dev)); +} + +static DEFINE_SIMPLE_DEV_PM_OPS(centauri_pm_ops, centauri_suspend, NULL); + +/* 0x1903 is the Beta (Bluetooth) function: btcentauri claims it. This driver needs functions + * 0 and 1 (1 is enabled here, its BAR is used by the Wi-Fi code) and reaches function 2 only + * through config space and its struct device for DMA. + */ +static const struct pci_device_id centauri_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1900), .driver_data = 0x8000 }, + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1901), .driver_data = 0x8000 }, + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1902), .driver_data = 0x0008 }, + { } +}; +MODULE_DEVICE_TABLE(pci, centauri_ids); + +static struct pci_driver centauri_driver = { + .name = "apple_centauri", + .id_table = centauri_ids, + .probe = centauri_probe, + .remove = centauri_remove, + .driver.pm = pm_sleep_ptr(¢auri_pm_ops), +}; + +static int __init centauri_init(void) +{ + int ret; + + centauri_debug = debugfs_create_dir("apple_centauri", NULL); + ret = pci_register_driver(¢auri_driver); + if (ret) + debugfs_remove_recursive(centauri_debug); + return ret; +} + +static void __exit centauri_exit(void) +{ + pci_unregister_driver(¢auri_driver); + debugfs_remove_recursive(centauri_debug); +} +module_init(centauri_init); +module_exit(centauri_exit); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 Centauri native PCI and ACIPC bring-up backend"); +MODULE_FIRMWARE("apple/centauri/j813-ftab.bin"); +MODULE_FIRMWARE("apple/centauri/j873-ftab.bin"); diff --git a/drivers/net/wireless/apple/centauri/centauri_port.c b/drivers/net/wireless/apple/centauri/centauri_port.c new file mode 100644 index 0000000000000..59eadc3f8521c --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_port.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Preboot -> boot port cycle, transcribed from the J813 measured proxy + * alpha_wifi_scan.py pcie_cycle/apple_teardown/apple_bringup sequence. + * Apertures and the PERST pad come from the per-machine table in centauri.h; those were + * verified from the live ADT. The J873 row reuses the J813 sequence with T8152 addresses and + * config +0x130 = 0x208: the reset table and the lane window bits equal the T8152 bring-up that + * links port 0 in m1n1, but the cycle itself (link down, retrain with firmware running) has not + * been run on a J873. This temporary board helper belongs in the host-controller driver. + */ +#include +#include +#include +#include +#include +#include "centauri.h" + +static void update(void __iomem *base, u32 offset, u32 clear, u32 set) +{ + writel((readl(base + offset) & ~clear) | set, base + offset); +} + +static bool wait_bits(void __iomem *address, u32 mask, u32 wanted, u32 us) +{ + u32 value; + + return !readl_poll_timeout(address, value, (value & mask) == wanted, 100, us); +} + +int centauri_port_cycle(struct pci_dev *pdev) +{ + static const u32 reset[][2] = { + {0x88, 0x110}, {0x100, ~0U}, {0x148, ~0U}, {0x210, ~0U}, + {0x80, 0}, {0x84, 0}, {0x104, 0xfffffff0}, {0x124, 0x100}, + {0x16c, 0}, {0x13c, 0}, {0x800, 0x00100100}, + {0x808, 0x001001ff}, {0x82c, 0x00010000}, + }; + u32 tune[][2] = { + {0x130, 0}, {0x140, 0x10}, {0x144, 0x00253770}, + {0x21c, 0}, {0x834, 0}, {0x83c, 0}, + }; + const struct centauri_board *board = centauri_board(); + struct pci_dev *bridge = pdev->bus->self; + void __iomem *port, *phy, *pad; + u32 value, gpio, identity; + u16 link; + unsigned int lanes, i; + int ret = -ETIMEDOUT, pm; + bool ack, l2; + u32 rid2sid[16], msimap[256], msicfg = 0, msiaddr = 0, valid = 0; + bool saved = false; + + if (!board || !board->port || !bridge || bridge->vendor != PCI_VENDOR_ID_APPLE || + (board->bridge_id && bridge->device != board->bridge_id)) + return -ENODEV; + tune[0][1] = board->tune_130; + port = ioremap(board->port, 0x4000); + phy = ioremap(board->phy, 0x1000); + pad = ioremap(board->pad, sizeof(u32)); + if (!port || !phy || !pad) { + ret = -ENOMEM; + goto out; + } + gpio = readl(pad); + if (gpio & BIT(21)) { + ret = -EPERM; + goto out; + } + ret = pci_save_state(bridge); + if (ret) + goto out; + ret = pci_save_state(pdev); + if (ret) + goto out; + if (board->keep_port_tables) + for (i = 0; i < ARRAY_SIZE(rid2sid); i++) + rid2sid[i] = readl(port + 0x3000 + 4 * i); + /* The reset below also clears an MSI controller's setup (pcie-apple, pcie-apple-msi). */ + for (i = 0; i < ARRAY_SIZE(msimap); i++) + msimap[i] = readl(port + 0x3800 + 4 * i); + msicfg = readl(port + 0x124); + msiaddr = readl(port + 0x16c); + saved = true; + pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &link); + lanes = (link & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT; + lanes = lanes >= 1 && lanes <= 16 ? BIT(lanes) - 1 : 1; + pm = pci_find_capability(pdev, PCI_CAP_ID_PM); + if (pm) { + pci_write_config_word(pdev, pm + PCI_PM_CTRL, 0x103); + msleep(10); + } + update(port, 0x800, 0, 0x100); + writel(0x11, port + 0x88); + ack = wait_bits(port + 0x88, 1, 0, 500000); + l2 = wait_bits(port + 0x208, BIT(6), BIT(6), 500000); + dev_info(&pdev->dev, "port preboot transition: PME ack=%d L2=%d\n", ack, l2); + /* Endpoint reads cease here until the root LTSSM is back in L0. */ + pci_write_config_word(bridge, PCI_COMMAND, 0); + pci_read_config_dword(bridge, PCI_PRIMARY_BUS, &value); + pci_write_config_dword(bridge, PCI_PRIMARY_BUS, value & 0xff0000ff); + update(phy, 0, BIT(31), 0); + writel((gpio & ~0xfU) | 2, pad); + if (readl(pad) != ((gpio & ~0xfU) | 2)) { + ret = -EIO; + goto out; + } + wait_bits(port + 0x804, BIT(2), 0, 50000); + wait_bits(port + 0x820, 0x1ff, 0, 100000); + update(port, 0x13c, 0, 0x100); + for (i = 0; i < 200; i++) { + pci_read_config_dword(bridge, 0x728, &value); + if ((value & 0x3f) == 0x14) + break; + usleep_range(1000, 1200); + } + wait_bits(port + 0x208, BIT(12), 0, 200000); + readl(phy); + wait_bits(port + 0xa8, 3, 0, 10000); + update(port, 0x82c, 0, BIT(24)); + wait_bits(port + 0xa8, lanes << 16, lanes << 16, 20000); + update(port, 0x82c, 0, BIT(16)); + udelay(10); + update(port, 0x82c, 1, 0); + udelay(10); + update(port, 0x82c, BIT(24), 0); + update(port, 0x13c, 0x100, 0); + update(phy, 0, BIT(31), 0); + update(phy, 0, 0x400, 0); + update(phy, 0, 0x200, 0); + update(phy, 0, 0, 0x10); + update(phy, 0, 1, 0); + update(phy, 0, 2, 0); + writel(~0U, port + 0x100); + update(port, 0x800, 1, 0); + wait_bits(port + 0x804, 1, 0, 200000); + update(phy, 0, BIT(31), 0); + for (i = 0; i < ARRAY_SIZE(reset); i++) + writel(reset[i][1], port + reset[i][0]); + for (i = 0; i < 16; i++) + writel(0, port + 0x3000 + 4 * i); + for (i = 0; i < 256; i++) + writel(0, port + 0x3800 + 4 * i); + for (i = 0; i < ARRAY_SIZE(tune); i++) + writel(tune[i][1], port + tune[i][0]); + update(port, 0x800, 0, 1); + udelay(10); + update(phy, 0, 0, 1); + if (!wait_bits(phy, 4, 4, 50000)) + goto timed_out; + update(phy, 0, 0, 2); + if (!wait_bits(phy, 8, 8, 50000)) + goto timed_out; + update(phy, 0, 0x10, 0); + udelay(100); + update(phy, 0, 0, 0x200); + update(phy, 0, 0, 0x400); + udelay(100); + update(port, 0x82c, 0, 1); + writel((gpio & ~0xfU) | 3, pad); + udelay(100); + readl(phy); + wait_bits(port + 0xa8, 3, 0, 20000); + update(port, 0x82c, BIT(16), 0); + if (!wait_bits(port + 0x804, 1, 1, 200000)) + goto timed_out; + update(port, 0x800, 0x100, 0); + pci_read_config_dword(bridge, 0x8bc, &value); + pci_write_config_dword(bridge, 0x8bc, value ^ 1); + pci_read_config_dword(bridge, 0x80, &value); + pci_write_config_dword(bridge, 0x80, value | 0x400); + update(phy, 0, 0, BIT(31)); + update(port, 0x80, 0, 1); + for (i = 0; i < 2000; i++) { + pci_read_config_dword(bridge, 0x728, &value); + if ((value & 0x3f) == 0x11) + break; + usleep_range(1000, 1200); + } + if (i == 2000) + goto timed_out; + msleep(100); + pci_restore_state(bridge); + /* Bus numbers/windows now restored; endpoint config may be read again. */ + pci_read_config_dword(pdev, PCI_VENDOR_ID, &identity); + if (identity != 0x1901106b && identity != 0x1900106b) { + ret = -ENODEV; + goto out; + } + pci_restore_state(pdev); + if (pm) { + pci_write_config_word(pdev, pm + PCI_PM_CTRL, 0); + msleep(10); + } + dev_info(&pdev->dev, "port transition complete: endpoint=%#x LTSSM=%#x\n", + identity, value & 0x3f); + ret = 0; + goto out; +timed_out: + ret = -ETIMEDOUT; +out: + if (saved) { + if (board->keep_port_tables) + for (i = 0; i < ARRAY_SIZE(rid2sid); i++) { + writel(rid2sid[i], port + 0x3000 + 4 * i); + valid += !!(rid2sid[i] & BIT(31)); + } + for (i = 0; i < ARRAY_SIZE(msimap); i++) + writel(msimap[i], port + 0x3800 + 4 * i); + writel(msiaddr, port + 0x16c); + writel(msicfg, port + 0x124); + dev_info(&pdev->dev, "restored %u RID2SID entries and the MSI setup %#x/%#x (ret %d)\n", + valid, msicfg, msiaddr, ret); + } + if (pad) + iounmap(pad); + if (phy) + iounmap(phy); + if (port) + iounmap(port); + return ret; +} diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c new file mode 100644 index 0000000000000..1c854cb459020 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -0,0 +1,2478 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* cfg80211 connection to the measured Alpha command/management rings. + * Control queues recycle across hardware-validated wraps. Packet data uses + * a separate protocol; never report transmit success before implementing it. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "centauri.h" + +#define CR_BASE 0x100000 +#define TR_BASE 0x400000 +#define RING_SPAN 0x20000 +#define TX_BUFFER 0x600000 +#define BUFFER_SIZE 2048 +#define DATA_TX_BUFFER 0xd00000 +#define DATA_RX_BUFFER 0xe00000 +#define DATA_TX_SLOTS 128 +#define DATA_TX_QUEUE_LIMIT 64 + +/* The firmware owns window +8 MiB through +13 MiB as scratch memory. */ +static_assert(DATA_TX_BUFFER >= CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); +static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= DATA_RX_BUFFER); + +static bool auto_rings = true; +module_param(auto_rings, bool, 0644); +MODULE_PARM_DESC(auto_rings, "Open the data rings (data_probe ids 76, 24, 15, 12 before the join, 25 after it) from the connect path; 1 = on (default). The data_probe debugfs files still work and are skipped for rings already open"); + +static int data_probe_locked(struct centauri_wifi *wifi, unsigned int id, bool internal); + +static bool roam_scan; +module_param(roam_scan, bool, 0644); +MODULE_PARM_DESC(roam_scan, "Let the firmware run its own roam and background scans after a join; 0 = off (default). The scans take the radio off channel for seconds (72 percent ping loss measured) until the host programs a roam profile"); + +static bool host_setup = true; +module_param(host_setup, bool, 0644); +MODULE_PARM_DESC(host_setup, "Before a join, tell the firmware the host is awake (command 6) and apply the reference TX defaults (commands 43 and 52) when the tx_config files have not; 1 = on (default)"); + +static bool verify_pmk; +module_param(verify_pmk, bool, 0400); +MODULE_PARM_DESC(verify_pmk, "Compare firmware PMK with the requested key without exposing either"); + +static bool eapol_after_assoc; +module_param(eapol_after_assoc, bool, 0644); +MODULE_PARM_DESC(eapol_after_assoc, "One EAPOL-Start probe at the next successful association event"); + +static bool link_diagnostics; +module_param(link_diagnostics, bool, 0644); +MODULE_PARM_DESC(link_diagnostics, "Query non-secret firmware link status during association"); + +static bool eapol_control_profile; +module_param(eapol_control_profile, bool, 0644); +MODULE_PARM_DESC(eapol_control_profile, "Use the measured control TX profile for the EAPOL-Start probe"); + +static bool eapol_unicast; +module_param(eapol_unicast, bool, 0644); +MODULE_PARM_DESC(eapol_unicast, "Address the bounded EAPOL-Start probe to the associated AP"); + +static bool eapol_relative_lifetime; +module_param(eapol_relative_lifetime, bool, 0644); +MODULE_PARM_DESC(eapol_relative_lifetime, "Use a measured two-second relative expiration for the probe"); + +static bool data_tx_broadcast; +module_param(data_tx_broadcast, bool, 0400); +MODULE_PARM_DESC(data_tx_broadcast, "Diagnostic: open TR 25 with measured broadcast metadata; transmit only broadcast Ethernet frames"); + +static bool data_tx_control_profile; +module_param(data_tx_control_profile, bool, 0600); +MODULE_PARM_DESC(data_tx_control_profile, "Diagnostic: select configured TX policy 1 for normal data; apply tx_config action 2 first"); + +static bool data_tx_native_prepare = true; +module_param(data_tx_native_prepare, bool, 0600); +MODULE_PARM_DESC(data_tx_native_prepare, "Use native packet preparation value 4; disable only to reproduce the legacy data TX failure"); + +static bool station_link_addresses; +module_param(station_link_addresses, bool, 0400); +MODULE_PARM_DESC(station_link_addresses, "Diagnostic: use the netdev address for each firmware link in this single-link WPA2 client"); + +struct centauri_rx_snapshot { + __le32 version, count; + struct { + u8 completion[32]; + u8 packet[4096]; + } sample[8]; +}; + +struct centauri_wifi { + struct centauri *control; + struct cen_alpha_resources a; + struct wiphy *wiphy; + struct wireless_dev wdev; + struct net_device *netdev; + struct work_struct scan_work; + struct work_struct connect_work; + struct delayed_work link_work; + struct cfg80211_scan_request *scan_request; + struct mutex lock; + /* The logger has its own coherent allocation. Reads/acknowledgements + * must remain possible while scan/connect hold the command mutex. + * Initialization takes lock then fwlog_lock; readers take only the latter. + */ + struct mutex fwlog_lock; + u16 main_head, command_head, sequence; + u32 rx_seen[5]; + u16 rx_next[5], rx_head[5]; + u16 reply_sequence, reply_status; + u32 reply_command, channels_seen, scans_done, beacons_seen; + bool reply_valid, registered, configured; + bool stopping; + bool connecting, connected, connect_event, disconnect_event; + bool associated, pmk_seen, pmk_matches; + u8 expected_pmk[32]; + u16 connect_status; + u8 bssid[ETH_ALEN], connect_payload[1024], request_ie[256]; + size_t connect_size, request_ie_size; + struct dentry *data_probe_file; + struct dentry *data_status_file; + struct dentry *data_rx_snapshot_file; + struct centauri_rx_snapshot rx_snapshot; + u32 data_rx_received, data_rx_reposted, data_rx_bad; + struct sk_buff_head data_tx_queue; + u16 data_tx_length[DATA_TX_SLOTS]; + bool data_tx_busy[DATA_TX_SLOTS]; + u32 data_tx_submitted, data_tx_completed, data_tx_failed; + u32 data_tx_coalesced, data_tx_bad; + struct dentry *station_stats_file; + struct dentry *eapol_start_file; + struct dentry *country_file; + struct dentry *link_status_file; + struct dentry *tx_config_file; + struct dentry *host_wakeup_file; + struct dentry *fwlog_start_file, *fwlog_status_file, *fwlog_buffer_file, *fwlog_ack_file; + void *fwlog_buffer; + dma_addr_t fwlog_dma; + size_t fwlog_size; + bool fwlog_attempted, fwlog_ready; + u8 tx_profile[10], tx_defaults_applied; + bool host_awake_sent; + bool tx_profile_valid; + u8 link_count, link_prime[3], link_channel[3], link_band[3], emlsr_mode; + bool link_status_valid, emlsr_valid; + u16 link_status_size; + u16 link_tags[12], link_lengths[12]; + u8 link_tlv_count; + u8 country[3]; + bool country_valid; + u8 station_stats[0x378]; + bool station_stats_valid; + u8 rx_crypto_stats[64]; + bool rx_crypto_stats_valid; + struct dentry *rx_crypto_stats_file; + u8 dot11_mib_stats[192]; + bool dot11_mib_stats_valid; + struct dentry *dot11_mib_stats_file; + u8 tx_power_report[280]; + bool tx_power_report_valid; + struct dentry *tx_power_report_file; + u16 stats_reply_size, stats_tag, stats_length; + u32 data_probe_done; + struct ieee80211_supported_band bands[2]; + struct ieee80211_channel channels[39]; +}; + +/* These helpers require wifi->lock, including from the connect worker. */ +static int transmit_eapol_start(struct centauri_wifi *wifi); +static int query_station_stats(struct centauri_wifi *wifi); +static int query_link_status(struct centauri_wifi *wifi); + +static const u32 rx_buffer[5] = { 0, 0, 0x620000, 0x680000, 0x700000 }; +static const u16 ring_size[5] = { 0, 64, 192, 96, 192 }; +static const u16 five_ghz[] = { + 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, + 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, +}; +static struct ieee80211_rate rates[] = { + { .bitrate = 10 }, { .bitrate = 20 }, { .bitrate = 55 }, { .bitrate = 110 }, + { .bitrate = 60 }, { .bitrate = 90 }, { .bitrate = 120 }, { .bitrate = 180 }, + { .bitrate = 240 }, { .bitrate = 360 }, { .bitrate = 480 }, { .bitrate = 540 }, +}; +static const u32 cipher_suites[] = { + WLAN_CIPHER_SUITE_CCMP, WLAN_CIPHER_SUITE_AES_CMAC, +}; + +static u16 index_read(struct centauri_wifi *wifi, unsigned int array, unsigned int index) +{ + return le16_to_cpu(READ_ONCE(*(__le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index))); +} + +static void index_write(struct centauri_wifi *wifi, unsigned int array, + unsigned int index, u16 value) +{ + WRITE_ONCE(*(__le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index), cpu_to_le16(value)); +} + +static void completions(struct centauri_wifi *wifi) +{ + unsigned int vector; + + for (vector = 0; vector <= 4; vector++) + index_write(wifi, 2, vector, index_read(wifi, 0, vector)); + dma_wmb(); +} + +static int alpha_message(struct centauri_wifi *wifi, const void *message, size_t size) +{ + u16 head = wifi->main_head; + u8 *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(2000); + + if (head >= 15 || size > 512) + return -ENOSPC; + memcpy(wifi->a.messages + (head + 1) * 512, message, size); + descriptor = wifi->a.context + wifi->a.mtr + 16 * head; + put_unaligned_le32(1 | (size << 8), descriptor); + put_unaligned_le64(wifi->a.messages_dma + (head + 1) * 512, descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + index_write(wifi, 3, 0, head + 1); + if (head) + index_write(wifi, 3, 8, head + 1); + dma_wmb(); + wifi->main_head++; + writel(1, wifi->a.bar + 0x1004); + do { + if (readl(wifi->a.bar + 8) != 2) + return -EIO; + if (index_read(wifi, 1, 0) >= head + 1) { + dma_rmb(); + completions(wifi); + return 0; + } + usleep_range(100, 200); + } while (time_before(jiffies, deadline)); + return -ETIMEDOUT; +} + +static int open_pair(struct centauri_wifi *wifi, u16 id) +{ + u8 cr[0x2c] = { 2 }, tr[0x34] = { 1 }; + int ret; + + put_unaligned_le32(id | (id << 16), cr + 4); + put_unaligned_le64(wifi->a.window_dma + CR_BASE + (id - 1) * RING_SPAN, cr + 8); + put_unaligned_le16(1024, cr + 0x10); + put_unaligned_le16(0xffff, cr + 0x12); + ret = alpha_message(wifi, cr, sizeof(cr)); + if (ret) + return ret; + put_unaligned_le32(id | (id << 16), tr + 4); + put_unaligned_le64(wifi->a.window_dma + TR_BASE + (id - 1) * RING_SPAN, tr + 8); + put_unaligned_le64(~0ULL, tr + 0x10); + put_unaligned_le16(ring_size[id], tr + 0x18); + put_unaligned_le16(id, tr + 0x1a); + put_unaligned_le16(id, tr + 0x1c); + put_unaligned_le16(0x40, tr + 0x1e); + put_unaligned_le16(2, tr + 0x24); + return alpha_message(wifi, tr, sizeof(tr)); +} + +static void report_beacon(struct centauri_wifi *wifi, const u8 *payload, size_t size) +{ + const u8 *frame, *ie, *end; + struct ieee80211_channel *channel; + struct cfg80211_bss *bss; + u16 frame_size; + int number = 0, frequency, signal; + enum nl80211_band band; + + if (!wifi->registered || size < 0x60 + 36) + return; + /* The ACIPC packet length is authoritative; the vendor descriptor's + * separate 802.11 length can include a stripped FCS. The measured parser + * consumes the frame bytes that were actually returned. + */ + frame_size = size - 0x60; + frame = payload + 0x60; + if (!ieee80211_is_beacon(*(__le16 *)frame) && !ieee80211_is_probe_resp(*(__le16 *)frame)) + return; + end = frame + frame_size; + for (ie = frame + 36; ie + 2 <= end && ie + 2 + ie[1] <= end; ie += 2 + ie[1]) { + if ((ie[0] == WLAN_EID_DS_PARAMS || ie[0] == WLAN_EID_HT_OPERATION) && ie[1]) + number = ie[2]; + } + if (!number) + return; + band = number <= 14 ? NL80211_BAND_2GHZ : NL80211_BAND_5GHZ; + frequency = ieee80211_channel_to_frequency(number, band); + channel = ieee80211_get_channel(wifi->wiphy, frequency); + if (!channel) + return; + signal = max((s8)payload[0x16], (s8)payload[0x17]); + if (signal >= 0 || signal < -127) + signal = -95; + bss = cfg80211_inform_bss_frame(wifi->wiphy, channel, + (struct ieee80211_mgmt *)frame, frame_size, signal * 100, GFP_KERNEL); + if (bss) { + wifi->beacons_seen++; + cfg80211_put_bss(wifi->wiphy, bss); + } +} + +static void post_receive(struct centauri_wifi *wifi, unsigned int ring, u16 slot) +{ + u8 *descriptor = wifi->a.window + TR_BASE + (ring - 1) * RING_SPAN + slot * 16; + u32 offset = rx_buffer[ring] + slot * BUFFER_SIZE; + + memset(wifi->a.window + offset, 0, BUFFER_SIZE); + put_unaligned_le32(1 | (BUFFER_SIZE << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + offset, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); +} + +/* Host-owned RX buffer cookies and completion length are the ACIPC fields + * validated in probe 28. Probe 29 validates the footer's prefix length and + * Ethernet framing with IPv4 header checksums and IPv6 payload lengths. + * Called with wifi->lock, including during connect/command polling. + */ +static void drain_data_rx(struct centauri_wifi *wifi) +{ + u16 head, tail, post; + unsigned int processed = 0; + + if (!(wifi->data_probe_done & BIT(4)) || readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 207); + tail = index_read(wifi, 2, 207); + post = index_read(wifi, 3, 12); + if (head >= 128 || tail >= 128 || post >= 128) { + wifi->data_rx_bad++; + return; + } + dma_rmb(); + while (tail != head && processed < 127) { + u8 *completion = wifi->a.window + 0x240000 + tail * 32; + u32 header = get_unaligned_le32(completion); + u32 info = get_unaligned_le32(completion + 4); + u16 cookie = info & 0xffff, length = info >> 16; + u16 next = (post + 1) % 128; + u32 count = le32_to_cpu(wifi->rx_snapshot.count); + u8 *packet, *descriptor; + u32 buffer; + + if ((header >> 16) != 12 || !cookie || cookie > 127 || + length > 4096 || get_unaligned_le32(completion + 8)) { + wifi->data_rx_bad++; + break; + } + /* Do not overwrite a descriptor the receive engine still owns. */ + if (next == index_read(wifi, 1, 12)) + break; + buffer = DATA_RX_BUFFER + (cookie - 1) * 4096; + packet = wifi->a.window + buffer; + if (count < ARRAY_SIZE(wifi->rx_snapshot.sample)) { + memcpy(wifi->rx_snapshot.sample[count].completion, completion, 32); + memcpy(wifi->rx_snapshot.sample[count].packet, packet, length); + wifi->rx_snapshot.version = cpu_to_le32(1); + wifi->rx_snapshot.count = cpu_to_le32(count + 1); + } + wifi->data_rx_received++; + if (wifi->connected && netif_running(wifi->netdev)) { + u32 footer = get_unaligned_le32(completion + 12); + unsigned int padding = (footer >> 12) & 15; + struct sk_buff *skb = NULL; + + /* Extended prefix headers are not decoded yet. */ + if (!(footer & BIT(24)) && length >= padding + ETH_HLEN) + skb = netdev_alloc_skb_ip_align(wifi->netdev, length - padding); + if (skb) { + skb_put_data(skb, packet + padding, length - padding); + skb->protocol = eth_type_trans(skb, wifi->netdev); + skb->ip_summed = CHECKSUM_NONE; + wifi->netdev->stats.rx_packets++; + wifi->netdev->stats.rx_bytes += length - padding; + netif_rx(skb); + } else { + wifi->netdev->stats.rx_dropped++; + } + } else { + wifi->netdev->stats.rx_dropped++; + } + descriptor = wifi->a.window + 0x520000 + post * 16; + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le32(cookie, descriptor + 12); + wifi->data_rx_reposted++; + tail = (tail + 1) % 128; + post = next; + processed++; + } + if (processed) { + dma_wmb(); + index_write(wifi, 2, 207, tail); + index_write(wifi, 3, 12, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + } +} + +/* Serialized with the firmware poller. Payload slots remain owned until a + * CR completion, even if the device has already consumed the TR descriptor. + */ +static void drain_data_tx(struct centauri_wifi *wifi) +{ + u16 head, tail, post; + unsigned int budget = 256; + bool posted = false, completed = false; + + if (!(wifi->data_probe_done & BIT(5)) || readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 192); + tail = index_read(wifi, 2, 192); + post = index_read(wifi, 3, 25); + if (head >= 2048 || tail >= 2048 || post >= DATA_TX_SLOTS) + return; + dma_rmb(); + while (tail != head && budget--) { + u8 *entry = wifi->a.window + 0x200000 + tail * 32; + u16 ring = get_unaligned_le32(entry) >> 16; + u16 cookie = get_unaligned_le16(entry + 4); + u16 count = get_unaligned_le16(entry + 9); + u8 status = entry[12]; + + if (ring == 25) { + unsigned int first, i; + + /* Airship cr_controller::sync maps CD bytes 9:10 to the + * transfer count. With the multicomp flag (CD byte 1 bit 0), + * the cookie names the LAST transfer; prior transfers share + * its completion status. We submit one contiguous DMA buffer + * per slot, so those preceding cookies must all still be busy. + */ + if (!cookie || cookie > DATA_TX_SLOTS || !count || + count >= DATA_TX_SLOTS || !(entry[1] & 1)) { + wifi->data_tx_bad++; + break; + } + first = (cookie + DATA_TX_SLOTS - count) % DATA_TX_SLOTS; + for (i = 0; i < count; i++) + if (!wifi->data_tx_busy[(first + i) % DATA_TX_SLOTS]) + break; + if (i != count) { + wifi->data_tx_bad++; + break; + } + if (count > 1) + wifi->data_tx_coalesced++; + for (i = 0; i < count; i++) { + unsigned int slot = (first + i) % DATA_TX_SLOTS; + + wifi->data_tx_completed++; + if (status == 0 || status == 0x83) { + wifi->netdev->stats.tx_packets++; + wifi->netdev->stats.tx_bytes += wifi->data_tx_length[slot]; + } else { + wifi->data_tx_failed++; + wifi->netdev->stats.tx_errors++; + } + wifi->data_tx_busy[slot] = false; + } + } + tail = (tail + 1) % 2048; + completed = true; + } + if (completed) + index_write(wifi, 2, 192, tail); + budget = DATA_TX_SLOTS - 1; + while (wifi->connected && !READ_ONCE(wifi->stopping) && + netif_running(wifi->netdev) && budget--) { + u16 next = (post + 1) % DATA_TX_SLOTS; + u32 buffer = DATA_TX_BUFFER + post * BUFFER_SIZE; + u8 *descriptor = wifi->a.window + 0x540000 + post * 32; + struct sk_buff *skb; + u8 destination[ETH_ALEN]; + u64 metadata = 6ULL << 27; + + if (wifi->data_tx_busy[post] || next == index_read(wifi, 1, 25)) + break; + skb = skb_dequeue(&wifi->data_tx_queue); + if (!skb) + break; + /* This diagnostic ring is only valid for its broadcast destination. + * Do not silently send unicast or IPv6 multicast through that flow. + */ + if (data_tx_broadcast && + (skb_copy_bits(skb, 0, destination, ETH_ALEN) || + !is_broadcast_ether_addr(destination))) { + wifi->netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + continue; + } + if (skb_copy_bits(skb, 0, wifi->a.window + buffer, skb->len)) { + wifi->netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + continue; + } + memset(descriptor, 0, 32); + put_unaligned_le32(3 | (skb->len << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le16(post + 1, descriptor + 12); + /* AirshipDataPacket type 1 supplies this 16-byte TX header inline. + * setTxPolicyId(1), 0x1809ed27c, sets validity bit 60 and ID bit 48. + * Keep normal policy zero by default; the switch permits a bounded + * comparison with the existing, read-back-verified control profile. + */ + if (READ_ONCE(data_tx_control_profile) && + (wifi->tx_defaults_applied & BIT(1))) + metadata |= BIT_ULL(60) | BIT_ULL(48); + /* Native 25G76 AirshipDataPacket::prepare (0x18090faf8) writes + * (argument & 7) to bits 17:15. Guest37's captured DHCP Discover + * has value 4 (0x20000). The same-connection Linux A/B changes + * zero DHCP offers into three offers by setting only this field. + * The opt-out retains the old value for diagnostic reproduction. + */ + if (READ_ONCE(data_tx_native_prepare)) + metadata |= 4ULL << 15; + put_unaligned_le64(metadata, descriptor + 16); + put_unaligned_le64(2000000, descriptor + 24); + wifi->data_tx_length[post] = skb->len; + wifi->data_tx_busy[post] = true; + wifi->data_tx_submitted++; + dev_kfree_skb_any(skb); + post = next; + posted = true; + } + if (posted || completed) { + dma_wmb(); + if (posted) + index_write(wifi, 3, 25, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1030); + } + if (wifi->connected && !READ_ONCE(wifi->stopping) && netif_running(wifi->netdev) && + skb_queue_len(&wifi->data_tx_queue) < DATA_TX_QUEUE_LIMIT) + netif_wake_queue(wifi->netdev); +} + +static void drain(struct centauri_wifi *wifi) +{ + unsigned int ring; + bool posted = false; + + for (ring = 2; ring <= 4; ring++) { + unsigned int budget = ring_size[ring]; + + while (budget--) { + u16 slot = wifi->rx_next[ring], size, status, sequence; + u32 command; + u8 *descriptor = wifi->a.window + TR_BASE + (ring - 1) * RING_SPAN + slot * 16; + u8 *packet = wifi->a.window + rx_buffer[ring] + slot * BUFFER_SIZE; + + if (!(READ_ONCE(descriptor[15]) & 4)) + break; + dma_rmb(); + size = get_unaligned_le16(packet + 0x10); + status = get_unaligned_le16(packet + 0xe); + sequence = get_unaligned_le16(packet + 0x12); + command = get_unaligned_le32(packet + 0x14) & ~BIT(30); + wifi->rx_seen[ring]++; + wifi->rx_next[ring] = (slot + 1) % ring_size[ring]; + if (size < 24 || size > BUFFER_SIZE) + goto refill; + if (ring == 4) { + report_beacon(wifi, packet + 24, size - 24); + } else if (command & BIT(31)) { + if (command == 0x80000030) + wifi->channels_seen++; + if (command == 0x80000005) + wifi->scans_done++; + if (command == 0x80000000 && wifi->connecting) { + wifi->connect_event = true; + wifi->connect_status = status; + } + if (command == 0x80000001) { + wifi->associated = status == 0; + dev_info(&wifi->control->pdev->dev, "Wi-Fi association event: status=%u\n", status); + if (eapol_after_assoc && wifi->associated && wifi->connecting && + !wifi->connect_event) { + int probe_ret = transmit_eapol_start(wifi); + + eapol_after_assoc = false; + dev_info(&wifi->control->pdev->dev, + "association-event EAPOL-Start: rc=%d\n", probe_ret); + } + } + /* ACI event 0x50 carries fixed, non-secret supplicant + * diagnostics: result, phase and handshake timestamps. + * Do not dump the event or any EAPOL/key material. + */ + if (command == 0x80000050 && size >= 64) + dev_info(&wifi->control->pdev->dev, + "Wi-Fi supplicant: result=%u phase=%u times=%u/%u/%u/%u/%u\n", + get_unaligned_le32(packet + 0x18), packet[0x1c], + get_unaligned_le32(packet + 0x24), + get_unaligned_le32(packet + 0x28), + get_unaligned_le32(packet + 0x2c), + get_unaligned_le32(packet + 0x30), + get_unaligned_le32(packet + 0x34)); + if (command == 0x80000003 || command == 0x80000004) + wifi->disconnect_event = true; + if (wifi->connecting && command != 0x80000000 && + command != 0x80000001 && command != 0x80000030 && + command != 0x80000005 && command != 0x80000050) + dev_info_ratelimited(&wifi->control->pdev->dev, + "Wi-Fi event: id=%#x status=%u size=%u\n", + command, status, size); + } else { + wifi->reply_valid = true; + wifi->reply_sequence = sequence; + wifi->reply_status = status; + wifi->reply_command = command; + if (command == 10) { + wifi->country_valid = !status && size >= 27; + if (wifi->country_valid) + memcpy(wifi->country, packet + 24, 3); + } + if (command == 56) { + wifi->tx_profile_valid = !status && size >= 34; + if (wifi->tx_profile_valid) + memcpy(wifi->tx_profile, packet + 24, sizeof(wifi->tx_profile)); + } + if (command == 20 && !status) { + /* getInterfaceStatus skips its 32-byte fixed response. */ + unsigned int pos = 32; + + wifi->link_status_size = size; + while (pos + 4 <= size) { + u16 tag = get_unaligned_le16(packet + pos); + u16 length = get_unaligned_le16(packet + pos + 2); + u8 *value = packet + pos + 4; + unsigned int i; + + if (length > size - pos - 4) + break; + if (wifi->link_tlv_count < ARRAY_SIZE(wifi->link_tags)) { + wifi->link_tags[wifi->link_tlv_count] = tag; + wifi->link_lengths[wifi->link_tlv_count++] = length; + } + /* ACIDebugAdapter::dumpMloStatus, TLVs 377/378. + * Keep only link count/channel/role. Do not retain MACs. + */ + if (tag == 377 && length >= 64 && value[0] <= 3) { + wifi->link_count = value[0]; + wifi->link_status_valid = true; + for (i = 0; i < value[0]; i++) { + u16 chanspec = get_unaligned_le16(value + 26 + 17 * i); + + wifi->link_prime[i] = value[25 + 17 * i]; + wifi->link_channel[i] = chanspec & 0xff; + wifi->link_band[i] = (chanspec >> 14) & 3; + } + } + if (tag == 378 && length >= 9) { + wifi->emlsr_valid = true; + wifi->emlsr_mode = value[0]; + } + pos += 4 + length; + } + } + if (command == 0xb0001) { + wifi->stats_reply_size = size; + wifi->stats_tag = size >= 28 ? get_unaligned_le16(packet + 24) : 0; + wifi->stats_length = size >= 28 ? get_unaligned_le16(packet + 26) : 0; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf928 && + (wifi->stats_length == 752 || wifi->stats_length == 888) && + size >= 28 + wifi->stats_length) { + /* Native firmware returns the older 752-byte counter record. */ + memset(wifi->station_stats, 0, sizeof(wifi->station_stats)); + memcpy(wifi->station_stats, packet + 28, wifi->stats_length); + wifi->station_stats_valid = true; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf921 && + wifi->stats_length == sizeof(wifi->rx_crypto_stats) && + size >= 28 + sizeof(wifi->rx_crypto_stats)) { + memcpy(wifi->rx_crypto_stats, packet + 28, sizeof(wifi->rx_crypto_stats)); + wifi->rx_crypto_stats_valid = true; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf91b && + wifi->stats_length == sizeof(wifi->dot11_mib_stats) && + size >= 28 + sizeof(wifi->dot11_mib_stats)) { + memcpy(wifi->dot11_mib_stats, packet + 28, sizeof(wifi->dot11_mib_stats)); + wifi->dot11_mib_stats_valid = true; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf920 && + wifi->stats_length == sizeof(wifi->tx_power_report) && + size >= 28 + sizeof(wifi->tx_power_report)) { + memcpy(wifi->tx_power_report, packet + 28, sizeof(wifi->tx_power_report)); + wifi->tx_power_report_valid = true; + } + if (command == 0xf0025) { + wifi->pmk_seen = !status && size >= 60; + wifi->pmk_matches = wifi->pmk_seen && + get_unaligned_le16(packet + 26) == 32 && + !memcmp(packet + 28, wifi->expected_pmk, 32); + memzero_explicit(packet + 24, size - 24); + } + } + refill: + post_receive(wifi, ring, wifi->rx_head[ring]); + wifi->rx_head[ring] = (wifi->rx_head[ring] + 1) % ring_size[ring]; + dma_wmb(); + index_write(wifi, 3, ring, wifi->rx_head[ring]); + posted = true; + } + } + completions(wifi); + if (posted) + writel(1, wifi->a.bar + 0x1004); + drain_data_rx(wifi); + drain_data_tx(wifi); +} + +static int command(struct centauri_wifi *wifi, u32 id, const void *payload, size_t size) +{ + u16 slot = wifi->command_head, sequence = ++wifi->sequence; + u8 *packet, *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(1500); + + if (size + 24 > BUFFER_SIZE) + return -EMSGSIZE; + if ((slot + 1) % ring_size[1] == index_read(wifi, 1, 1)) + return -ENOSPC; + packet = wifi->a.window + TX_BUFFER + slot * BUFFER_SIZE; + memset(packet, 0, 24 + size); + put_unaligned_le16(24 + size, packet + 0x10); + put_unaligned_le16(sequence, packet + 0x12); + put_unaligned_le32(id, packet + 0x14); + if (size) + memcpy(packet + 24, payload, size); + descriptor = wifi->a.window + TR_BASE + slot * 16; + put_unaligned_le32(1 | ((24 + size) << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + TX_BUFFER + slot * BUFFER_SIZE, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + dma_wmb(); + wifi->command_head = (slot + 1) % ring_size[1]; + index_write(wifi, 3, 1, wifi->command_head); + wifi->reply_valid = false; + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + do { + drain(wifi); + if (readl(wifi->a.bar + 8) != 2) + return -EIO; + /* Bit 30 is more-data-pending, not part of the command identity. + * Scan/connect replies set it before their later completion event. + */ + if (wifi->reply_valid && wifi->reply_sequence == sequence && + (wifi->reply_command & ~BIT(30)) == id) { + if (wifi->reply_status) + dev_info(&wifi->control->pdev->dev, + "Wi-Fi command %#x rejected: firmware_status=%u\n", + id, wifi->reply_status); + return wifi->reply_status ? -EREMOTEIO : 0; + } + usleep_range(500, 1000); + } while (time_before(jiffies, deadline)); + dev_err(&wifi->control->pdev->dev, "Wi-Fi command %#x timed out (rx=%u)\n", id, wifi->rx_seen[2]); + return -ETIMEDOUT; +} + +static int radio_start(struct centauri_wifi *wifi) +{ + u8 start[5 + 4 + 3 * ETH_ALEN] = { 0, 0x0b, 0, 0, 0 }; + size_t start_size = 5; + unsigned int ring, slot; + int ret; + + /* The stock 26.x startInterface supplies TLV 385, length 18; the macOS 27.0 + * adapter uses 386 (board table). Firmware 0x5736c0 otherwise generates three unrelated + * random addresses. This diagnostic keeps the sole active link equal + * to the netdev identity; it does not enable multi-link association. + */ + if (station_link_addresses) { + const struct centauri_board *board = centauri_board(); + + put_unaligned_le16(board ? board->link_tlv : 385, start + 5); + put_unaligned_le16(3 * ETH_ALEN, start + 7); + for (slot = 0; slot < 3; slot++) + ether_addr_copy(start + 9 + slot * ETH_ALEN, wifi->netdev->dev_addr); + start_size = sizeof(start); + } + for (ring = 1; ring <= 4; ring++) { + ret = open_pair(wifi, ring); + if (ret) + return ret; + } + for (ring = 2; ring <= 4; ring++) { + for (slot = 0; slot < ring_size[ring] - 1; slot++) + post_receive(wifi, ring, slot); + wifi->rx_head[ring] = ring_size[ring] - 1; + dma_wmb(); + index_write(wifi, 3, ring, ring_size[ring] - 1); + } + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + msleep(400); + ret = command(wifi, 0, NULL, 0); + if (!ret) + ret = command(wifi, 1, wifi->netdev->dev_addr, ETH_ALEN); + if (!ret) + ret = command(wifi, 2, start, start_size); + if (ret) + return ret; + msleep(1000); + wifi->configured = true; + dev_info(&wifi->control->pdev->dev, "Wi-Fi command rings ready; interface started\n"); + return 0; +} + +static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, u16 length); + +/* Dwell times in ms for the scan request. The macOS 27 scan adapter copies + * the first request dwell field into frame +0x20 and +0x22 and the second into +0x24 and logs + * "dwelltime active[%d, %d] passive %d". A zero passive dwell makes the J873 scan state machine + * assert in SCHED_Req_Add (the dwell it selects must not be zero) and the whole firmware traps. + */ +#define SCAN_ACTIVE_DWELL_MS 110 +#define SCAN_PASSIVE_DWELL_MS 110 + +/* Fixed scan request (command 0x1000f), 14 bytes, chunk number first. */ +static size_t build_scan(struct centauri_wifi *wifi, unsigned int chunk, u8 *buf) +{ + u8 fixed[14] = { 1, 0, 0, 0x80, 0, 2, 0, 0x10 }; + const struct centauri_board *board = centauri_board(); + u8 list[1 + 4 * 13], oce[5] = { 0, 0, 1, 0, 0 }; + unsigned int first = chunk * 13, n, i; + size_t size = sizeof(fixed); + + put_unaligned_le16(SCAN_ACTIVE_DWELL_MS, fixed + 8); + put_unaligned_le16(SCAN_ACTIVE_DWELL_MS, fixed + 10); + put_unaligned_le16(SCAN_PASSIVE_DWELL_MS, fixed + 12); + memcpy(buf, fixed, sizeof(fixed)); + buf[0] = chunk + 1; + if (!board || !board->scan_tlvs) + return size; + /* The macOS 27.0 scan request appends TLV 0x104, a count and then + * {channel, band (0 = 2.4 GHz, 1 = 5 GHz), 0, 0} per channel, and TLV 0x165, an OCE block + * whose flags byte always has bit 0 set. One chunk covers 13 entries of the wiphy table. + */ + n = min_t(unsigned int, 13, 13 + ARRAY_SIZE(five_ghz) - first); + list[0] = n; + for (i = 0; i < n; i++) { + const struct ieee80211_channel *channel = &wifi->channels[first + i]; + + list[1 + 4 * i] = channel->hw_value; + list[2 + 4 * i] = channel->band == NL80211_BAND_5GHZ; + list[3 + 4 * i] = 0; + list[4 + 4 * i] = 0; + } + append_tlv(buf, &size, 0x104, list, 1 + 4 * n); + append_tlv(buf, &size, 0x165, oce, sizeof(oce)); + return size; +} + +static void scan_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = container_of(work, struct centauri_wifi, scan_work); + struct cfg80211_scan_info info = {}; + struct cfg80211_scan_request *request; + u8 scan[14 + 4 + 1 + 4 * 13 + 4 + 5]; + size_t scan_size; + unsigned int chunk; + int ret = 0; + + mutex_lock(&wifi->lock); + for (chunk = 0; chunk < 3 && !ret; chunk++) { + u32 channels = wifi->channels_seen, done = wifi->scans_done; + bool stopped = false; + unsigned long deadline; + + scan_size = build_scan(wifi, chunk, scan); + ret = command(wifi, 65551, scan, scan_size); + if (ret) + break; + deadline = jiffies + msecs_to_jiffies(8000); + do { + drain(wifi); + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + break; + } + if (!stopped && wifi->channels_seen - channels >= 8) { + stopped = true; + ret = command(wifi, 65537, NULL, 0); + if (ret) + break; + } + if (stopped && wifi->scans_done > done) + break; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + if (!ret && wifi->scans_done == done) + ret = -ETIMEDOUT; + } + info.aborted = !!ret; + request = wifi->scan_request; + wifi->scan_request = NULL; + mutex_unlock(&wifi->lock); + dev_info(&wifi->control->pdev->dev, "Wi-Fi scan finished: rc=%d channels=%u beacons=%u\n", + ret, wifi->channels_seen, wifi->beacons_seen); + if (request) + cfg80211_scan_done(request, &info); +} + +static int wifi_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (!wifi->configured) + ret = -ENETDOWN; + else if (wifi->scan_request || wifi->connecting) + ret = -EBUSY; + else { + wifi->scan_request = request; + schedule_work(&wifi->scan_work); + } + mutex_unlock(&wifi->lock); + return ret; +} + +static int rsn_caps_offset(const u8 *body, size_t size) +{ + size_t offset = 6; + u16 count; + unsigned int i; + + for (i = 0; i < 2; i++) { + if (offset + 2 > size) + return -EINVAL; + count = get_unaligned_le16(body + offset); + offset += 2 + 4 * count; + if (offset > size) + return -EINVAL; + } + return offset + 2 <= size ? offset : -EINVAL; +} + +static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, u16 length) +{ + put_unaligned_le16(tag, payload + *size); + put_unaligned_le16(length, payload + *size + 2); + memcpy(payload + *size + 4, data, length); + *size += 4 + length; +} + +static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, + struct cfg80211_connect_params *params) +{ + /* The stock 26.x join emits a 12-byte TLV 376. The macOS 27.0.1 + * adapter writes tag 0x178 with length 14: the MLD configuration bytes + * that its default initialisation fills with the defaults below, bytes + * 12 and 13 being 0 and the flags byte (bits from the connection parameters, 0 by default). + * Preserve the unspecified-link masks; all-zero is not the non-MLO default. + */ + static const u8 mld_defaults[14] = { + 1, 0, 1, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 3, 0, 0, + }; + /* Firmware connection-context defaults (aciw 0x511ee8). TLV 0xf913 + * replaces all five values. Sending zeros arms Supp_AddTimer with a + * zero duration and aborts WPA before message 2 can be transmitted. + */ + static const __le32 connection_timeouts[] = { + cpu_to_le32(260), cpu_to_le32(6000), cpu_to_le32(2000), + cpu_to_le32(15000), cpu_to_le32(300), + }; + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + struct cfg80211_bss *bss; + const struct cfg80211_bss_ies *ies; + const u8 *rsn, *ap_rsn; + u8 rsn_body[255], ssid[35], zero[20] = {}, pmf; + u8 *payload = wifi->connect_payload; + size_t size = 130, rsn_size; + u16 caps = 0; + int ret = 0, offset; + + if (!params->ssid_len || params->ssid_len > 32 || !params->crypto.psk || + params->crypto.n_akm_suites != 1 || params->crypto.akm_suites[0] != WLAN_AKM_SUITE_PSK || + params->crypto.n_ciphers_pairwise != 1 || params->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || + params->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || params->ie_len > sizeof(wifi->request_ie)) + return -EOPNOTSUPP; + rsn = cfg80211_find_ie(WLAN_EID_RSN, params->ie, params->ie_len); + if (!rsn || rsn_caps_offset(rsn + 2, rsn[1]) < 0) + return -EINVAL; + mutex_lock(&wifi->lock); + if (!wifi->configured || wifi->connecting || wifi->connected || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + bss = cfg80211_get_bss(wiphy, params->channel, params->bssid, + params->ssid, params->ssid_len, IEEE80211_BSS_TYPE_ESS, IEEE80211_PRIVACY_ON); + if (!bss) { + ret = -ENOENT; + goto out; + } + /* Match the measured successful J813 association shape. The firmware + * expects the AP RSN capability bits even with the legal single-AKM STA IE. + */ + rcu_read_lock(); + ies = rcu_dereference(bss->ies); + ap_rsn = ies ? cfg80211_find_ie(WLAN_EID_RSN, ies->data, ies->len) : NULL; + if (ap_rsn) { + offset = rsn_caps_offset(ap_rsn + 2, ap_rsn[1]); + if (offset >= 0) + caps = get_unaligned_le16(ap_rsn + 2 + offset); + } + rcu_read_unlock(); + rsn_size = rsn[1]; + memcpy(rsn_body, rsn + 2, rsn_size); + offset = rsn_caps_offset(rsn_body, rsn_size); + put_unaligned_le16(caps, rsn_body + offset); + pmf = !!(caps & BIT(7)); + memset(payload, 0, sizeof(wifi->connect_payload)); + payload[0] = 1; + memcpy(payload + 1, bss->bssid, ETH_ALEN); + memcpy(wifi->bssid, bss->bssid, ETH_ALEN); + payload[7] = ieee80211_frequency_to_channel(bss->channel->center_freq); + /* AppleChanSpecToACI maps chanspec 0xc000 (5 GHz) to ACI 1; + * 0x4000 (6 GHz) maps to ACI 2. Earlier RE notes swapped these. + */ + payload[8] = bss->channel->band == NL80211_BAND_2GHZ ? 0 : 1; + payload[0x47] = 4; + /* ACIConnectionAdapter::mapRSNCipherSuiteToACIWsec maps CCMP to 4. + * The RSN IE does not replace this fixed cipher field. + */ + put_unaligned_le16(4, payload + 0x48); + put_unaligned_le16(32, payload + 0x4a); + memcpy(payload + 0x4c, params->crypto.psk, 32); + if (verify_pmk) + memcpy(wifi->expected_pmk, params->crypto.psk, 32); + append_tlv(payload, &size, 304, zero, 1); + append_tlv(payload, &size, 63763, connection_timeouts, sizeof(connection_timeouts)); + ssid[0] = 1; + ssid[1] = 4; + ssid[2] = params->ssid_len; + memcpy(ssid + 3, params->ssid, params->ssid_len); + append_tlv(payload, &size, 342, ssid, 3 + params->ssid_len); + append_tlv(payload, &size, WLAN_EID_RSN, rsn_body, rsn_size); + /* The fixed field carries the PMK. The firmware accepts a join with the PMK alone (measured); the macOS TLV 313 carries the + * passphrase and is not sent. + */ + append_tlv(payload, &size, 316, &pmf, 1); + append_tlv(payload, &size, 376, mld_defaults, sizeof(mld_defaults)); + wifi->connect_size = size; + memcpy(wifi->request_ie, params->ie, params->ie_len); + wifi->request_ie_size = params->ie_len; + wifi->connecting = true; + wifi->connect_status = 0; + wifi->associated = wifi->pmk_seen = wifi->pmk_matches = false; + wifi->connect_event = wifi->disconnect_event = false; + cfg80211_put_bss(wiphy, bss); + schedule_work(&wifi->connect_work); +out: + mutex_unlock(&wifi->lock); + return ret; +} + +/* Without the host power state the firmware treats the host as asleep, powers the PHYs down and traps about 15 s after the + * join. Same three commands as the host_wakeup and tx_config debugfs files; called with wifi->lock held. + */ +static void host_setup_commands(struct centauri_wifi *wifi) +{ + static const u8 awake[3] = {}; + static const u8 control_profile[] = { 0x54, 1, 10, 0, 0, 8, 0, 0, 0xff, 0xff, 0xff, 0xff, 1, 1 }; + u8 lifetime[20] = {}; + int ret, i; + + if (!host_setup) + return; + if (!wifi->host_awake_sent) { + ret = command(wifi, 6, awake, sizeof(awake)); + dev_info(&wifi->control->pdev->dev, "host power state awake: rc=%d status=%u\n", ret, wifi->reply_status); + wifi->host_awake_sent = !ret; + } + if (!(wifi->tx_defaults_applied & BIT(0))) { + for (i = 0; i < 4; i++) + put_unaligned_le32(2000000, lifetime + 4 + 4 * i); + ret = command(wifi, 43, lifetime, sizeof(lifetime)); + if (!ret) + wifi->tx_defaults_applied |= BIT(0); + dev_info(&wifi->control->pdev->dev, "TX packet lifetime defaults: rc=%d\n", ret); + } + if (!(wifi->tx_defaults_applied & BIT(1))) { + ret = command(wifi, 52, control_profile, sizeof(control_profile)); + if (!ret) + wifi->tx_defaults_applied |= BIT(1); + dev_info(&wifi->control->pdev->dev, "TX control profile defaults: rc=%d\n", ret); + } +} + +static void connect_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = container_of(work, struct centauri_wifi, connect_work); + unsigned long deadline; + u16 slot, status, command_status; + bool checked_pmk = false, probed_eapol = false, queried_link = false; + int ret; + + mutex_lock(&wifi->lock); + if (auto_rings) { + static const unsigned int ring_ids[] = { 76, 24, 15, 12 }; + unsigned int n; + + for (n = 0; n < ARRAY_SIZE(ring_ids); n++) { + int ring_ret = data_probe_locked(wifi, ring_ids[n], true); + + if (ring_ret && ring_ret != -EBUSY) + dev_warn(&wifi->control->pdev->dev, "data ring %u setup failed: %d\n", ring_ids[n], ring_ret); + } + } + host_setup_commands(wifi); + slot = wifi->command_head; + ret = command(wifi, 131072, wifi->connect_payload, wifi->connect_size); + command_status = wifi->reply_status; + memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); + deadline = jiffies + msecs_to_jiffies(15000); + while (!ret && !wifi->connect_event && time_before(jiffies, deadline)) { + drain(wifi); + if (link_diagnostics && wifi->associated && !queried_link) { + unsigned int i; + int link_ret; + + queried_link = true; + link_ret = query_link_status(wifi); + dev_info(&wifi->control->pdev->dev, + "connection link status: rc=%d complete=%d valid=%d count=%u emlsr=%d/%u\n", + link_ret, wifi->connect_event, wifi->link_status_valid, wifi->link_count, + wifi->emlsr_valid, wifi->emlsr_mode); + for (i = 0; i < wifi->link_tlv_count; i++) + dev_info(&wifi->control->pdev->dev, "connection link TLV: tag=%u length=%u\n", + wifi->link_tags[i], wifi->link_lengths[i]); + if (wifi->link_status_valid) + for (i = 0; i < wifi->link_count; i++) + dev_info(&wifi->control->pdev->dev, + "connection link %u: prime=%u channel=%u band=%u\n", + i, wifi->link_prime[i], wifi->link_channel[i], wifi->link_band[i]); + } + if (eapol_after_assoc && wifi->associated && !probed_eapol) { + unsigned int i; + int probe_ret; + + probed_eapol = true; + probe_ret = query_station_stats(wifi); + dev_info(&wifi->control->pdev->dev, + "connection counters: complete=%d rc=%d valid=%d length=%u\n", + wifi->connect_event, probe_ret, wifi->station_stats_valid, wifi->stats_length); + if (!probe_ret && wifi->station_stats_valid) + for (i = 0; i < wifi->stats_length; i += 4) { + u32 value = get_unaligned_le32(wifi->station_stats + i); + + if (value) + dev_info(&wifi->control->pdev->dev, + "station counter %03x=%u\n", i, value); + } + } + if (verify_pmk && wifi->associated && !checked_pmk) { + int key_ret = command(wifi, 0xf0025, wifi->bssid, ETH_ALEN); + + checked_pmk = true; + dev_info(&wifi->control->pdev->dev, + "Wi-Fi firmware key check: rc=%d available=%d matches=%d\n", + key_ret, wifi->pmk_seen, wifi->pmk_matches); + } + if (readl(wifi->a.bar + 8) != 2) + ret = -EIO; + usleep_range(1000, 2000); + } + if (!ret && !wifi->connect_event) + ret = -ETIMEDOUT; + status = !ret && wifi->connect_status == 0 ? WLAN_STATUS_SUCCESS : WLAN_STATUS_UNSPECIFIED_FAILURE; + wifi->connecting = false; + wifi->connected = status == WLAN_STATUS_SUCCESS; + if (wifi->connected && !roam_scan) { + int roam_ret = command(wifi, 0x20016, NULL, 0); + + dev_info(&wifi->control->pdev->dev, "roam scan off: rc=%d status=%u\n", roam_ret, wifi->reply_status); + } + memzero_explicit(wifi->expected_pmk, sizeof(wifi->expected_pmk)); + /* The command and connection completion have consumed the PMK buffer. */ + if (slot < ring_size[1]) + memzero_explicit(wifi->a.window + TX_BUFFER + slot * BUFFER_SIZE, BUFFER_SIZE); + mutex_unlock(&wifi->lock); + dev_info(&wifi->control->pdev->dev, + "Wi-Fi authentication result: rc=%d command_status=%u event_status=%u complete_event=%d\n", + ret, command_status, wifi->connect_status, wifi->connect_event); + cfg80211_connect_result(wifi->netdev, wifi->bssid, + wifi->request_ie, wifi->request_ie_size, NULL, 0, status, GFP_KERNEL); + if (status == WLAN_STATUS_SUCCESS && auto_rings && !READ_ONCE(wifi->stopping)) { + /* The normal TX ring is opened after the connection result, as in the manual recipe (data_probe 25 after wpa completed); + * doing it after cfg80211_connect_result keeps the join inside the supplicant's timeout. + */ + int ring_ret; + + mutex_lock(&wifi->lock); + ring_ret = data_probe_locked(wifi, 25, true); + mutex_unlock(&wifi->lock); + if (ring_ret && ring_ret != -EBUSY) + dev_warn(&wifi->control->pdev->dev, "data ring 25 setup failed: %d\n", ring_ret); + } + if (status == WLAN_STATUS_SUCCESS && !READ_ONCE(wifi->stopping)) { + netif_carrier_on(wifi->netdev); + if (wifi->data_probe_done & BIT(5)) + netif_wake_queue(wifi->netdev); + schedule_delayed_work(&wifi->link_work, msecs_to_jiffies(20)); + } +} + +static void link_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = container_of(to_delayed_work(work), struct centauri_wifi, link_work); + bool disconnected; + + mutex_lock(&wifi->lock); + drain(wifi); + disconnected = wifi->disconnect_event || READ_ONCE(wifi->stopping) || + readl(wifi->a.bar + 8) != 2; + if (disconnected) + wifi->connected = false; + mutex_unlock(&wifi->lock); + if (disconnected) { + netif_stop_queue(wifi->netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(wifi->netdev); + cfg80211_disconnected(wifi->netdev, WLAN_REASON_UNSPECIFIED, NULL, 0, false, GFP_KERNEL); + } else { + schedule_delayed_work(&wifi->link_work, msecs_to_jiffies(20)); + } +} + +/* MSI from Control or Alpha: service the data rings now instead of at the next poll. */ +void centauri_wifi_kick(struct centauri *c) +{ + struct centauri_wifi *wifi = READ_ONCE(c->wifi); + + if (wifi && READ_ONCE(wifi->connected) && !READ_ONCE(wifi->stopping)) + mod_delayed_work(system_wq, &wifi->link_work, 0); +} + +static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 payload[16] = {}; + bool connected; + int ret; + + mutex_lock(&wifi->lock); + if (wifi->connecting) { + mutex_unlock(&wifi->lock); + return -EBUSY; + } + put_unaligned_le16(reason, payload); + put_unaligned_le16(328, payload + 8); + put_unaligned_le16(4, payload + 10); + /* Preserve the firmware's default disconnect budget. */ + put_unaligned_le32(500, payload + 12); + ret = command(wifi, 131073, payload, sizeof(payload)); + connected = wifi->connected; + wifi->connected = false; + mutex_unlock(&wifi->lock); + cancel_delayed_work_sync(&wifi->link_work); + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(netdev); + if (connected) + cfg80211_disconnected(netdev, reason, NULL, 0, true, GFP_KERNEL); + return ret == -EREMOTEIO && wifi->reply_status == 326 ? 0 : ret; +} + +static int wifi_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, + bool enabled, int timeout) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 mode = enabled ? 0 : 3; + int ret; + + /* ACIIO80211PowerAdapter::setPowerSaveMode, command 7. */ + mutex_lock(&wifi->lock); + ret = wifi->configured ? command(wifi, 7, &mode, sizeof(mode)) : -ENETDOWN; + mutex_unlock(&wifi->lock); + return ret; +} + +static void wifi_regulatory(struct wiphy *wiphy, struct regulatory_request *request) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 country[3] = { request->alpha2[0], request->alpha2[1], 1 }; + int ret; + + /* ACIRegulatoryAdapter::setCountryCode, command 9. Leave world-domain + * handling to cfg80211; do not invent a device country or bypass SAR. + */ + if (country[0] < 'A' || country[0] > 'Z' || country[1] < 'A' || country[1] > 'Z') + return; + mutex_lock(&wifi->lock); + if (wifi->configured) { + ret = command(wifi, 9, country, sizeof(country)); + dev_info(&wifi->control->pdev->dev, + "Wi-Fi country %c%c: rc=%d\n", country[0], country[1], ret); + } + mutex_unlock(&wifi->lock); +} + +/* + * The joined BSS as cfg80211 knows it from the last scan. No firmware query + * for live link statistics is known yet, so its signal is the scan's. + */ +static struct cfg80211_bss *wifi_joined_bss(struct centauri_wifi *wifi) +{ + if (!wifi->connected) + return NULL; + return cfg80211_get_bss(wifi->wiphy, NULL, wifi->bssid, NULL, 0, + IEEE80211_BSS_TYPE_ESS, IEEE80211_PRIVACY_ANY); +} + +static int wifi_get_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_info *sinfo) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + struct net_device_stats *stats = &wifi->netdev->stats; + struct cfg80211_bss *bss; + int ret = 0; + + mutex_lock(&wifi->lock); + if (!wifi->connected || !ether_addr_equal(mac, wifi->bssid)) { + ret = -ENOENT; + goto out; + } + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS) | + BIT_ULL(NL80211_STA_INFO_TX_PACKETS) | + BIT_ULL(NL80211_STA_INFO_RX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_TX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_TX_FAILED) | + BIT_ULL(NL80211_STA_INFO_RX_DROP_MISC); + sinfo->rx_packets = stats->rx_packets; + sinfo->tx_packets = stats->tx_packets; + sinfo->rx_bytes = stats->rx_bytes; + sinfo->tx_bytes = stats->tx_bytes; + sinfo->tx_failed = stats->tx_errors; + sinfo->rx_dropped_misc = stats->rx_dropped; + + bss = wifi_joined_bss(wifi); + if (bss) { + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); + sinfo->signal = bss->signal / 100; /* mBm */ + cfg80211_put_bss(wiphy, bss); + } +out: + mutex_unlock(&wifi->lock); + return ret; +} + +static int wifi_get_channel(struct wiphy *wiphy, struct wireless_dev *wdev, + unsigned int link_id, struct cfg80211_chan_def *chandef) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + struct cfg80211_bss *bss; + int ret = -ENODATA; + + mutex_lock(&wifi->lock); + bss = wifi_joined_bss(wifi); + if (bss) { + /* The join asks the firmware for a 20 MHz channel. */ + cfg80211_chandef_create(chandef, bss->channel, NL80211_CHAN_NO_HT); + cfg80211_put_bss(wiphy, bss); + ret = 0; + } + mutex_unlock(&wifi->lock); + return ret; +} + +static const struct cfg80211_ops wifi_ops = { + .scan = wifi_scan, + .connect = wifi_connect, + .disconnect = wifi_disconnect, + .set_power_mgmt = wifi_power_mgmt, + .get_station = wifi_get_station, + .get_channel = wifi_get_channel, +}; + +static int net_open(struct net_device *netdev) +{ + netif_carrier_off(netdev); + netif_stop_queue(netdev); + return 0; +} + +static int net_stop(struct net_device *netdev) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)netdev_priv(netdev); + + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + return 0; +} + +static netdev_tx_t net_transmit(struct sk_buff *skb, struct net_device *netdev) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)netdev_priv(netdev); + + if (READ_ONCE(wifi->stopping) || !READ_ONCE(wifi->connected) || + skb->len < ETH_HLEN || skb->len > BUFFER_SIZE) { + netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) { + netif_stop_queue(netdev); + return NETDEV_TX_BUSY; + } + skb_queue_tail(&wifi->data_tx_queue, skb); + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) + netif_stop_queue(netdev); + mod_delayed_work(system_wq, &wifi->link_work, 0); + return NETDEV_TX_OK; +} + +static const struct net_device_ops net_ops = { + .ndo_open = net_open, + .ndo_stop = net_stop, + .ndo_start_xmit = net_transmit, + .ndo_validate_addr = eth_validate_addr, +}; + +/* Airship direct_memregion::map (0x1802864e0) sends opcode 8, a region ID, + * byte size and a 64-bit DVA in a 24-byte header. Alpha's region-debug is ID 1. + * Firmware 0x175340 passes those fields to DramPhyService_handle_buffer; + * InitDramLoggerMgr writes a 0x158-byte control header to the supplied buffer. + * This is distinct from the fw_debug transfer ring. Never print log payloads: + * firmware diagnostics may contain private connection material. + */ +static ssize_t fwlog_start_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + u8 message[24] = { 8, 0, 1, 0 }; + unsigned int action; + unsigned long deadline; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &action); + if (ret || action != 1) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + mutex_lock(&wifi->fwlog_lock); + if (!wifi->configured || wifi->connecting || wifi->connected || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + if (wifi->fwlog_attempted) { + ret = -EALREADY; + goto out; + } + ret = centauri_alpha_log_buffer(wifi->control, &wifi->fwlog_buffer, + &wifi->fwlog_dma, &wifi->fwlog_size); + if (ret) + goto out; + put_unaligned_le32(wifi->fwlog_size, message + 4); + put_unaligned_le64(wifi->fwlog_dma, message + 8); + wifi->fwlog_attempted = true; + ret = alpha_message(wifi, message, sizeof(message)); + if (ret) + goto report; + deadline = jiffies + msecs_to_jiffies(2000); + do { + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + goto report; + } + dma_rmb(); + if (get_unaligned_le16(wifi->fwlog_buffer) == 0x201) { + wifi->fwlog_ready = true; + goto report; + } + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + ret = -ETIMEDOUT; +report: + dev_info(&wifi->control->pdev->dev, + "Wi-Fi DRAM logger: rc=%d initialized=%u stage=%u\n", + ret, wifi->fwlog_ready, readl(wifi->a.bar + 8)); +out: + mutex_unlock(&wifi->fwlog_lock); + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations fwlog_start_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = fwlog_start_write, +}; + +static ssize_t fwlog_status_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + char text[320]; + u8 header[0xb4] = { 0 }; + int length; + + mutex_lock(&wifi->fwlog_lock); + if (wifi->fwlog_buffer) { + dma_rmb(); + memcpy(header, wifi->fwlog_buffer, sizeof(header)); + } + length = scnprintf(text, sizeof(text), + "attempted=%u initialized=%u bytes=%zu version=%#x blocks=%u block_size=%u mask=%#x data_offset=%u consumer=%u producer=%u header14=%u header18=%u header1c=%u connecting=%u scanning=%u\n", + wifi->fwlog_attempted, wifi->fwlog_ready, wifi->fwlog_size, + get_unaligned_le16(header), get_unaligned_le16(header + 2), + get_unaligned_le32(header + 4), get_unaligned_le32(header + 8), + get_unaligned_le32(header + 12), get_unaligned_le32(header + 16), + get_unaligned_le32(header + 0xb0), + get_unaligned_le32(header + 20), get_unaligned_le32(header + 24), + get_unaligned_le32(header + 28), + !!READ_ONCE(wifi->connecting), !!READ_ONCE(wifi->scan_request)); + mutex_unlock(&wifi->fwlog_lock); + return simple_read_from_buffer(output, count, position, text, length); +} + +static const struct file_operations fwlog_status_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = fwlog_status_read, + .llseek = default_llseek, +}; + +/* The exact firmware's 0x144cfc compares header +0xb0 (producer) against + * +0x10 (host consumer). InitDramLoggerMgr registers a GDMA read of +0x10. + * Only acknowledge records after a private snapshot has been verified. + * Leave three entries reserved for the firmware's three DMA descriptors + * until completion/record sequencing is independently decoded. + */ +static ssize_t fwlog_ack_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + u8 *header; + u32 requested, consumer, producer, pending; + u16 blocks; + int ret; + + ret = kstrtou32_from_user(input, count, 0, &requested); + if (ret) + return ret; + mutex_lock(&wifi->fwlog_lock); + if (!wifi->fwlog_ready || readl(wifi->a.bar + 8) != 2) { + ret = -ENETDOWN; + goto out; + } + header = wifi->fwlog_buffer; + dma_rmb(); + blocks = get_unaligned_le16(header + 2); + consumer = get_unaligned_le32(header + 0x10); + producer = get_unaligned_le32(header + 0xb0); + pending = producer - consumer; + if (get_unaligned_le16(header) != 0x201 || blocks < 8 || + pending > blocks || pending < 3 || requested - consumer > pending - 3) { + ret = -EINVAL; + goto out; + } + WRITE_ONCE(*(__le32 *)(header + 0x10), cpu_to_le32(requested)); + dma_wmb(); + ret = 0; +out: + mutex_unlock(&wifi->fwlog_lock); + return ret ?: count; +} + +static const struct file_operations fwlog_ack_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = fwlog_ack_write, +}; + +/* Root-only binary diagnostics. Save directly to a protected artifact, never + * the console or a text log. The device may update blocks during a read. + */ +static ssize_t fwlog_buffer_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + ssize_t ret; + + mutex_lock(&wifi->fwlog_lock); + if (!wifi->fwlog_ready) { + ret = -ENODATA; + } else { + dma_rmb(); + ret = simple_read_from_buffer(output, count, position, + wifi->fwlog_buffer, wifi->fwlog_size); + } + mutex_unlock(&wifi->fwlog_lock); + return ret; +} + +static const struct file_operations fwlog_buffer_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = fwlog_buffer_read, + .llseek = default_llseek, +}; + +/* The hardware data-ring callback requires the ten-byte WLAN clientdata + * extension after the 44-byte ACIPC header. Without it UMAC asserts at + * 0x11760e. Keep these empty-ring probes explicit until packet DMA is proven. + * The reserved regions below do not overlap control rings or scan buffers. + */ +/* Called with wifi->lock held. From the connect path (internal) the "connecting" flag is expected to be set. */ +static int data_probe_locked(struct centauri_wifi *wifi, unsigned int id, bool internal) +{ + u8 message[0x34 + 10] = { 0 }; + unsigned int vector, offset, size, bit, doorbell, irq; + bool transfer = false; + int ret; + + if (id == 76) { + vector = 192; + offset = 0x200000; + size = 2048; + bit = 0; + doorbell = 18; + irq = 3; + } else if (id == 14) { + vector = 206; + offset = 0x220000; + size = 256; + bit = 1; + doorbell = 11; + irq = 8; + } else if (id == 15) { + vector = 207; + offset = 0x240000; + size = 128; + bit = 2; + doorbell = 11; + irq = 8; + } else if (id == 24 || id == 25 || id == 12) { + transfer = true; + vector = id; + offset = id == 12 ? 0x520000 : id == 24 ? 0x500000 : 0x540000; + size = 128; + bit = id == 12 ? 4 : id == 24 ? 3 : 5; + doorbell = id == 12 ? 11 : 18; + irq = id == 12 ? 8 : 3; + } else { + return -EINVAL; + } + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + return ret; + } + if ((!internal && wifi->connecting) || wifi->scan_request || + (wifi->data_probe_done & BIT(bit))) { + ret = -EBUSY; + return ret; + } + wifi->data_probe_done |= BIT(bit); + memset(wifi->a.window + offset, 0, size * 32); + index_write(wifi, transfer ? 1 : 0, vector, 0); + index_write(wifi, transfer ? 3 : 2, vector, 0); + message[0] = transfer ? 1 : 2; + message[2] = transfer && id == 12 ? 0 : 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le16(vector, message + 6); + put_unaligned_le64(wifi->a.window_dma + offset, message + 8); + if (transfer) { + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(size, message + 0x18); + put_unaligned_le16(id == 12 ? 12 : 76, message + 0x1a); + put_unaligned_le16(doorbell, message + 0x1c); + put_unaligned_le16(id == 12 ? 0x220 : 0, message + 0x1e); + put_unaligned_le32(16, message + 0x20); + put_unaligned_le16(irq, message + 0x24); + } else { + put_unaligned_le16(size, message + 0x10); + put_unaligned_le16(id == 15 ? 12 : 0xffff, message + 0x12); + put_unaligned_le16(doorbell, message + 0x14); + put_unaligned_le32(1, message + 0x18); + put_unaligned_le16(irq, message + 0x1c); + } + /* AirshipDataTxTransferRing::initWithOptions maps the host MCBC + * parameters to version 0, hardware interface 0, TID 6, AC 3, and + * six broadcast destination bytes. The default probe is unchanged. + */ + if (id == 25 && data_tx_broadcast) { + message[0x34 + 2] = 6; + message[0x34 + 3] = 3; + eth_broadcast_addr(message + 0x34 + 4); + } + ret = alpha_message(wifi, message, (transfer ? 0x34 : 0x2c) + 10); + msleep(1000); + dev_info(&wifi->control->pdev->dev, + "data %s probe id=%u index=%u extension=10 rc=%d stage=%u head=%u tail=%u\n", + transfer ? "TR" : "CR", id, vector, ret, readl(wifi->a.bar + 8), + index_read(wifi, transfer ? 3 : 0, vector), + index_read(wifi, transfer ? 1 : 2, vector)); + if (!ret && readl(wifi->a.bar + 8) != 2) + ret = -EIO; + if (!ret && transfer && id == 12) { + unsigned int slot; + + /* Give the receive engine bounded, individually identifiable buffers. + * No recycling or network delivery until the completion ABI is proven. + */ + for (slot = 0; slot < size - 1; slot++) { + u8 *descriptor = wifi->a.window + offset + slot * 16; + u32 buffer = DATA_RX_BUFFER + slot * 4096; + + memset(wifi->a.window + buffer, 0, 4096); + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + } + dma_wmb(); + index_write(wifi, 3, vector, size - 1); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + msleep(100); + dev_info(&wifi->control->pdev->dev, + "data RX buffers posted=%u stage=%u tr_tail=%u cr_head=%u\n", + size - 1, readl(wifi->a.bar + 8), index_read(wifi, 1, 12), + index_read(wifi, 0, 207)); + } + return ret; +} + +static ssize_t data_probe_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + unsigned int id; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &id); + if (ret) + return ret; + mutex_lock(&wifi->lock); + ret = data_probe_locked(wifi, id, false); + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static int transmit_eapol_start(struct centauri_wifi *wifi) +{ + u16 head = index_read(wifi, 3, 24); + u8 *descriptor = wifi->a.window + 0x500000 + head * 32; + u8 *packet = wifi->a.window + 0xf00000 + head * 64; + u64 metadata = 0; + unsigned int attempt; + + if (readl(wifi->a.bar + 8) != 2 || !(wifi->data_probe_done & BIT(3))) + return -ENETDOWN; + /* Eight independent buffers bound the probe without ring reuse. */ + if (head >= 8 || index_read(wifi, 1, 24) != head) + return -EBUSY; + if (eapol_unicast && (!wifi->associated || !is_valid_ether_addr(wifi->bssid))) + return -ENOTCONN; + /* A standards-defined EAPOL-Start, with no credential/key material. + * Descriptor flags 3 mean a DMA buffer plus the 16-byte WLAN footer. + * Interface zero and default TX policy have a zero footer. + */ + memset(packet, 0, 64); + if (eapol_unicast) + memcpy(packet, wifi->bssid, ETH_ALEN); + else + memcpy(packet, "\x01\x80\xc2\x00\x00\x03", ETH_ALEN); + memcpy(packet + ETH_ALEN, wifi->netdev->dev_addr, ETH_ALEN); + put_unaligned_be16(ETH_P_PAE, packet + 12); + packet[14] = 2; + packet[15] = 1; + memset(descriptor, 0, 32); + put_unaligned_le32(3 | (18 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + 0xf00000 + head * 64, descriptor + 4); + put_unaligned_le16(head + 1, descriptor + 12); + if (eapol_control_profile) + metadata |= BIT_ULL(60) | BIT_ULL(48); + if (eapol_relative_lifetime) { + /* AirshipDataPacket::setRelativeExpirationTime uses mode 6 in + * bits 29:27 and microseconds in the second metadata word. + */ + metadata |= 6ULL << 27; + put_unaligned_le64(2000000, descriptor + 24); + } + put_unaligned_le64(metadata, descriptor + 16); + dma_wmb(); + index_write(wifi, 3, 24, head + 1); + dma_wmb(); + writel(1, wifi->a.bar + 0x1030); + for (attempt = 0; attempt < 20; attempt++) { + if (index_read(wifi, 1, 24) == head + 1 || readl(wifi->a.bar + 8) != 2) + break; + usleep_range(5000, 6000); + } + dev_info(&wifi->control->pdev->dev, + "EAPOL-Start DMA probe: slot=%u associated=%d unicast=%d relative_expiry=%d stage=%u tr_tail=%u cr_head=%u\n", + head, wifi->associated, eapol_unicast, eapol_relative_lifetime, readl(wifi->a.bar + 8), + index_read(wifi, 1, 24), index_read(wifi, 0, 192)); + return readl(wifi->a.bar + 8) == 2 ? 0 : -EIO; +} + +static ssize_t eapol_start_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + bool enabled; + int ret; + + ret = kstrtobool_from_user(input, count, &enabled); + if (ret || !enabled) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + if (wifi->connecting || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + ret = transmit_eapol_start(wifi); +out: + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations eapol_start_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = eapol_start_write, +}; + +static const struct file_operations data_probe_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = data_probe_write, +}; + +static ssize_t data_status_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + char status[1536]; + unsigned int i, length; + + mutex_lock(&wifi->lock); + dma_rmb(); + length = scnprintf(status, sizeof(status), + "stage=%u tx_cr=%u tx_tr=%u/%u rx_cr=%u rx_tr=%u/%u received=%u reposted=%u bad=%u samples=%u normal_tx_tr=%u/%u submitted=%u completed=%u failed=%u queued=%u coalesced=%u tx_bad=%u\n", + readl(wifi->a.bar + 8), index_read(wifi, 0, 192), + index_read(wifi, 1, 24), index_read(wifi, 3, 24), + index_read(wifi, 0, 207), index_read(wifi, 1, 12), + index_read(wifi, 3, 12), wifi->data_rx_received, + wifi->data_rx_reposted, wifi->data_rx_bad, + le32_to_cpu(wifi->rx_snapshot.count), + index_read(wifi, 1, 25), index_read(wifi, 3, 25), + wifi->data_tx_submitted, wifi->data_tx_completed, + wifi->data_tx_failed, skb_queue_len(&wifi->data_tx_queue), + wifi->data_tx_coalesced, wifi->data_tx_bad); + for (i = 0; i < 8; i++) { + u8 *entry = wifi->a.window + 0x240000 + i * 32; + + length += scnprintf(status + length, sizeof(status) - length, + "rx_completion[%u]=%08x %08x %08x %08x %08x %08x %08x %08x\n", + i, get_unaligned_le32(entry), get_unaligned_le32(entry + 4), + get_unaligned_le32(entry + 8), get_unaligned_le32(entry + 12), + get_unaligned_le32(entry + 16), get_unaligned_le32(entry + 20), + get_unaligned_le32(entry + 24), get_unaligned_le32(entry + 28)); + } + for (i = 0; i < 8; i++) { + u8 *entry = wifi->a.window + 0x200000 + i * 32; + + length += scnprintf(status + length, sizeof(status) - length, + "tx_completion[%u]=%08x %08x %08x %08x %08x %08x %08x %08x\n", + i, get_unaligned_le32(entry), get_unaligned_le32(entry + 4), + get_unaligned_le32(entry + 8), get_unaligned_le32(entry + 12), + get_unaligned_le32(entry + 16), get_unaligned_le32(entry + 20), + get_unaligned_le32(entry + 24), get_unaligned_le32(entry + 28)); + } + mutex_unlock(&wifi->lock); + return simple_read_from_buffer(output, count, position, status, length); +} + +static const struct file_operations data_status_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = data_status_read, +}; + +/* Contains private network traffic: retrieve only as a protected binary file. + * Samples are copied once into host memory and never change after count 8. + */ +static ssize_t data_rx_snapshot_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + ssize_t ret; + + mutex_lock(&wifi->lock); + ret = simple_read_from_buffer(output, count, position, &wifi->rx_snapshot, + sizeof(wifi->rx_snapshot)); + mutex_unlock(&wifi->lock); + return ret; +} + +static const struct file_operations data_rx_snapshot_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = data_rx_snapshot_read, + .llseek = default_llseek, +}; + +static ssize_t country_status_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + char status[96]; + unsigned int length; + int ret; + + mutex_lock(&wifi->lock); + wifi->country_valid = false; + ret = wifi->configured ? command(wifi, 10, NULL, 0) : -ENETDOWN; + length = scnprintf(status, sizeof(status), "rc=%d valid=%d country_bytes=%02x/%02x/%02x\n", + ret, wifi->country_valid, wifi->country[0], wifi->country[1], wifi->country[2]); + mutex_unlock(&wifi->lock); + return simple_read_from_buffer(output, count, position, status, length); +} + +static const struct file_operations country_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = country_status_read, +}; + +static int query_link_status(struct centauri_wifi *wifi) +{ + wifi->link_status_valid = wifi->emlsr_valid = false; + wifi->link_count = wifi->link_status_size = 0; + wifi->link_tlv_count = 0; + return wifi->configured ? command(wifi, 20, NULL, 0) : -ENETDOWN; +} + +static int link_status_show(struct seq_file *seq, void *unused) +{ + struct centauri_wifi *wifi = seq->private; + unsigned int i; + int ret; + + mutex_lock(&wifi->lock); + ret = query_link_status(wifi); + seq_printf(seq, "rc=%d valid=%d size=%u count=%u emlsr=%d/%u\n", ret, + wifi->link_status_valid, wifi->link_status_size, wifi->link_count, + wifi->emlsr_valid, wifi->emlsr_mode); + for (i = 0; i < wifi->link_tlv_count; i++) + seq_printf(seq, "tag=%u length=%u\n", wifi->link_tags[i], wifi->link_lengths[i]); + if (wifi->link_status_valid) + for (i = 0; i < wifi->link_count; i++) + seq_printf(seq, "link=%u prime=%u channel=%u band=%u\n", i, + wifi->link_prime[i], wifi->link_channel[i], wifi->link_band[i]); + mutex_unlock(&wifi->lock); + return 0; +} + +static int link_status_open(struct inode *inode, struct file *file) +{ + return single_open(file, link_status_show, inode->i_private); +} + +static const struct file_operations link_status_ops = { + .owner = THIS_MODULE, + .open = link_status_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int tx_config_show(struct seq_file *seq, void *unused) +{ + struct centauri_wifi *wifi = seq->private; + u8 id; + int ret; + + mutex_lock(&wifi->lock); + seq_printf(seq, "applied=%u\n", wifi->tx_defaults_applied); + for (id = 0; id <= 1; id++) { + wifi->tx_profile_valid = false; + ret = wifi->configured ? command(wifi, 56, &id, sizeof(id)) : -ENETDOWN; + seq_printf(seq, "profile=%u rc=%d status=%u valid=%d", id, ret, + wifi->reply_status, wifi->tx_profile_valid); + if (wifi->tx_profile_valid) + seq_printf(seq, " rate=%#x rts_threshold=%u rts_retries=%u retries=%u type=%u id=%u", + get_unaligned_le32(wifi->tx_profile), + get_unaligned_le16(wifi->tx_profile + 4), wifi->tx_profile[6], + wifi->tx_profile[7], wifi->tx_profile[8], wifi->tx_profile[9]); + seq_putc(seq, '\n'); + } + mutex_unlock(&wifi->lock); + return 0; +} + +static int tx_config_open(struct inode *inode, struct file *file) +{ + return single_open(file, tx_config_show, inode->i_private); +} + +static ssize_t tx_config_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = ((struct seq_file *)file->private_data)->private; + /* Exact configureTxDataCtrlPacketProfile: command 52, TLV 340/10. + * 0xa0154 is the TLV header, not the command ID. + */ + static const u8 control_profile[] = { + 0x54, 1, 10, 0, 0, 8, 0, 0, 0xff, 0xff, 0xff, 0xff, 1, 1, + }; + u8 lifetime[20] = {}; + unsigned int action, i; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &action); + if (ret || (action != 1 && action != 2)) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + if (!wifi->configured || wifi->connecting || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + if (action == 1) { + /* AppleWLAN80211STA::resetTxPacketLifetime selects 2000 ms for + * each AC. The ACI adapter converts to firmware microseconds. + */ + for (i = 0; i < 4; i++) + put_unaligned_le32(2000000, lifetime + 4 + 4 * i); + ret = command(wifi, 43, lifetime, sizeof(lifetime)); + } else { + ret = command(wifi, 52, control_profile, sizeof(control_profile)); + } + if (!ret) + wifi->tx_defaults_applied |= BIT(action - 1); + dev_info(&wifi->control->pdev->dev, "reference TX setup action=%u rc=%d\n", action, ret); +out: + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations tx_config_ops = { + .owner = THIS_MODULE, + .open = tx_config_open, + .read = seq_read, + .write = tx_config_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static ssize_t host_wakeup_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + /* ACIIO80211PowerAdapter::hostWakeup selects state 0 in command 6. + * This firmware's SysCmdSetHostPowerState accepts the fixed 27-byte + * request without TLVs. Keep its existing event filter: the newer host + * adapter's 92-byte event list exceeds this firmware's 86-byte limit. + */ + const u8 payload[3] = {}; + unsigned int action; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &action); + if (ret || action != 1) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + if (!wifi->configured || wifi->connecting || wifi->scan_request || wifi->connected) { + ret = -EBUSY; + goto out; + } + ret = command(wifi, 6, payload, sizeof(payload)); + dev_info(&wifi->control->pdev->dev, "host wakeup state=0 rc=%d status=%u stage=%u\n", + ret, wifi->reply_status, readl(wifi->a.bar + 8)); +out: + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations host_wakeup_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = host_wakeup_write, +}; + +static int query_station_stats(struct centauri_wifi *wifi) +{ + u8 payload[22] = {}; + int ret; + + /* ACIDebugAdapter::queryStaCounterStats: statistic bit 37, all peers. + * Its fixed counter record contains no key or packet payload bytes. + */ + put_unaligned_le64(BIT_ULL(37), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + wifi->station_stats_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + return ret; +} + +static int station_stats_show(struct seq_file *seq, void *unused) +{ + struct centauri_wifi *wifi = seq->private; + unsigned int i; + int ret; + + mutex_lock(&wifi->lock); + ret = query_station_stats(wifi); + seq_printf(seq, "rc=%d valid=%d size=%u tag=%#x length=%u\n", ret, + wifi->station_stats_valid, wifi->stats_reply_size, wifi->stats_tag, wifi->stats_length); + if (!ret && wifi->station_stats_valid) + for (i = 0; i < wifi->stats_length; i += 4) { + u32 value = get_unaligned_le32(wifi->station_stats + i); + + if (value) + seq_printf(seq, "%03x %u\n", i, value); + } + mutex_unlock(&wifi->lock); + return 0; +} + +static int station_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, station_stats_show, inode->i_private); +} + +static const struct file_operations station_stats_ops = { + .owner = THIS_MODULE, + .open = station_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int rx_crypto_stats_show(struct seq_file *seq, void *unused) +{ + /* ACIStatisticsAdapter::printRxChipCounterStats at 0x1809d7b80: + * statistic bit 30, response TLV 0xf921, sixteen LE32 counters. + * Unlike the station record, this fixed structure contains no MACs. + */ + static const char * const names[] = { + "rx_duplicate_errors", "rx_ampdu_duplicate_errors", + "rx_unicast_tkip_replay_errors", "rx_unicast_tkip_decrypt_errors", + "rx_multicast_tkip_replay_errors", "rx_multicast_tkip_decrypt_errors", + "rx_unicast_ccmp_replay_errors", "rx_unicast_ccmp_decrypt_errors", + "rx_multicast_ccmp_replay_errors", "rx_multicast_ccmp_decrypt_errors", + "phy_total_detect_count", "phy_ofdm_detect_count", "phy_11b_detect_count", + "phy_plcp_error_count", "phy_total_receive_count", "phy_plcp_good_count", + }; + struct centauri_wifi *wifi = seq->private; + u8 payload[22] = {}; + unsigned int i; + int ret; + + put_unaligned_le64(BIT_ULL(30), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + mutex_lock(&wifi->lock); + wifi->rx_crypto_stats_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + seq_printf(seq, "rc=%d valid=%d size=%u tag=%#x length=%u\n", ret, + wifi->rx_crypto_stats_valid, wifi->stats_reply_size, wifi->stats_tag, wifi->stats_length); + if (!ret && wifi->rx_crypto_stats_valid) + for (i = 0; i < ARRAY_SIZE(names); i++) + seq_printf(seq, "%s=%u\n", names[i], + get_unaligned_le32(wifi->rx_crypto_stats + i * 4)); + mutex_unlock(&wifi->lock); + return 0; +} + +static int rx_crypto_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, rx_crypto_stats_show, inode->i_private); +} + +static const struct file_operations rx_crypto_stats_ops = { + .owner = THIS_MODULE, + .open = rx_crypto_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int dot11_mib_stats_show(struct seq_file *seq, void *unused) +{ + /* ACIStatisticsAdapter::printDot11MibStats at 0x1809d7fb0: + * statistic bit 25, scope 2/interface 0, TLV 0xf91b, 24 LE64s. + * Keep the host field names: tx_status_success is firmware accounting, + * not independent proof of over-the-air acknowledgement or AP delivery. + */ + static const char * const names[] = { + "rx_msdu_pushed", "rx_msdu_pushed_multi", "rx_mpdu", "rx_mpdu_dup", + "rx_ampdu_dup", "rx_mpdu_drop_error", "rx_msdu_drop_filter", + "rx_holes_0", "rx_holes_1", "rx_holes_2", "rx_holes_3", "rx_holes_4", + "rx_holes_5", "rx_holes_6", "rx_holes_7", "rx_holes_8", + "tx_pulled", "tx_pulled_multi", "tx_pulled_core", "tx_status_success", + "tx_status_abort_request", "tx_status_peer_gone", + "tx_status_retries_exhausted", "tx_status_lifetime_expired", + }; + struct centauri_wifi *wifi = seq->private; + u8 payload[22] = {}; + unsigned int i; + int ret; + + put_unaligned_le64(BIT_ULL(25), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + /* The scope byte follows the eight-byte interface value in TLV 363. */ + payload[21] = 2; + mutex_lock(&wifi->lock); + wifi->dot11_mib_stats_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + seq_printf(seq, "rc=%d valid=%d size=%u tag=%#x length=%u\n", ret, + wifi->dot11_mib_stats_valid, wifi->stats_reply_size, wifi->stats_tag, wifi->stats_length); + if (!ret && wifi->dot11_mib_stats_valid) + for (i = 0; i < ARRAY_SIZE(names); i++) + seq_printf(seq, "%s=%llu\n", names[i], + (unsigned long long)get_unaligned_le64(wifi->dot11_mib_stats + i * 8)); + mutex_unlock(&wifi->lock); + return 0; +} + +static int dot11_mib_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, dot11_mib_stats_show, inode->i_private); +} + +static const struct file_operations dot11_mib_stats_ops = { + .owner = THIS_MODULE, + .open = dot11_mib_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int tx_power_report_show(struct seq_file *seq, void *unused) +{ + /* Exact stock 25G76 WLANACIAdapter::queryLastTxStats, 0x180914d58: + * bit29, scope1/radio bitmap, TLV 0xf920 with 280 bytes. The newer + * macOS27 adapter expects 320 bytes and has a different record stride. + * Stock getLastTxInfo (0x1809150f8) reads the first five 28-byte records. + * These are rate/power observations, not proof of ACK or AP delivery. + */ + struct centauri_wifi *wifi = seq->private; + u8 payload[22] = {}; + unsigned int radio, i; + int ret; + + put_unaligned_le64(BIT_ULL(29), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + payload[21] = 1; + mutex_lock(&wifi->lock); + for (radio = 1; radio <= 2; radio++) { + put_unaligned_le64(radio, payload + 13); + wifi->tx_power_report_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + seq_printf(seq, "radio_bitmap=%u rc=%d valid=%d size=%u tag=%#x length=%u\n", + radio, ret, wifi->tx_power_report_valid, wifi->stats_reply_size, + wifi->stats_tag, wifi->stats_length); + if (ret || !wifi->tx_power_report_valid) + continue; + for (i = 0; i < 5; i++) { + const u8 *record = wifi->tx_power_report + i * 28; + + /* Keep hardware enums and power bytes raw until interpreted + * against this exact firmware; omit unreviewed record fields. + */ + seq_printf(seq, "sample=%u power0_qdb_raw=%u power1_qdb_raw=%u power_qdb_raw=%u " + "hw_preamble=%u hw_rate=%u nss=%u hw_bw=%u hw_gi=%u\n", + i, record[14], record[15], record[16], record[17], + record[20], record[21] + 1, record[24], record[27]); + } + } + mutex_unlock(&wifi->lock); + return 0; +} + +static int tx_power_report_open(struct inode *inode, struct file *file) +{ + return single_open(file, tx_power_report_show, inode->i_private); +} + +static const struct file_operations tx_power_report_ops = { + .owner = THIS_MODULE, + .open = tx_power_report_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +int centauri_wifi_register(struct centauri *c) +{ + struct centauri_wifi *wifi; + unsigned int i; + int ret; + + wifi = kzalloc(sizeof(*wifi), GFP_KERNEL); + if (!wifi) + return -ENOMEM; + c->wifi = wifi; + wifi->control = c; + mutex_init(&wifi->lock); + mutex_init(&wifi->fwlog_lock); + skb_queue_head_init(&wifi->data_tx_queue); + INIT_WORK(&wifi->scan_work, scan_worker); + INIT_WORK(&wifi->connect_work, connect_worker); + INIT_DELAYED_WORK(&wifi->link_work, link_worker); + ret = centauri_alpha_resources(c, &wifi->a); + if (ret) + goto failed; + wifi->wiphy = wiphy_new(&wifi_ops, sizeof(struct centauri_wifi *)); + wifi->netdev = alloc_etherdev(sizeof(struct centauri_wifi *)); + if (!wifi->wiphy || !wifi->netdev) { + ret = -ENOMEM; + goto failed; + } + *(struct centauri_wifi **)wiphy_priv(wifi->wiphy) = wifi; + *(struct centauri_wifi **)netdev_priv(wifi->netdev) = wifi; + set_wiphy_dev(wifi->wiphy, &c->pdev->dev); + wifi->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION); + wifi->wiphy->max_scan_ssids = 1; + wifi->wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM; + wifi->wiphy->reg_notifier = wifi_regulatory; + wifi->wiphy->cipher_suites = cipher_suites; + wifi->wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites); + wiphy_ext_feature_set(wifi->wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); + for (i = 0; i < 13; i++) { + wifi->channels[i].band = NL80211_BAND_2GHZ; + wifi->channels[i].hw_value = i + 1; + wifi->channels[i].center_freq = ieee80211_channel_to_frequency(i + 1, NL80211_BAND_2GHZ); + wifi->channels[i].max_power = 20; + } + for (i = 0; i < ARRAY_SIZE(five_ghz); i++) { + wifi->channels[13 + i].band = NL80211_BAND_5GHZ; + wifi->channels[13 + i].hw_value = five_ghz[i]; + wifi->channels[13 + i].center_freq = ieee80211_channel_to_frequency(five_ghz[i], NL80211_BAND_5GHZ); + wifi->channels[13 + i].max_power = 20; + } + wifi->bands[0].channels = wifi->channels; + wifi->bands[0].n_channels = 13; + wifi->bands[0].bitrates = rates; + wifi->bands[0].n_bitrates = ARRAY_SIZE(rates); + wifi->bands[1].channels = wifi->channels + 13; + wifi->bands[1].n_channels = ARRAY_SIZE(five_ghz); + wifi->bands[1].bitrates = rates + 4; + wifi->bands[1].n_bitrates = ARRAY_SIZE(rates) - 4; + wifi->wiphy->bands[NL80211_BAND_2GHZ] = &wifi->bands[0]; + wifi->wiphy->bands[NL80211_BAND_5GHZ] = &wifi->bands[1]; + ret = wiphy_register(wifi->wiphy); + if (ret) + goto failed; + wifi->registered = true; + wifi->wdev.wiphy = wifi->wiphy; + wifi->wdev.iftype = NL80211_IFTYPE_STATION; + wifi->wdev.netdev = wifi->netdev; + wifi->netdev->ieee80211_ptr = &wifi->wdev; + wifi->netdev->netdev_ops = &net_ops; + SET_NETDEV_DEV(wifi->netdev, &c->pdev->dev); + strscpy(wifi->netdev->name, "wlan%d", IFNAMSIZ); + wifi->netdev->max_mtu = ETH_DATA_LEN; + eth_hw_addr_random(wifi->netdev); + ret = register_netdev(wifi->netdev); + if (ret) + goto failed; + ret = radio_start(wifi); + if (ret) { + dev_err(&c->pdev->dev, "Wi-Fi ring/interface setup failed: %d\n", ret); + goto failed; + } + dev_info(&c->pdev->dev, "cfg80211 interface %s ready for scan tests; packet data unavailable\n", + wifi->netdev->name); + wifi->data_probe_file = debugfs_create_file("data_probe", 0200, c->debug, wifi, &data_probe_ops); + wifi->data_status_file = debugfs_create_file("data_status", 0400, c->debug, wifi, &data_status_ops); + wifi->data_rx_snapshot_file = debugfs_create_file("data_rx_snapshot", 0400, + c->debug, wifi, &data_rx_snapshot_ops); + wifi->station_stats_file = debugfs_create_file("station_stats", 0400, c->debug, wifi, &station_stats_ops); + wifi->rx_crypto_stats_file = debugfs_create_file("rx_crypto_stats", 0400, + c->debug, wifi, &rx_crypto_stats_ops); + wifi->dot11_mib_stats_file = debugfs_create_file("dot11_mib_stats", 0400, + c->debug, wifi, &dot11_mib_stats_ops); + wifi->tx_power_report_file = debugfs_create_file("tx_power_report", 0400, + c->debug, wifi, &tx_power_report_ops); + wifi->eapol_start_file = debugfs_create_file("eapol_start", 0200, c->debug, wifi, &eapol_start_ops); + wifi->country_file = debugfs_create_file("country", 0400, c->debug, wifi, &country_ops); + wifi->link_status_file = debugfs_create_file("link_status", 0400, c->debug, wifi, &link_status_ops); + wifi->tx_config_file = debugfs_create_file("tx_config", 0600, c->debug, wifi, &tx_config_ops); + wifi->host_wakeup_file = debugfs_create_file("host_wakeup", 0200, c->debug, wifi, &host_wakeup_ops); + wifi->fwlog_start_file = debugfs_create_file("fwlog_start", 0200, c->debug, wifi, &fwlog_start_ops); + wifi->fwlog_status_file = debugfs_create_file("fwlog_status", 0400, c->debug, wifi, &fwlog_status_ops); + wifi->fwlog_buffer_file = debugfs_create_file("fwlog_buffer", 0400, c->debug, wifi, &fwlog_buffer_ops); + wifi->fwlog_ack_file = debugfs_create_file("fwlog_ack", 0200, c->debug, wifi, &fwlog_ack_ops); + return 0; +failed: + centauri_wifi_unregister(c); + return ret; +} + +void centauri_wifi_unregister(struct centauri *c) +{ + struct centauri_wifi *wifi = c->wifi; + + if (!wifi) + return; + WRITE_ONCE(wifi->stopping, true); + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + netif_tx_disable(wifi->netdev); + debugfs_remove(wifi->data_probe_file); + debugfs_remove(wifi->data_status_file); + debugfs_remove(wifi->data_rx_snapshot_file); + debugfs_remove(wifi->station_stats_file); + debugfs_remove(wifi->rx_crypto_stats_file); + debugfs_remove(wifi->dot11_mib_stats_file); + debugfs_remove(wifi->tx_power_report_file); + debugfs_remove(wifi->eapol_start_file); + debugfs_remove(wifi->country_file); + debugfs_remove(wifi->link_status_file); + debugfs_remove(wifi->tx_config_file); + debugfs_remove(wifi->host_wakeup_file); + debugfs_remove(wifi->fwlog_start_file); + debugfs_remove(wifi->fwlog_status_file); + debugfs_remove(wifi->fwlog_buffer_file); + debugfs_remove(wifi->fwlog_ack_file); + cancel_work_sync(&wifi->scan_work); + cancel_work_sync(&wifi->connect_work); + cancel_delayed_work_sync(&wifi->link_work); + skb_queue_purge(&wifi->data_tx_queue); + if (wifi->scan_request) { + struct cfg80211_scan_info info = { .aborted = true }; + + cfg80211_scan_done(wifi->scan_request, &info); + wifi->scan_request = NULL; + } + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + unregister_netdev(wifi->netdev); + if (wifi->registered) + wiphy_unregister(wifi->wiphy); + if (wifi->wiphy) + wiphy_free(wifi->wiphy); + if (wifi->netdev) + free_netdev(wifi->netdev); + memzero_explicit(wifi, sizeof(*wifi)); + kfree(wifi); + c->wifi = NULL; +} diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 05f73eb1dd5e9..e4c9a1ccd38a0 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2728,6 +2728,18 @@ static const struct apple_nvme_hw apple_nvme_t8140_hw = { .quirks = NVME_QUIRK_BROKEN_FUA, }; +/* + * T8142 (M5) is laid out like T8140; the final shutdown sends SHN, as on + * every post-M4 controller. + */ +static const struct apple_nvme_hw apple_nvme_t8142_hw = { + .has_lsq_nvmmu = true, + .has_separate_nvmmu = true, + .needs_ioq_registers = true, + .max_queue_depth = 64, + .quirks = NVME_QUIRK_BROKEN_FUA, +}; + /* * T6040 (M4 Pro) with the 26.x ANS firmware: the T8132 controller, but the * "nvme" register block ends at 0x10000 and the linear submission queue @@ -2748,6 +2760,7 @@ static const struct of_device_id apple_nvme_of_match[] = { { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, { .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw }, { .compatible = "apple,t8140-nvme-ans2", .data = &apple_nvme_t8140_hw }, + { .compatible = "apple,t8142-nvme-ans2", .data = &apple_nvme_t8142_hw }, { .compatible = "apple,nvme-ans2", .data = &apple_nvme_t8103_hw }, {}, }; diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 7a72e76b08262..a58abe6b1921d 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -83,6 +83,14 @@ config PCIE_APPLE If unsure, say Y if you have an Apple Silicon system. +config PCIE_APPLE_MSI + bool "Apple PCIe MSI controller for a bootloader-trained port" + depends on PCIE_APPLE=y + select IRQ_MSI_LIB + help + MSIs for an Apple PCIe port that Linux drives as a generic ECAM + host because the bootloader trained the link (T8142 Wi-Fi). + config PCI_VERSATILE bool "ARM Versatile PB PCI controller" depends on ARCH_VERSATILE || COMPILE_TEST diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 978c38a61f44c..d09974900927c 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -40,6 +40,7 @@ obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o obj-$(CONFIG_PCIE_HISI_ERR) += pcie-hisi-error.o obj-$(CONFIG_PCIE_APPLE) += pcie-apple.o +obj-$(CONFIG_PCIE_APPLE_MSI) += pcie-apple-msi.o obj-$(CONFIG_PCIE_MT7621) += pcie-mt7621.o obj-$(CONFIG_PCIE_ASPEED) += pcie-aspeed.o diff --git a/drivers/pci/controller/pcie-apple-msi.c b/drivers/pci/controller/pcie-apple-msi.c new file mode 100644 index 0000000000000..452e124c43b9b --- /dev/null +++ b/drivers/pci/controller/pcie-apple-msi.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MSI controller for one Apple PCIe port whose link the bootloader trained. + * + * pcie-apple owns the port when Linux brings the link up itself. On T8142 + * (J813) the bootloader trains apcie port 0 and Linux drives the root port as + * a generic ECAM host, which has no MSI support. The port still turns writes + * to the MSI doorbell into AIC interrupts through its MSI map; this driver + * programs that map and provides the parent domain, as pcie-apple does. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PORT_MSICFG 0x0124 +#define PORT_MSICFG_EN BIT(0) +#define PORT_MSIADDR 0x016c +#define PORT_MSIMAP 0x3800 +#define PORT_MSIMAP_ENABLE BIT(31) + +#define DOORBELL_ADDR CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR + +struct apple_msi { + struct mutex lock; + struct irq_fwspec fwspec; + unsigned long *bitmap; + u32 nvecs; +}; + +static void apple_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) +{ + msg->address_hi = upper_32_bits(DOORBELL_ADDR); + msg->address_lo = lower_32_bits(DOORBELL_ADDR); + msg->data = data->hwirq; +} + +static struct irq_chip apple_msi_bottom_chip = { + .name = "MSI", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_set_type = irq_chip_set_type_parent, + .irq_compose_msi_msg = apple_msi_compose_msg, +}; + +static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct apple_msi *msi = domain->host_data; + struct irq_fwspec fwspec = msi->fwspec; + unsigned int i; + int ret, hwirq; + + mutex_lock(&msi->lock); + hwirq = bitmap_find_free_region(msi->bitmap, msi->nvecs, + order_base_2(nr_irqs)); + mutex_unlock(&msi->lock); + if (hwirq < 0) + return -ENOSPC; + + fwspec.param[fwspec.param_count - 2] += hwirq; + + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &fwspec); + if (ret) { + mutex_lock(&msi->lock); + bitmap_release_region(msi->bitmap, hwirq, order_base_2(nr_irqs)); + mutex_unlock(&msi->lock); + return ret; + } + + for (i = 0; i < nr_irqs; i++) + irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, + &apple_msi_bottom_chip, msi); + return 0; +} + +static void apple_msi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct apple_msi *msi = domain->host_data; + + mutex_lock(&msi->lock); + bitmap_release_region(msi->bitmap, d->hwirq, order_base_2(nr_irqs)); + mutex_unlock(&msi->lock); +} + +static const struct irq_domain_ops apple_msi_domain_ops = { + .alloc = apple_msi_domain_alloc, + .free = apple_msi_domain_free, +}; + +static const struct msi_parent_ops apple_msi_parent_ops = { + .supported_flags = (MSI_GENERIC_FLAGS_MASK | + MSI_FLAG_PCI_MSIX | + MSI_FLAG_MULTI_PCI_MSI), + .required_flags = (MSI_FLAG_USE_DEF_DOM_OPS | + MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSI_MASK_PARENT), + .chip_flags = MSI_CHIP_FLAG_SET_EOI, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int apple_msi_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct fwnode_handle *fwnode = dev_fwnode(dev); + struct of_phandle_args args = {}; + struct irq_domain_info info = { + .fwnode = fwnode, + .ops = &apple_msi_domain_ops, + }; + struct apple_msi *msi; + void __iomem *port; + int ret; + u32 i; + + msi = devm_kzalloc(dev, sizeof(*msi), GFP_KERNEL); + if (!msi) + return -ENOMEM; + mutex_init(&msi->lock); + + port = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(port)) + return PTR_ERR(port); + + ret = of_parse_phandle_with_args(dev->of_node, "msi-ranges", + "#interrupt-cells", 0, &args); + if (ret) + return ret; + ret = of_property_read_u32_index(dev->of_node, "msi-ranges", + args.args_count + 1, &msi->nvecs); + if (ret) { + of_node_put(args.np); + return ret; + } + of_phandle_args_to_fwspec(args.np, args.args, args.args_count, + &msi->fwspec); + of_node_put(args.np); + + msi->bitmap = devm_bitmap_zalloc(dev, msi->nvecs, GFP_KERNEL); + if (!msi->bitmap) + return -ENOMEM; + + info.parent = irq_find_matching_fwspec(&msi->fwspec, DOMAIN_BUS_WIRED); + if (!info.parent) + return dev_err_probe(dev, -EPROBE_DEFER, "no parent domain\n"); + info.size = msi->nvecs; + info.host_data = msi; + + /* MSI data N goes to the Nth interrupt of msi-ranges. */ + writel(lower_32_bits(DOORBELL_ADDR), port + PORT_MSIADDR); + for (i = 0; i < msi->nvecs; i++) + writel(PORT_MSIMAP_ENABLE | i, port + PORT_MSIMAP + 4 * i); + writel(readl(port + PORT_MSICFG) | PORT_MSICFG_EN, port + PORT_MSICFG); + + if (!msi_create_parent_irq_domain(&info, &apple_msi_parent_ops)) + return -ENOMEM; + dev_info(dev, "%u MSIs at %#x\n", msi->nvecs, DOORBELL_ADDR); + return 0; +} + +static const struct of_device_id apple_msi_of_match[] = { + { .compatible = "apple,t8142-pcie-msi" }, + { } +}; + +static struct platform_driver apple_msi_driver = { + .probe = apple_msi_probe, + .driver = { + .name = "pcie-apple-msi", + .of_match_table = apple_msi_of_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(apple_msi_driver); diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 8664a9fd3b7c4..48453fd72e46b 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -88,6 +88,7 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PHY_LANE_CFG_REFCLK1REQ BIT(1) #define PHY_LANE_CFG_REFCLK0ACK BIT(2) #define PHY_LANE_CFG_REFCLK1ACK BIT(3) +#define PHY_LANE_CFG_REFBUF_OFF BIT(4) #define PHY_LANE_CFG_REFCLKEN (BIT(9) | BIT(10)) #define PHY_LANE_CFG_REFCLKCGEN (BIT(30) | BIT(31)) #define PHY_LANE_CTL 0x00004 @@ -172,6 +173,10 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PORT_T602X_MSIADDR 0x016c #define PORT_T602X_MSIADDR_HI 0x0170 #define PORT_T602X_PERST 0x082c +/* T8152: lane PIPE reset is released by clearing this after PERST#. */ +#define PORT_T8152_PIPE_RST BIT(16) +#define PORT_T8152_PIPE_RST_STS 0x00a8 +#define PORT_T8152_PIPE_RST_STS_LANES GENMASK(3, 0) #define PORT_T602X_RID2SID 0x3000 #define PORT_T602X_MSIMAP 0x3800 @@ -209,6 +214,13 @@ struct hw_info { * keeps its link while the domain stays on, see apple_pcie_hold_domain(). */ bool s2idle_hold_domain; + /* AppleT8152PCIePort::_initializeRefclkBuffer clears REFBUF_OFF. */ + bool refbuf_off; + /* + * APCIECoreRCGen5Port::enablePortHardware: after PERST#, wait for the + * lane PIPE reset status, then release PORT_T8152_PIPE_RST. + */ + bool pipe_rst; }; static const struct hw_info t8103_hw = { @@ -252,6 +264,20 @@ static const struct hw_info t8140_hw = { .root_bus_only = true, }; +/* T8152 (M6) apcie: T602x port layout, refclk buffer powered down by iBoot. */ +static const struct hw_info t8152_hw = { + .phy_lane_ctl = 0, + .port_msiaddr = PORT_T602X_MSIADDR, + .port_msiaddr_hi = PORT_T602X_MSIADDR_HI, + .port_refclk = 0, + .port_perst = PORT_T602X_PERST, + .port_rid2sid = PORT_T602X_RID2SID, + .port_msimap = PORT_T602X_MSIMAP, + .max_rid2sid = 512, + .refbuf_off = true, + .pipe_rst = true, +}; + static const struct hw_info t8103_pciec_hw = { .port_msiaddr = PORT_MSIADDR, .port_perst = PORT_PERST, @@ -1521,6 +1547,11 @@ static int apple_pcie_setup_refclk(struct apple_pcie *pcie, if (pcie->hw->phy_lane_ctl) rmw_clear(PHY_LANE_CTL_CFGACC, port->phy + pcie->hw->phy_lane_ctl); + if (pcie->hw->refbuf_off) { + rmw_clear(PHY_LANE_CFG_REFBUF_OFF, port->phy + PHY_LANE_CFG); + udelay(1); + } + rmw_set(PHY_LANE_CFG_REFCLKEN, port->phy + PHY_LANE_CFG); if (pcie->hw->port_refclk) @@ -1635,6 +1666,17 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, for (u32 idx = 0; idx < num_aux_resets; idx++) gpiod_set_value_cansleep(aux_reset[idx], 0); + if (pcie->hw->pipe_rst) { + ret = readl_relaxed_poll_timeout(port->base + PORT_T8152_PIPE_RST_STS, + stat, stat & PORT_T8152_PIPE_RST_STS_LANES, + 1, 20000); + if (ret < 0) { + dev_err(pcie->dev, "port %pOF lanes stuck in PIPE reset\n", np); + return ret; + } + rmw_clear(PORT_T8152_PIPE_RST, port->base + pcie->hw->port_perst); + } + /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ msleep(100); @@ -3941,6 +3983,7 @@ static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t6000-pciec", .data = &t8112_pciec_hw }, /* T8132 currently inherits controller and PHY setup from m1n1. */ { .compatible = "apple,t8132-pcie", .data = &t602x_hw }, + { .compatible = "apple,t8152-pcie", .data = &t8152_hw }, { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, { .compatible = "apple,t8140-pcie", .data = &t8140_hw }, { .compatible = "apple,pcie", .data = &t8103_hw }, diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index b6b5ccdb3e53e..3765e9a775ff7 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -1337,6 +1337,23 @@ static const struct apple_pmc_hw apple_pmc_hw_t8140 = { .ap_agent = 3, }; +/* + * T8142: the AppleT8142PMC accessors put the floors at agent << 14, the + * per-agent states at 0x106000 and the static voter enables at 0x108000; + * rails DCS, SOC and DISP (ADT voter-rail-data). The PMC clamps a floor to + * DCS 7 and SOC 4. + */ +static const struct apple_pmc_hw apple_pmc_hw_t8142 = { + .agent_stride = 0x4000, + .agents = 4, + .rails = 3, + .agent_state = 0x0, + .rail_enable = 0x2000, + .dcs_rail = 0, + .dcs_max = 7, + .ap_agent = 3, +}; + /* * T6040 (M4 Pro): the T8140 layout, with DCS states 0-5 and fabric states * 0-2. The ADT's voter-agent rail mask is 0x7 (DCS, SOC and DISP_INT). @@ -1356,9 +1373,23 @@ static const struct apple_pmc_hw apple_pmc_hw_t6040 = { .ap_agent = 3, }; +/* T8152 uses the T8142 voter interface with its rails at 0x3029c6000. */ +static const struct apple_pmc_hw apple_pmc_hw_t8152 = { + .agent_stride = 0x4000, + .agents = 4, + .rails = 3, + .agent_state = 0x0, + .rail_enable = 0x2000, + .dcs_rail = 0, + .dcs_max = 7, + .ap_agent = 3, +}; + static const struct of_device_id apple_pmc_of_match[] = { { .compatible = "apple,t8132-pmc", .data = &apple_pmc_hw_t8132 }, { .compatible = "apple,t8140-pmc", .data = &apple_pmc_hw_t8140 }, + { .compatible = "apple,t8142-pmc", .data = &apple_pmc_hw_t8142 }, + { .compatible = "apple,t8152-pmc", .data = &apple_pmc_hw_t8152 }, { .compatible = "apple,t6041-pmc", .data = &apple_pmc_hw_t6040 }, {} }; diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index ae0d6d48bac7f..c7b29f0e6a49b 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -78,6 +78,9 @@ enum apple_clpc_rail { * @pmp_agent, limits the DCS, see apple_clpc_gpu_limited() * @gpu_dense: submissions per sampling period that make light GPU work * dense, see apple_clpc_gpu_dense(); 0 for none + * @gpu_input: the GPU driver reports its work in flight through + * apple_clpc_gpu_get(); while there is some, both rails go to the top + * of @map */ struct apple_clpc_hw { unsigned int rail[APPLE_CLPC_RAILS]; @@ -89,6 +92,7 @@ struct apple_clpc_hw { bool pmp_backoff; unsigned int pmp_agent; unsigned int gpu_dense; + bool gpu_input; }; struct apple_clpc_cluster { @@ -228,7 +232,8 @@ int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) return of_find_property(dev->of_node, "apple,clpc", NULL) ? -EINVAL : 0; if (!of_device_is_compatible(np, "apple,t6041-clpc") && !of_device_is_compatible(np, "apple,t8132-clpc") && - !of_device_is_compatible(np, "apple,t8140-clpc")) { + !of_device_is_compatible(np, "apple,t8140-clpc") && + !of_device_is_compatible(np, "apple,t8142-clpc")) { of_node_put(np); return -EINVAL; } @@ -744,13 +749,11 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->pmc = dev->parent; clpc->hw = of_device_get_match_data(dev); /* T6040 firmware keeps GFX powered for its whole lifetime. A separate - * host publication/idle input, not that supplier state, drives demand. - * On T8140 the same input counts submissions for dense light work. The - * T8132 GPU driver reports its work through it as well. + * host publication/idle input, not that supplier state, drives demand; + * the T8132 and M5 (T8142) GPU drivers report through the same input. + * On T8140 the same input counts submissions for dense light work. */ - if (of_device_is_compatible(dev->of_node, "apple,t6041-clpc") || - of_device_is_compatible(dev->of_node, "apple,t8132-clpc") || - clpc->hw->gpu_dense) { + if (clpc->hw->gpu_input || clpc->hw->gpu_dense) { clpc->gpu = kzalloc_obj(*clpc->gpu); if (!clpc->gpu) return -ENOMEM; @@ -882,6 +885,7 @@ static const struct apple_clpc_hw apple_clpc_hw_t8132 = { [APPLE_CLPC_SOC] = { 3, 3, 3, 3 }, }, .gpu_demand = false, + .gpu_input = true, }; /* @@ -948,6 +952,26 @@ static const struct apple_clpc_hw apple_clpc_hw_t6040 = { }, .sustain = 3, .gpu_demand = false, + .gpu_input = true, +}; + +/* + * T8142: DCS states 0-7 on rail 0, fabric states 0-4 on rail 1. The fabric + * state bounds CPU memory bandwidth: four P cores copy at ~40 GB/s with the + * fabric at 1 and at ~110 GB/s at 4 (macOS ~124). + * + * The M5 GPU driver reports its submissions in flight. Once the PMP runs, its + * vote (agent 0) drops from iBoot's DCS 7 / fabric 1 to 0 and nothing else + * raises the floor for the GPU: a 4K RGBA16F fill then takes 5.1 ms instead + * of 0.9 ms with both rails at the top of the map. + */ +static const struct apple_clpc_hw apple_clpc_hw_t8142 = { + .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, + .map = { + [APPLE_CLPC_DCS] = { 2, 4, 6, 7 }, + [APPLE_CLPC_SOC] = { 1, 2, 3, 4 }, + }, + .gpu_input = true, }; #if IS_BUILTIN(CONFIG_KUNIT) && !defined(MODULE) @@ -1117,9 +1141,20 @@ static struct kunit_suite apple_clpc_suite = { kunit_test_suite(apple_clpc_suite); #endif +/* T8152: DCS states 0-7 and fabric states 0-4. */ +static const struct apple_clpc_hw apple_clpc_hw_t8152 = { + .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, + .map = { + [APPLE_CLPC_DCS] = { 2, 4, 6, 7 }, + [APPLE_CLPC_SOC] = { 1, 2, 3, 4 }, + }, +}; + static const struct of_device_id apple_clpc_of_match[] = { { .compatible = "apple,t8132-clpc", .data = &apple_clpc_hw_t8132 }, { .compatible = "apple,t8140-clpc", .data = &apple_clpc_hw_t8140 }, + { .compatible = "apple,t8142-clpc", .data = &apple_clpc_hw_t8142 }, + { .compatible = "apple,t8152-clpc", .data = &apple_clpc_hw_t8152 }, { .compatible = "apple,t6041-clpc", .data = &apple_clpc_hw_t6040 }, {} }; diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 6f0d1f7957ec0..c13c03f9d92ab 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) @@ -66,6 +67,83 @@ #define APPLE_T6040_GFX_WRAPPER_BASE 0x28a600000ULL #define APPLE_T6040_GFX_IOP_VBAR_BASE 0x28a050000ULL #define APPLE_T6050_ASCWRAP_BASE_DELTA 0x380000000ULL +/* T8142 (M5) moves T8140's 0x4... GPU ASC bank to 0x5.... */ +#define APPLE_T8142_ASCWRAP_BASE_DELTA 0x100000000ULL + +/* + * T8152 (J873) GPU ASCWrap v8 providers. The ADT gfx-asc/gfx1-asc nodes are + * iop,ascwrap-v8 with reg[0] = wrapper (0x74000), reg[1] = iop-vbar (8) and + * reg[2] = the post-office bank 0x316020000/0x10000, which is shared with + * PMP, DCP, DCPEXT0/1 and SCALER0. Only the GFX/GFX1 0x80-byte windows of + * that bank are claimed: offset (2 * msgbox-mailbox-num + !postoffice) * + * 0x80 with msgbox-using-alias=1 (m1n1 asc.c) gives 0x316020100 (mailbox 1) + * and 0x316020200 (mailbox 2). Each window is a counted FIFO: RX at +0, + * TX at +0x10 and a status word at +0x20 that holds the RX occupancy and the + * free TX slots. + * + * The wrapper is only touched through the lifecycle helpers, and every one + * of those accesses is preceded by a read of the block's PMGR2 PSD service + * (implicit services 81/82 at 0x303a15100/0x303a15200: state +0x80, ack + * +0xbc). The provider never writes PSD registers. + * + * Receive side. ASCWrap v8 has no mailbox-level interrupt control, so + * has_irq_controls is false and a level interrupt can only be masked at the + * AIC. By default the provider therefore never enables either GPU mailbox + * interrupt: apple_mbox_start() starts a one-jiffy receive poller, and a + * full TX FIFO is waited out by sleeping polls. Only ascwrap_v8_rx_irq=1 + * uses the recv-not-empty line; even then a line that keeps firing with + * nothing to read is disabled and receive falls back to the poller. Every + * post-office receive poll is preceded by the same PSD check as the wrapper + * accesses. + */ +#define APPLE_ASCWRAP_V8_WRAPPER_SIZE 0x74000 +#define APPLE_ASCWRAP_V8_IOP_VBAR_SIZE 0x8 +#define APPLE_ASCWRAP_V8_MBOX_SIZE 0x80 +#define APPLE_ASCWRAP_V8_PSD_SIZE 0x100 +/* m1n1 asc.c starts a v8 CPU with reg[0] + 0x44 bit 4. */ +#define APPLE_ASCWRAP_V8_CPU_CONTROL 0x44 +#define APPLE_ASCWRAP_V8_CPU_STATUS 0x48 +#define APPLE_ASCWRAP_V8_CPU_RUN BIT(4) +#define APPLE_PSD_STATE 0x80 +#define APPLE_PSD_ACK 0xbc +#define APPLE_PSD_STATE_MASK GENMASK(2, 0) +#define APPLE_T8152_GPU_ASC_PSD_ACTIVE 2 +/* apple_mbox_ascwrap_v6_lifecycle.missing bit: CPU control not enabled. */ +#define APPLE_ASCWRAP_V8_MISSING_CPU_CONTROL_APPROVAL BIT(0) +/* Receive poll period (poll-only mode, or after the storm guard tripped). */ +#define APPLE_ASCWRAP_V8_RX_POLL_JIFFIES 1 +/* Consecutive recv interrupts that drained nothing before the line is cut. */ +#define APPLE_ASCWRAP_V8_RX_STORM_LIMIT 1000 + +#define APPLE_T8152_GFX_WRAPPER_BASE 0x582600000ULL +#define APPLE_T8152_GFX_IOP_VBAR_BASE 0x582050000ULL +#define APPLE_T8152_GFX_MBOX_BASE 0x316020100ULL +#define APPLE_T8152_GFX_PSD_BASE 0x303a15100ULL +#define APPLE_T8152_GFX1_WRAPPER_BASE 0x582e00000ULL +#define APPLE_T8152_GFX1_IOP_VBAR_BASE 0x582850000ULL +#define APPLE_T8152_GFX1_MBOX_BASE 0x316020200ULL +#define APPLE_T8152_GFX1_PSD_BASE 0x303a15200ULL + +/* + * Starting or stopping the ASC CPU of a T8152 GPU wrapper changes the GPU's + * state and is not qualified as a default. Every v8 CPU start/stop request + * is refused (-EPERM) unless this is set; lifecycle reads are always allowed. + */ +static bool ascwrap_v8_cpu_control; +module_param(ascwrap_v8_cpu_control, bool, 0444); +MODULE_PARM_DESC(ascwrap_v8_cpu_control, + "Allow T8152 GPU ASCWrap v8 CPU start/stop writes (default: refuse)"); + +/* + * The GFX/GFX1 interrupt assignment has not been observed on hardware, and a + * wrongly wired level line would storm with interrupts that the handler + * cannot mask. The default services receive by polling and never enables a + * GPU mailbox interrupt. + */ +static bool ascwrap_v8_rx_irq; +module_param(ascwrap_v8_rx_irq, bool, 0444); +MODULE_PARM_DESC(ascwrap_v8_rx_irq, + "Service T8152 GPU ASCWrap v8 receive with the recv-not-empty interrupt (default: poll, no GPU mailbox interrupt is enabled)"); static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); @@ -140,6 +218,10 @@ struct apple_mbox_hw { bool is_ascwrap_v6; const struct apple_mbox_ascwrap_v6_role_desc *ascwrap_v6_roles; unsigned int num_ascwrap_v6_roles; + /* T8152 GPU ASCWrap v8: wrapper + iop-vbar + post-office window + PSD. */ + bool is_ascwrap_v8; + /* Status word counts RX occupancy and free TX slots; paired transfers. */ + bool counted_fifo; bool ap_initializes_mailboxes; unsigned int control_full; @@ -160,6 +242,68 @@ struct apple_mbox_hw { unsigned int irq_bit_send_empty; }; +/* + * Provider-private state kept outside the consumer-visible struct apple_mbox. + * Every provider is allocated as one of these; apple_mbox_get() hands out a + * pointer to the embedded mailbox. + */ +struct apple_mbox_priv { + struct apple_mbox mbox; + /* ASCWrap v8 only: the provider's PMGR2 PSD service block (read-only). */ + void __iomem *psd_regs; + /* ASCWrap v8 only: receive by polling; fixed at probe. */ + bool rx_poll_only; + /* rx_lock: recv interrupt enabled and not yet disabled again. */ + bool rx_irq_live; + /* rx_lock: consecutive recv interrupts that drained nothing. */ + unsigned int rx_irq_empty_run; + /* rx_lock: an unpowered-PSD receive skip was logged this activation. */ + bool rx_psd_skip_logged; + struct delayed_work rx_poll_work; +}; + +static struct apple_mbox_priv *apple_mbox_to_priv(struct apple_mbox *mbox) +{ + return container_of(mbox, struct apple_mbox_priv, mbox); +} + +/* Only providers allocated by apple_mbox_probe() have a receive poller. */ +static bool apple_mbox_has_rx_poller(const struct apple_mbox *mbox) +{ + return mbox->hw->is_ascwrap_v8; +} + +/* Both ASCWrap generations share the consumer lifecycle API below. */ +static bool apple_mbox_has_cpu_lifecycle(const struct apple_mbox *mbox) +{ + return mbox->hw->is_ascwrap_v6 || mbox->hw->is_ascwrap_v8; +} + +/* + * Read-only check that a v8 ASC block is powered before any wrapper MMIO. + * The T8152 GFX/GFX1 services are active in state 2. Reading STATE clears + * the transition ACK, so ACK is sampled first and not required to remain + * set for steady-state access. The cold state 6 is rejected. Never writes + * the PSD request register. + */ +static int apple_mbox_ascwrap_v8_psd_powered(struct apple_mbox *mbox, + u32 *state, u32 *ack) +{ + void __iomem *psd = apple_mbox_to_priv(mbox)->psd_regs; + + *state = 0; + *ack = 0; + if (!mbox->hw->is_ascwrap_v8 || !psd) + return -ENODEV; + + *ack = readl(psd + APPLE_PSD_ACK); + *state = readl(psd + APPLE_PSD_STATE); + if (FIELD_GET(APPLE_PSD_STATE_MASK, *state) != APPLE_T8152_GPU_ASC_PSD_ACTIVE) + return -EHOSTDOWN; + + return 0; +} + int apple_mbox_ascwrap_v6_get_lifecycle( struct apple_mbox *mbox, struct apple_mbox_ascwrap_v6_lifecycle *lifecycle) @@ -168,7 +312,7 @@ int apple_mbox_ascwrap_v6_get_lifecycle( return -EINVAL; if (READ_ONCE(mbox->removing)) return -ENODEV; - if (!mbox->hw->is_ascwrap_v6) + if (!apple_mbox_has_cpu_lifecycle(mbox)) return -EOPNOTSUPP; *lifecycle = mbox->ascwrap_v6; @@ -178,7 +322,7 @@ EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_get_lifecycle); int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) { - if (!mbox || !mbox->hw->is_ascwrap_v6) + if (!mbox || !apple_mbox_has_cpu_lifecycle(mbox)) return -EOPNOTSUPP; if (READ_ONCE(mbox->removing)) return -ENODEV; @@ -199,14 +343,35 @@ static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) if (!mbox->ascwrap_v6_cpu_running) return; - cpu_control = (u8 __iomem *)mbox->wrapper_regs + - APPLE_ASCWRAP_V6_CPU_CONTROL; + if (mbox->hw->is_ascwrap_v8) { + u32 state, ack; - /* Stop: clear RUN, reread, clear bit 5. */ - val = readl(cpu_control); - writel(val & ~APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); - val = readl(cpu_control); - writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); + /* + * Reached only after an allowed v8 start. As m1n1 + * asc_cpu_stop(): clear RUN at +0x44. An unpowered block cannot + * be running, and touching its wrapper would fault, so skip the + * write then. + */ + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_CONTROL; + if (apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack)) { + dev_warn(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; CPU stop write skipped\n", + state, ack); + } else { + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V8_CPU_RUN, cpu_control); + } + } else { + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; + + /* Stop: clear RUN, reread, clear bit 5. */ + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); + } mbox->ascwrap_v6_cpu_running = false; pm_runtime_mark_last_busy(mbox->dev); @@ -231,8 +396,11 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) u32 val; int ret = 0; - if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + if (!mbox || !apple_mbox_has_cpu_lifecycle(mbox) || !mbox->wrapper_regs) return -EOPNOTSUPP; + /* See the ascwrap_v8_cpu_control parameter. */ + if (mbox->hw->is_ascwrap_v8 && !ascwrap_v8_cpu_control) + return -EPERM; mutex_lock(&mbox->lifecycle_lock); if (mbox->removing) { @@ -247,6 +415,17 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) ret = -EBUSY; goto out_unlock; } + if (mbox->hw->is_ascwrap_v8) { + u32 state, ack; + + ret = apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack); + if (ret) { + dev_err(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; CPU start refused\n", + state, ack); + goto out_unlock; + } + } ret = pm_runtime_resume_and_get(mbox->dev); if (ret) { @@ -255,12 +434,20 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) goto out_unlock; } - cpu_control = (u8 __iomem *)mbox->wrapper_regs + - APPLE_ASCWRAP_V6_CPU_CONTROL; + if (mbox->hw->is_ascwrap_v8) { + /* As m1n1 asc_cpu_start(): set RUN at reg[0] + 0x44. */ + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_CONTROL; + val = readl(cpu_control); + writel(val | APPLE_ASCWRAP_V8_CPU_RUN, cpu_control); + } else { + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; - /* Start: read, set RUN, write. */ - val = readl(cpu_control); - writel(val | APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + /* Start: read, set RUN, write. */ + val = readl(cpu_control); + writel(val | APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + } mbox->ascwrap_v6_cpu_running = true; out_unlock: @@ -273,8 +460,11 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) { int ret = 0; - if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + if (!mbox || !apple_mbox_has_cpu_lifecycle(mbox) || !mbox->wrapper_regs) return -EOPNOTSUPP; + /* See the ascwrap_v8_cpu_control parameter. */ + if (mbox->hw->is_ascwrap_v8 && !ascwrap_v8_cpu_control) + return -EPERM; mutex_lock(&mbox->lifecycle_lock); if (mbox->removing) { @@ -282,6 +472,7 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) goto out_unlock; } + /* A no-op unless this provider started the CPU itself. */ apple_mbox_ascwrap_v6_stop_cpu_locked(mbox); out_unlock: @@ -300,12 +491,32 @@ int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, { int ret = 0; - if (!mbox || !control || !status || !mbox->hw->is_ascwrap_v6 || + if (!mbox || !control || !status || !apple_mbox_has_cpu_lifecycle(mbox) || !mbox->wrapper_regs) return -EOPNOTSUPP; mutex_lock(&mbox->lifecycle_lock); if (mbox->removing) { ret = -ENODEV; + } else if (mbox->hw->is_ascwrap_v8) { + u32 state, ack; + + /* PSD check first; an unpowered wrapper read would SError. */ + *control = 0; + *status = 0; + ret = apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack); + if (ret) { + dev_info_ratelimited(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; CPU_CONTROL/STATUS reads skipped\n", + state, ack); + } else { + *control = readl((u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_CONTROL); + *status = readl((u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_STATUS); + dev_dbg(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x CPU_CONTROL=%#x CPU_STATUS=%#x\n", + state, ack, *control, *status); + } } else { *control = readl(mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_CONTROL); *status = readl(mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_STATUS); @@ -315,6 +526,72 @@ int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_get_cpu_state); +static bool apple_mbox_tx_full(struct apple_mbox *mbox, u32 status) +{ + if (mbox->hw->counted_fifo) + return !(status & mbox->hw->control_full); + return status & mbox->hw->control_full; +} + +static bool apple_mbox_rx_empty(struct apple_mbox *mbox, u32 status) +{ + if (mbox->hw->counted_fifo) + return !(status & mbox->hw->control_empty); + return status & mbox->hw->control_empty; +} + +/* + * A counted FIFO transfers each message with one paired access. Its status + * word holds the RX occupancy and the free TX slots, not full/empty flags. + * Keep the two words of a message together. + */ +static void apple_mbox_write_msg(struct apple_mbox *mbox, + struct apple_mbox_msg msg) +{ + u64 high = FIELD_PREP(APPLE_MBOX_MSG1_MSG, msg.msg1); + +#ifdef CONFIG_ARM64 + if (mbox->hw->counted_fifo) { + u64 low = (__force u64)cpu_to_le64(msg.msg0); + + high = (__force u64)cpu_to_le64(high); + wmb(); + asm volatile("stp %0, %1, [%2]" + : : "r" (low), "r" (high), + "r" (mbox->regs + mbox->hw->a2i_send0) + : "memory"); + } else +#endif + { + writeq_relaxed(msg.msg0, mbox->regs + mbox->hw->a2i_send0); + writeq_relaxed(high, mbox->regs + mbox->hw->a2i_send1); + } +} + +static struct apple_mbox_msg apple_mbox_read_msg(struct apple_mbox *mbox) +{ + struct apple_mbox_msg msg; + u64 low, high; + +#ifdef CONFIG_ARM64 + if (mbox->hw->counted_fifo) { + asm volatile("ldp %0, %1, [%2]" + : "=&r" (low), "=&r" (high) + : "r" (mbox->regs + mbox->hw->i2a_recv0) + : "memory"); + rmb(); + low = le64_to_cpu((__force __le64)low); + high = le64_to_cpu((__force __le64)high); + } else +#endif + { + low = readq_relaxed(mbox->regs + mbox->hw->i2a_recv0); + high = readq_relaxed(mbox->regs + mbox->hw->i2a_recv1); + } + msg.msg0 = low; + msg.msg1 = FIELD_GET(APPLE_MBOX_MSG1_MSG, high); + return msg; +} int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) @@ -332,7 +609,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, } mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); - while (mbox_ctrl & mbox->hw->control_full) { + while (apple_mbox_tx_full(mbox, mbox_ctrl)) { now = jiffies; if (time_after_eq(now, deadline)) { u32 i2a = readl_relaxed(mbox->regs + mbox->hw->i2a_control); @@ -348,6 +625,23 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, return -ETIMEDOUT; } remaining = deadline - now; + if (!atomic && apple_mbox_has_rx_poller(mbox) && + apple_mbox_to_priv(mbox)->rx_poll_only) { + /* + * Poll-only v8: never enable the send-empty line. + * Sleep-poll the TX status, touching MMIO only under + * tx_lock so that removal cannot unmap it underneath. + */ + spin_unlock_irqrestore(&mbox->tx_lock, flags); + usleep_range(100, 200); + spin_lock_irqsave(&mbox->tx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ENODEV; + } + mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); + continue; + } if (apple_mbox_is_g17(mbox) && !atomic && mbox->irq_send_empty < 0) { /* The GPU has no send-empty IRQ. Do not busy wait with IRQs * off while dead firmware keeps its FIFO full. Recheck both @@ -366,7 +660,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, if (atomic || mbox->irq_send_empty < 0) { ret = readl_poll_timeout_atomic( mbox->regs + mbox->hw->a2i_control, mbox_ctrl, - !(mbox_ctrl & mbox->hw->control_full), 100, + !apple_mbox_tx_full(mbox, mbox_ctrl), 100, apple_mbox_is_g17(mbox) ? APPLE_G17_MBOX_ATOMIC_TX_US : APPLE_MBOX_TX_TIMEOUT * 1000); @@ -419,9 +713,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); } - writeq_relaxed(msg.msg0, mbox->regs + mbox->hw->a2i_send0); - writeq_relaxed(FIELD_PREP(APPLE_MBOX_MSG1_MSG, msg.msg1), - mbox->regs + mbox->hw->a2i_send1); + apple_mbox_write_msg(mbox, msg); spin_unlock_irqrestore(&mbox->tx_lock, flags); @@ -467,13 +759,25 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) /* A stopped receiver may still have firmware traffic in its FIFO. */ if (!mbox->rx) return 0; + if (mbox->hw->is_ascwrap_v8) { + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + u32 state, ack; + + /* PSD check before the post-office status read. */ + ret = apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack); + if (ret) { + if (!priv->rx_psd_skip_logged) + dev_info(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; receive poll skipped\n", + state, ack); + priv->rx_psd_skip_logged = true; + return ret; + } + } mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); - while (!(mbox_ctrl & mbox->hw->control_empty) && budget--) { - msg.msg0 = readq_relaxed(mbox->regs + mbox->hw->i2a_recv0); - msg.msg1 = FIELD_GET( - APPLE_MBOX_MSG1_MSG, - readq_relaxed(mbox->regs + mbox->hw->i2a_recv1)); + while (!apple_mbox_rx_empty(mbox, mbox_ctrl) && budget--) { + msg = apple_mbox_read_msg(mbox); mbox->rx(mbox, msg, mbox->cookie); ret++; @@ -529,6 +833,32 @@ int apple_mbox_set_rx_util_min(struct apple_mbox *mbox, int util_min) } EXPORT_SYMBOL_GPL(apple_mbox_set_rx_util_min); +/* + * ASCWrap v8 storm guard, rx_lock held. The handler cannot mask the line at + * the mailbox (no irq controls), so a line that keeps firing with nothing to + * read (for example a send-empty line wired as recv-not-empty) would lock up + * the CPU while always returning IRQ_HANDLED. Cut it at the AIC and poll. + */ +static void apple_mbox_v8_rx_irq_guard(struct apple_mbox *mbox, int drained) +{ + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + + if (drained > 0) { + priv->rx_irq_empty_run = 0; + return; + } + if (!priv->rx_irq_live || + ++priv->rx_irq_empty_run < APPLE_ASCWRAP_V8_RX_STORM_LIMIT) + return; + + priv->rx_irq_live = false; + disable_irq_nosync(mbox->irq_recv_not_empty); + dev_err(mbox->dev, + "ASCWrap v8 recv-not-empty IRQ %d fired %u times without a message; disabled, receive falls back to polling\n", + mbox->irq_recv_not_empty, priv->rx_irq_empty_run); + queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, 0); +} + static irqreturn_t apple_mbox_recv_irq(int irq, void *data) { struct apple_mbox *mbox = data; @@ -542,6 +872,8 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) spin_lock_irqsave(&mbox->rx_lock, flags); if (likely(!READ_ONCE(mbox->removing))) consumed = apple_mbox_poll_locked(mbox); + if (mbox->hw->is_ascwrap_v8) + apple_mbox_v8_rx_irq_guard(mbox, consumed); spin_unlock_irqrestore(&mbox->rx_lock, flags); /* G17 uses a oneshot threaded IRQ. Leave it masked during this sleep @@ -570,6 +902,61 @@ int apple_mbox_poll(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_poll); +/* ASCWrap v8 receive poller: poll-only mode, or after the storm guard. */ +static void apple_mbox_v8_rx_poll_work(struct work_struct *work) +{ + struct apple_mbox_priv *priv = container_of(to_delayed_work(work), + struct apple_mbox_priv, + rx_poll_work); + struct apple_mbox *mbox = &priv->mbox; + + if (!READ_ONCE(mbox->active) || READ_ONCE(mbox->removing)) + return; + apple_mbox_poll(mbox); + queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, + APPLE_ASCWRAP_V8_RX_POLL_JIFFIES); +} + +/* lifecycle_lock held, mbox->active already set. */ +static void apple_mbox_v8_rx_start(struct apple_mbox *mbox) +{ + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + unsigned long flags; + + spin_lock_irqsave(&mbox->rx_lock, flags); + priv->rx_irq_empty_run = 0; + priv->rx_psd_skip_logged = false; + priv->rx_irq_live = !priv->rx_poll_only; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + if (priv->rx_poll_only) + queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, 0); + else + enable_irq(mbox->irq_recv_not_empty); +} + +/* + * lifecycle_lock held (or removal), mbox->active already cleared. The storm + * guard and this path each claim rx_irq_live under rx_lock, so exactly one + * of them balances the enable_irq(). The poller re-queues itself, which + * cancel_delayed_work_sync() handles; it never takes lifecycle_lock. + */ +static void apple_mbox_v8_rx_stop(struct apple_mbox *mbox) +{ + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + unsigned long flags; + bool live; + + spin_lock_irqsave(&mbox->rx_lock, flags); + live = priv->rx_irq_live; + priv->rx_irq_live = false; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + if (live) + disable_irq(mbox->irq_recv_not_empty); + cancel_delayed_work_sync(&priv->rx_poll_work); +} + int apple_mbox_start(struct apple_mbox *mbox) { u32 control; @@ -610,6 +997,12 @@ int apple_mbox_start(struct apple_mbox *mbox) mbox->regs + mbox->hw->irq_enable); } + if (apple_mbox_has_rx_poller(mbox)) { + WRITE_ONCE(mbox->active, true); + apple_mbox_v8_rx_start(mbox); + goto out_unlock; + } + enable_irq(mbox->irq_recv_not_empty); mbox->active = true; @@ -624,8 +1017,13 @@ static void apple_mbox_stop_locked(struct apple_mbox *mbox) if (!mbox->active) return; - mbox->active = false; - disable_irq(mbox->irq_recv_not_empty); + if (apple_mbox_has_rx_poller(mbox)) { + WRITE_ONCE(mbox->active, false); + apple_mbox_v8_rx_stop(mbox); + } else { + mbox->active = false; + disable_irq(mbox->irq_recv_not_empty); + } pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } @@ -682,13 +1080,19 @@ static void apple_mbox_quiesce(void *data) * inversion. The removal fence prevents any matching restart. */ was_active = mbox->active; - mbox->active = false; + WRITE_ONCE(mbox->active, false); mutex_unlock(&mbox->lifecycle_lock); if (was_active) { - disable_irq(mbox->irq_recv_not_empty); + if (apple_mbox_has_rx_poller(mbox)) + apple_mbox_v8_rx_stop(mbox); + else + disable_irq(mbox->irq_recv_not_empty); pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } + /* The v8 poller also stops on the removal fence; make sure it is gone. */ + if (apple_mbox_has_rx_poller(mbox)) + cancel_delayed_work_sync(&apple_mbox_to_priv(mbox)->rx_poll_work); /* Serialize once-only masking with the IRQ and send-error paths. */ spin_lock_irqsave(&mbox->tx_lock, flags); @@ -764,6 +1168,121 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name) } EXPORT_SYMBOL(apple_mbox_get_byname); +static void apple_mbox_iounmap(void *addr) +{ + iounmap((__force void __iomem *)addr); +} + +/* + * Validate and map one T8152 GPU ASCWrap v8 provider. Nothing here reads or + * writes device registers: the four windows are only mapped. The wrapper, + * iop-vbar and post-office window are requested exclusively; the PSD block + * is mapped without a request so that a GPU power owner may also map the + * same service block (the provider itself only ever reads it). + */ +static int apple_mbox_probe_ascwrap_v8(struct platform_device *pdev, + struct apple_mbox *mbox) +{ + struct device *dev = &pdev->dev; + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + struct resource *wrapper, *iop_vbar, *window, *psd; + u64 wrapper_base, iop_vbar_base, window_base, psd_base; + const char *firmware_role; + int ret; + + wrapper = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wrapper"); + iop_vbar = platform_get_resource_byname(pdev, IORESOURCE_MEM, "iop-vbar"); + window = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mailbox"); + psd = platform_get_resource_byname(pdev, IORESOURCE_MEM, "psd"); + if (!wrapper || !iop_vbar || !window || !psd || + resource_size(wrapper) != APPLE_ASCWRAP_V8_WRAPPER_SIZE || + resource_size(iop_vbar) != APPLE_ASCWRAP_V8_IOP_VBAR_SIZE || + resource_size(window) != APPLE_ASCWRAP_V8_MBOX_SIZE || + resource_size(psd) != APPLE_ASCWRAP_V8_PSD_SIZE) + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v8 provider resources\n"); + + ret = of_property_read_string(dev->of_node, "apple,firmware-role", + &firmware_role); + if (ret) + return dev_err_probe(dev, ret, + "missing ASCWrap v8 firmware role\n"); + + if (!strcmp(firmware_role, "GFX")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; + wrapper_base = APPLE_T8152_GFX_WRAPPER_BASE; + iop_vbar_base = APPLE_T8152_GFX_IOP_VBAR_BASE; + window_base = APPLE_T8152_GFX_MBOX_BASE; + psd_base = APPLE_T8152_GFX_PSD_BASE; + } else if (!strcmp(firmware_role, "GFX1")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; + wrapper_base = APPLE_T8152_GFX1_WRAPPER_BASE; + iop_vbar_base = APPLE_T8152_GFX1_IOP_VBAR_BASE; + window_base = APPLE_T8152_GFX1_MBOX_BASE; + psd_base = APPLE_T8152_GFX1_PSD_BASE; + } else { + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v8 firmware role\n"); + } + + /* + * Fixed per-SoC CPU-physical windows. In particular the post-office + * window must be exactly this role's 0x80 bytes: the rest of the bank + * belongs to PMP, DCP, DCPEXT and SCALER0. + */ + if (wrapper->start != wrapper_base || iop_vbar->start != iop_vbar_base || + window->start != window_base || psd->start != psd_base) + return dev_err_probe(dev, -EINVAL, + "%s ASCWrap v8 resources do not match the T8152 profile\n", + firmware_role); + + mbox->ascwrap_v6.wrapper_start = wrapper->start; + mbox->ascwrap_v6.wrapper_size = resource_size(wrapper); + mbox->ascwrap_v6.iop_vbar_start = iop_vbar->start; + mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar); + mbox->ascwrap_v6.missing = ascwrap_v8_cpu_control ? + 0 : APPLE_ASCWRAP_V8_MISSING_CPU_CONTROL_APPROVAL; + priv->rx_poll_only = !ascwrap_v8_rx_irq; + INIT_DELAYED_WORK(&priv->rx_poll_work, apple_mbox_v8_rx_poll_work); + + mbox->wrapper_regs = devm_platform_ioremap_resource_byname(pdev, "wrapper"); + if (IS_ERR(mbox->wrapper_regs)) + return PTR_ERR(mbox->wrapper_regs); + + mbox->iop_vbar_regs = + devm_platform_ioremap_resource_byname(pdev, "iop-vbar"); + if (IS_ERR(mbox->iop_vbar_regs)) + return PTR_ERR(mbox->iop_vbar_regs); + + mbox->regs = devm_platform_ioremap_resource_byname(pdev, "mailbox"); + if (IS_ERR(mbox->regs)) + return PTR_ERR(mbox->regs); + + /* + * Same posted/non-posted choice devm_ioremap_resource() makes, but + * without claiming the region. Released after apple_mbox_quiesce(), + * which is registered later and may still read the PSD state. + */ + if (psd->flags & IORESOURCE_MEM_NONPOSTED) + priv->psd_regs = ioremap_np(psd->start, resource_size(psd)); + else + priv->psd_regs = ioremap(psd->start, resource_size(psd)); + if (!priv->psd_regs) + return -ENOMEM; + ret = devm_add_action_or_reset(dev, apple_mbox_iounmap, + (__force void *)priv->psd_regs); + if (ret) + return ret; + + dev_info(dev, + "%s ASCWrap v8 provider: wrapper %pR, post-office window %pR, PSD %pR; CPU control %s; receive %s\n", + firmware_role, wrapper, window, psd, + ascwrap_v8_cpu_control ? "allowed" : "refused (default)", + ascwrap_v8_rx_irq ? "recv-not-empty IRQ" : + "polled, no IRQ enabled (default)"); + return 0; +} + static int apple_mbox_probe(struct platform_device *pdev) { int ret; @@ -771,19 +1290,23 @@ static int apple_mbox_probe(struct platform_device *pdev) const char *firmware_role; const struct apple_mbox_ascwrap_v6_role_desc *role; unsigned int i; + struct apple_mbox_priv *priv; struct apple_mbox *mbox; struct device *dev = &pdev->dev; struct resource *res, *iop_vbar_res; - mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); - if (!mbox) + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) return -ENOMEM; + mbox = &priv->mbox; mbox->dev = &pdev->dev; mbox->hold_rx_in_sleep = of_property_read_bool(dev->of_node, "apple,hold-rx-in-sleep"); mbox->hw = of_device_get_match_data(dev); if (!mbox->hw) return -EINVAL; + if (mbox->hw->counted_fifo && !IS_ENABLED(CONFIG_ARM64)) + return -EOPNOTSUPP; if (mbox->hw->is_ascwrap_v6) { /* @@ -825,7 +1348,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->ascwrap_v6.wrapper_size = resource_size(res); mbox->ascwrap_v6.iop_vbar_start = iop_vbar_res->start; mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar_res); - /* T8140 and T6040 implement both the CPU start and the CPU stop. */ + /* T8140, T8142 and T6040 implement both the CPU start and stop. */ mbox->ascwrap_v6.missing = 0; mbox->wrapper_regs = @@ -840,6 +1363,10 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->regs = (u8 __iomem *)mbox->wrapper_regs + mbox->hw->reg_offset; + } else if (mbox->hw->is_ascwrap_v8) { + ret = apple_mbox_probe_ascwrap_v8(pdev, mbox); + if (ret) + return ret; } else { res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res || resource_size(res) < mbox->hw->reg_offset + @@ -978,6 +1505,15 @@ static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t8140_ascwrap_v6_ APPLE_T8140_GFX1_WRAPPER_BASE, APPLE_T8140_GFX1_IOP_VBAR_BASE }, }; +static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t8142_ascwrap_v6_roles[] = { + { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_T8140_GFX_WRAPPER_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA, + APPLE_T8140_GFX_IOP_VBAR_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA }, + { "GFX1", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, + APPLE_T8140_GFX1_WRAPPER_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA, + APPLE_T8140_GFX1_IOP_VBAR_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA }, +}; + /* The T6040 runs a single GPU firmware instance. */ static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6040_ascwrap_v6_roles[] = { { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, @@ -1004,6 +1540,27 @@ static const struct apple_mbox_hw apple_mbox_t8140_ascwrap_v6_hw = { .has_irq_controls = false, }; +/* T8142 (M5): T8140's two GPU firmware CPUs at the 0x5... bank. */ +static const struct apple_mbox_hw apple_mbox_t8142_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, + .ascwrap_v6_roles = apple_mbox_t8142_ascwrap_v6_roles, + .num_ascwrap_v6_roles = ARRAY_SIZE(apple_mbox_t8142_ascwrap_v6_roles), + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_t6040_ascwrap_v6_hw = { .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, .is_ascwrap_v6 = true, @@ -1058,6 +1615,26 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { .irq_bit_send_empty = APPLE_M3_MBOX_IRQ_A2I_EMPTY, }; +/* + * T8152/J873 GPU GFX/GFX1 ASCWrap v8: a counted FIFO at the role's + * post-office window (reg "mailbox"), plus wrapper lifecycle reads gated on + * the role's PSD service. + */ +static const struct apple_mbox_hw apple_mbox_t8152_gfx_ascwrap_v8_hw = { + .is_ascwrap_v8 = true, + .counted_fifo = true, + .control_full = GENMASK(11, 8), /* free TX slots */ + .control_empty = GENMASK(3, 0), /* RX occupancy */ + .a2i_control = 0x20, + .a2i_send0 = 0x10, + .a2i_send1 = 0x18, + .i2a_control = 0x20, + .i2a_recv0 = 0x00, + .i2a_recv1 = 0x08, + .has_irq_controls = false, +}; + +static_assert(0x20 + sizeof(u32) <= APPLE_ASCWRAP_V8_MBOX_SIZE); /* T6050's native ASCWrap v6 uses the same RUN/stop sequence. Aura grants * these complete wrapper/VBAR windows in the retained GPU aperture. */ @@ -1112,11 +1689,31 @@ static const struct apple_mbox_hw apple_mbox_t6050_apif_ascwrap_v6_hw = { .has_irq_controls = false, }; +/* + * T8152 SMC post-office mailbox: counted FIFOs share the status register. + * Interrupt masking is provided by the interrupt controller, not the mailbox. + */ +static const struct apple_mbox_hw apple_mbox_t8152_smc_hw = { + .counted_fifo = true, + .control_full = GENMASK(11, 8), /* free TX slots */ + .control_empty = GENMASK(3, 0), /* RX occupancy */ + .a2i_control = 0x20, + .a2i_send0 = 0x10, + .a2i_send1 = 0x18, + .i2a_control = 0x20, + .i2a_recv0 = 0x00, + .i2a_recv1 = 0x08, + .has_irq_controls = false, +}; + static const struct of_device_id apple_mbox_of_match[] = { + { .compatible = "apple,t8152-smc-mailbox", .data = &apple_mbox_t8152_smc_hw }, { .compatible = "apple,t6031-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8122-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, + { .compatible = "apple,t8152-gfx-ascwrap-v8", .data = &apple_mbox_t8152_gfx_ascwrap_v8_hw }, { .compatible = "apple,t6040-ascwrap-v6", .data = &apple_mbox_t6040_ascwrap_v6_hw }, { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_t8140_ascwrap_v6_hw }, + { .compatible = "apple,t8142-ascwrap-v6", .data = &apple_mbox_t8142_ascwrap_v6_hw }, { .compatible = "apple,t6050-apif-ascwrap-v6", .data = &apple_mbox_t6050_apif_ascwrap_v6_hw }, { .compatible = "apple,t6050-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, { .compatible = "apple,t6050-x1n1-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index c239fae558cee..2212509efd82c 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -692,6 +692,15 @@ const HW_CFG_T6041: PmpHwConfig = PmpHwConfig { coreanalytics: Ep7::Always, }; +/// T8142: the DART window starts at 1 TiB as on T8140. Its registry-v2 +/// firmware starts endpoint 7 and takes its 8 KiB buffer before it +/// acknowledges the IOP power state. +const HW_CFG_T8142: PmpHwConfig = PmpHwConfig { + dma_mask: Some(DmaMask::new::<42>()), + check_bootargs_uuid: false, + coreanalytics: Ep7::Always, +}; + const HW_CFG_T8132: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, @@ -711,6 +720,7 @@ kernel::of_device_table!( [ (of::DeviceId::new(c"apple,t8132-pmp-v2"), &HW_CFG_T8132), (of::DeviceId::new(c"apple,t8140-pmp-v2"), &HW_CFG_T8140), + (of::DeviceId::new(c"apple,t8142-pmp-v2"), &HW_CFG_T8142), (of::DeviceId::new(c"apple,t6041-pmp-v2"), &HW_CFG_T6041), (of::DeviceId::new(c"apple,t6000-pmp-v2"), &HW_CFG_DEFAULT), ] diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index d352083aaa26c..536ba515f826f 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -153,6 +153,18 @@ static bool apple_rtkit_needs_coreanalytics(struct apple_rtkit *rtk) return rtk->ops->flags & APPLE_RTKIT_NEEDS_COREANALYTICS; } +static bool apple_rtkit_firmware_owns_crashlog(struct apple_rtkit *rtk) +{ + return rtk->ops->flags & APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG; +} + +/* The crashlog endpoint is started while the endpoint map is handled. */ +static bool apple_rtkit_early_crashlog(struct apple_rtkit *rtk) +{ + return apple_rtkit_coproc_places_buffers(rtk) || + (rtk->ops->flags & APPLE_RTKIT_EARLY_CRASHLOG); +} + static int apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type, u64 msg) { @@ -232,10 +244,11 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) } /* - * For co-processors that place their own buffers, the crashlog - * endpoint is started before the map is acknowledged. + * Co-processors that place all buffers, and clients that explicitly + * request early crashlog startup, need the endpoint before the map is + * acknowledged: G17P can request its crash DVA right after. */ - if (apple_rtkit_coproc_places_buffers(rtk) && base == 0 && + if (apple_rtkit_early_crashlog(rtk) && base == 0 && test_bit(APPLE_RTKIT_EP_CRASHLOG, &bitmap)) { ret = apple_rtkit_start_ep(rtk, APPLE_RTKIT_EP_CRASHLOG); if (ret) { @@ -276,7 +289,7 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_EP_TRACEKIT: /* already started while handling the map */ if (ep == APPLE_RTKIT_EP_CRASHLOG && - apple_rtkit_coproc_places_buffers(rtk)) + apple_rtkit_early_crashlog(rtk)) break; dev_dbg(rtk->dev, "RTKit: Starting system endpoint 0x%02x\n", ep); @@ -410,6 +423,13 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n", request.size, &request.iova); + if ((rtk->ops->flags & APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG) && + ep == APPLE_RTKIT_EP_CRASHLOG && request.iova) { + *buffer = request; + buffer->is_mapped = true; + return apple_rtkit_send_message(rtk, ep, msg, NULL, false); + } + if (request.iova && !rtk->ops->shmem_setup && !rtk->ops->shmem_setup_ep) { err = -EINVAL; goto error; @@ -440,7 +460,9 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, buffer->iova); } apple_rtkit_send_message(rtk, ep, reply, NULL, false); - } else if (apple_rtkit_coproc_places_buffers(rtk)) { + } else if (apple_rtkit_coproc_places_buffers(rtk) || + (apple_rtkit_firmware_owns_crashlog(rtk) && + ep == APPLE_RTKIT_EP_CRASHLOG)) { /* acknowledge the co-processor's own buffer with its request */ apple_rtkit_send_message(rtk, ep, msg, NULL, false); } @@ -734,6 +756,18 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) dev_err(rtk->dev, "RTKit: co-processor has crashed\n"); + /* A firmware-owned crash buffer has no CPU mapping to read the log from. */ + if (!rtk->crashlog_buffer.buffer && !rtk->crashlog_buffer.iomem) { + if (apple_rtkit_firmware_owns_crashlog(rtk)) + dev_err(rtk->dev, + "RTKit: firmware-owned crash log at DVA %pad (0x%zx bytes)\n", + &rtk->crashlog_buffer.iova, rtk->crashlog_buffer.size); + rtk->crashed = true; + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); + return; + } + /* * create a shadow copy here to make sure the co-processor isn't able * to change the log while we're dumping it. this also ensures diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 65bd28a6f0240..35508001fbea4 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -57,6 +57,16 @@ struct apple_rtkit_inherited_buffer { /* This client requires endpoint 7 setup before IOP power can be acknowledged. */ #define APPLE_RTKIT_NEEDS_COREANALYTICS BIT(1) +/* + * The co-processor keeps its crash buffer in memory it maps itself (the T8142 + * AOP) and asks for it with that IOVA: take the buffer as the firmware's, + * without a CPU mapping, and acknowledge the request unchanged. + */ +#define APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG BIT(2) + +/* Start the crashlog endpoint before acknowledging its endpoint map. */ +#define APPLE_RTKIT_EARLY_CRASHLOG BIT(3) + /* * Struct to represent implementation-specific RTKit operations. * diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 5641384b51bfe..55092da885071 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -209,7 +209,8 @@ struct drm_asahi_params_global { * * G13-G15 on 14.x report DRM_ASAHI_GPU_HAL_LEGACY. * The exact J613 25G83 runtime and the G17P runtime report - * DRM_ASAHI_GPU_HAL_200. + * DRM_ASAHI_GPU_HAL_200. M5 (T8142) and G18 (M6) report + * DRM_ASAHI_GPU_HAL_300. */ __u32 gpu_hal_generation; @@ -241,7 +242,7 @@ enum drm_asahi_gpu_hal_generation { */ DRM_ASAHI_GPU_HAL_200 = 200, - /** @DRM_ASAHI_GPU_HAL_300: HAL300 command stream. Never reported by this kernel. */ + /** @DRM_ASAHI_GPU_HAL_300: HAL300 command stream, reported by M5 (T8142) and G18 (M6). */ DRM_ASAHI_GPU_HAL_300 = 300, }; @@ -422,8 +423,33 @@ enum drm_asahi_feature { * include the earlier points of that timeline without adding GPU waits. */ DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, + + /* + * Bit 30 is used by the M5 runtime's userspace. Bits 31 and 32 belong + * to the G18 (M6, T8152) backend, which alone reports them. Bit 32 is + * DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES, defined after this enum so + * that the enum keeps its 32-bit type. + */ + + /** + * @DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH: G18: render commands accept + * DRM_ASAHI_RENDER_G18_RESOURCES. The kernel sizes the shared TA/3D + * private-memory pool from the struct drm_asahi_g18_render_resources + * the command carries and programs both launch pairs; userspace emits + * nonzero SPECTPR scratch frames only for such renders. + */ + DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH = (1ULL) << 31, }; +/* + * DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES: G18: single-sample, + * single-layer renders with 32x32 utiles also accept 40-, 56- and 72-byte + * samples (the hardware's native layouts), besides tiles of at most 32 KiB + * and 48..128-byte samples in steps of 16. A feature bit like those of enum + * drm_asahi_feature; a 64-bit enumerator would widen that enum's type. + */ +#define DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES ((1ULL) << 32) + #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 #define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 @@ -1252,6 +1278,18 @@ enum drm_asahi_render_flags { * may depend on it. Requires DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS. */ DRM_ASAHI_RENDER_FRAGMENT_BARRIERS = (1U << 20), + + /* + * Bits 21 to 24 are used by the M5 runtime's userspace. + */ + + /** + * @DRM_ASAHI_RENDER_G18_RESOURCES: G18 only; requires + * DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH. The payload is a complete + * struct drm_asahi_cmd_render_g18: the render command followed by a + * struct drm_asahi_g18_render_resources. Any bytes after it are zero. + */ + DRM_ASAHI_RENDER_G18_RESOURCES = (1U << 25), }; /** @@ -1738,6 +1776,139 @@ enum drm_asahi_compute_flags { * ordered helpers cannot time out while waiting for the build. */ DRM_ASAHI_COMPUTE_AS_BUILD = (1L << 2), + + /* + * Bits 16 and 17 are used by the M5 runtime's userspace. + */ + + /** + * @DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE: G18 only. The command's + * dispatch binds texture or image state (a texture table, a sampled + * texture, a storage image), so the kernel selects the compute mode + * the native driver uses for such commands. Other backends refuse it. + */ + DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE = (1L << 18), + + /** + * @DRM_ASAHI_COMPUTE_G18_RESOURCES: G18 only. The payload is a complete + * struct drm_asahi_cmd_compute_g18: the compute command, with + * @scs_layout zero, followed by a struct drm_asahi_g18_compute_resources. + * Any bytes after it are zero. + */ + DRM_ASAHI_COMPUTE_G18_RESOURCES = (1L << 19), +}; + +/** + * struct drm_asahi_g18_render_resources - G18 per-render program demand + * + * Carried by render commands with DRM_ASAHI_RENDER_G18_RESOURCES, after the + * complete struct drm_asahi_cmd_render (see struct drm_asahi_cmd_render_g18). + * + * Effective (linked) program demand in compiler units, maximized over every + * draw of the render: 16-bit registers, the static thread-private (TPS) + * frame in bytes as programmed in SPECTPR (a multiple of 16), and IPR/staging + * halfwords. Tile programs are background, end-of-tile, partial background + * and partial end-of-tile; a zero register count marks an absent program. + * Tile programs must not use scratch. At least one of the vertex and + * fragment frames is nonzero. Tile storage is taken from the render command. + */ +struct drm_asahi_g18_render_resources { + /** @version: Must be 1. */ + __u32 version; + /** @flags: Must be 3 (vertex and fragment programs present). */ + __u32 flags; + /** @vertex_registers_16: Vertex program registers, in 16-bit units. */ + __u32 vertex_registers_16; + /** @vertex_scratch_bytes: Vertex SPECTPR frame bytes. */ + __u32 vertex_scratch_bytes; + /** @vertex_staging_16: Vertex staging halfwords. */ + __u32 vertex_staging_16; + /** @fragment_registers_16: Fragment program registers, in 16-bit units. */ + __u32 fragment_registers_16; + /** @fragment_scratch_bytes: Fragment SPECTPR frame bytes. */ + __u32 fragment_scratch_bytes; + /** @fragment_staging_16: Fragment staging halfwords. */ + __u32 fragment_staging_16; + /** @tile_registers_16: Tile program registers, in 16-bit units. */ + __u32 tile_registers_16[4]; + /** @tile_staging_16: Tile program staging halfwords. */ + __u32 tile_staging_16[4]; + /** @reserved: MBZ. */ + __u32 reserved[4]; +}; + +/** + * struct drm_asahi_g18_compute_resources - G18 per-dispatch metadata + * + * Carried by compute commands with DRM_ASAHI_COMPUTE_G18_RESOURCES, after the + * complete struct drm_asahi_cmd_compute (see struct drm_asahi_cmd_compute_g18). + * + * Counts are compiler halfwords or bytes as named; the kernel converts them + * to native units. Version 1 scales @local_bytes + @variable_shared_bytes by + * four for shaders using byte offsets in element-index fields; version 2 uses + * the actual byte footprints. @flags: bit 0 main staging known, bit 1 + * constant program present, bit 2 constant staging known, bit 3 local + * geometry unknown (@local_size must then be zero). Other flag bits and + * @reserved must be zero. + */ +struct drm_asahi_g18_compute_resources { + /** @version: 1 or 2. */ + __u32 version; + /** @flags: See above. */ + __u32 flags; + /** @registers_16: Main program registers, in 16-bit units. */ + __u32 registers_16; + /** @scratch_bytes: Main program scratch bytes. */ + __u32 scratch_bytes; + /** @spectpr_scratch_bytes: SPECTPR frame bytes. */ + __u32 spectpr_scratch_bytes; + /** @staging_16: Main program staging halfwords. */ + __u32 staging_16; + /** @constant_registers_16: Constant program registers, in 16-bit units. */ + __u32 constant_registers_16; + /** @constant_scratch_bytes: Constant program scratch bytes. */ + __u32 constant_scratch_bytes; + /** @constant_staging_16: Constant program staging halfwords. */ + __u32 constant_staging_16; + /** @local_bytes: Threadgroup memory bytes. */ + __u32 local_bytes; + /** @imageblock_stride: Imageblock stride. */ + __u32 imageblock_stride; + /** @variable_shared_bytes: Variable shared memory bytes. */ + __u32 variable_shared_bytes; + /** @local_size: Workgroup size, or zero when unknown. */ + __u32 local_size[3]; + /** @reserved: MBZ. */ + __u32 reserved; +}; + +/** + * struct drm_asahi_cmd_render_g18 - G18 render payload with program demand + * + * The payload of a render command with DRM_ASAHI_RENDER_G18_RESOURCES. Every + * field of @legacy keeps its meaning; fields whose feature G18 does not + * report (timestamp copies, ray queries) are zero. + */ +struct drm_asahi_cmd_render_g18 { + /** @legacy: The render command. */ + struct drm_asahi_cmd_render legacy; + /** @resources: Program demand of the render. */ + struct drm_asahi_g18_render_resources resources; +}; + +/** + * struct drm_asahi_cmd_compute_g18 - G18 compute payload with dispatch metadata + * + * The payload of a compute command with DRM_ASAHI_COMPUTE_G18_RESOURCES. + * Every field of @base keeps its meaning; fields whose feature G18 does not + * report (timestamp copies, the auxiliary allocation, intersection tables) + * are zero, and so is @base.scs_layout. + */ +struct drm_asahi_cmd_compute_g18 { + /** @base: The compute command. */ + struct drm_asahi_cmd_compute base; + /** @resources: Metadata of the command's dispatches. */ + struct drm_asahi_g18_compute_resources resources; }; /** diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 56f9f84689804..b7d6e19cc22ad 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -158,6 +158,8 @@ const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; const u32 RUST_CONST_HELPER_APPLE_RTKIT_NEEDS_COREANALYTICS = APPLE_RTKIT_NEEDS_COREANALYTICS; +const u32 RUST_CONST_HELPER_APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG = APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG; +const u32 RUST_CONST_HELPER_APPLE_RTKIT_EARLY_CRASHLOG = APPLE_RTKIT_EARLY_CRASHLOG; const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; diff --git a/rust/kernel/drm/mm.rs b/rust/kernel/drm/mm.rs index a01ed8b64c5a0..a182a5d608b85 100644 --- a/rust/kernel/drm/mm.rs +++ b/rust/kernel/drm/mm.rs @@ -7,7 +7,7 @@ use crate::{ alloc::flags::*, bindings, - error::{code::EINVAL, to_result, Result}, + error::{code::{EINVAL, EOVERFLOW}, to_result, Result}, sync::{new_mutex, Arc, Mutex, UniqueArc}, types::Opaque, }; @@ -48,9 +48,9 @@ pub struct NodeData, T> { inner: T, } -// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. +// SAFETY: Allocator ops take the mutex; node mutation requires exclusive pinned access. unsafe impl, T: Send> Send for NodeData {} -// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. +// SAFETY: Allocator ops take the mutex; node mutation requires exclusive pinned access. unsafe impl, T: Sync> Sync for NodeData {} /// Available MM node insertion modes @@ -139,6 +139,95 @@ impl, T> Deref for NodeData { } } +impl NodeData<(), T> { + /// Split a trailing reservation while retaining ownership of every address. + /// + /// This is restricted to allocators without callbacks. The caller decides + /// whether its payload can be split; no payload or page-table state is + /// copied. Allocation precedes mutation, and both new reservations become + /// visible under the same allocator mutex. + pub fn split_tail(self: Pin<&mut Self>, size: u64, inner: T) -> Result> { + if !self.valid || size == 0 || size >= self.node.size { + return Err(EINVAL); + } + let mut tail = KBox::new( + NodeData { + // SAFETY: The native node starts empty before reservation. + node: unsafe { core::mem::zeroed() }, + mm: self.mm.clone(), valid: false, _pin: PhantomPinned, inner, + }, + GFP_KERNEL, + )?; + // SAFETY: Neither pinned node nor its payload moves. Only allocator + // bookkeeping fields are changed while holding the allocator mutex. + let front = unsafe { self.get_unchecked_mut() }; + let guard = front.mm.lock(); + // SAFETY: The allocator is initialized and protected by guard. + if unsafe { (*guard.0.get()).color_adjust.is_some() } { return Err(EINVAL); } + let old_size = front.node.size; + tail.node.start = front.node.start + old_size - size; + tail.node.size = size; + tail.node.color = front.node.color; + // SAFETY: Both node allocations remain at fixed addresses. The + // smaller front and adjacent tail partition the original reservation. + unsafe { + bindings::drm_mm_remove_node(&mut front.node); + front.node.size = old_size - size; + let first = bindings::drm_mm_reserve_node(guard.0.get(), &mut front.node); + if first != 0 { + front.node.size = old_size; + assert_eq!(bindings::drm_mm_reserve_node(guard.0.get(), &mut front.node), 0); + return to_result(first).map(|_| unreachable!()); + } + let second = bindings::drm_mm_reserve_node(guard.0.get(), &mut tail.node); + if second != 0 { + bindings::drm_mm_remove_node(&mut front.node); + front.node.size = old_size; + assert_eq!(bindings::drm_mm_reserve_node(guard.0.get(), &mut front.node), 0); + return to_result(second).map(|_| unreachable!()); + } + } + tail.valid = true; + Ok(Pin::from(tail)) + } + + /// Rejoin an adjacent tail without allocation. On success the tail is + /// detached and may only be dropped; its payload is not destroyed here. + /// Errors leave both reservations unchanged. + pub fn rejoin_tail(self: Pin<&mut Self>, tail: Pin<&mut Self>) -> Result { + if !self.valid || !tail.valid || !core::ptr::eq(&*self.mm, &*tail.mm) + || self.node.start.checked_add(self.node.size) != Some(tail.node.start) + || self.node.color != tail.node.color { + return Err(EINVAL); + } + let combined = self.node.size.checked_add(tail.node.size).ok_or(EOVERFLOW)?; + // SAFETY: Exclusive pinned access permits field mutation, not moving + // the linked native nodes. The mutex excludes every other MM operation. + let front = unsafe { self.get_unchecked_mut() }; + let tail = unsafe { tail.get_unchecked_mut() }; + let guard = front.mm.lock(); + // SAFETY: The allocator remains initialized and locked. + if unsafe { (*guard.0.get()).color_adjust.is_some() } { return Err(EINVAL); } + let old_size = front.node.size; + // SAFETY: The two adjacent reservations cover exactly combined bytes. + // No other node can enter the temporary hole while the mutex is held. + unsafe { + bindings::drm_mm_remove_node(&mut tail.node); + bindings::drm_mm_remove_node(&mut front.node); + front.node.size = combined; + let result = bindings::drm_mm_reserve_node(guard.0.get(), &mut front.node); + if result != 0 { + front.node.size = old_size; + assert_eq!(bindings::drm_mm_reserve_node(guard.0.get(), &mut front.node), 0); + assert_eq!(bindings::drm_mm_reserve_node(guard.0.get(), &mut tail.node), 0); + return to_result(result); + } + } + tail.valid = false; + Ok(()) + } +} + impl, T> Drop for NodeData { fn drop(&mut self) { if self.valid { diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 73896fea53e00..ac9d7ab5032b6 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -245,6 +245,16 @@ impl RtKit { ..Self::VTABLE }; + const FIRMWARE_CRASHLOG_VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + flags: Self::VTABLE.flags | bindings::APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG, + ..Self::VTABLE + }; + + const G17P_VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + flags: Self::FIRMWARE_CRASHLOG_VTABLE.flags | bindings::APPLE_RTKIT_EARLY_CRASHLOG, + ..Self::FIRMWARE_CRASHLOG_VTABLE + }; + /// Creates a new RTKit client for a given device and optional mailbox name or index. pub fn new( dev: &device::Device, @@ -268,6 +278,30 @@ impl RtKit { Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::COREANALYTICS_VTABLE) } + /// Creates a client whose co-processor keeps its crash buffer in memory it + /// maps itself and asks for it with that IOVA (the T8142 AOP). + pub fn new_with_firmware_crashlog( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ) -> Result { + Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::FIRMWARE_CRASHLOG_VTABLE) + } + + /// Creates a G17P/G18G client whose crash DVA belongs to the GPU UAT. + /// + /// The crashlog endpoint starts before its map is acknowledged. The + /// firmware-owned buffer is acknowledged without mapping it on the CPU. + pub fn new_g17p( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ) -> Result { + Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::G17P_VTABLE) + } + fn new_with_ops( dev: &device::Device, mbox_name: Option<&'static CStr>, diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py index 961e86f7d824b..7cd520208b22d 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py @@ -604,7 +604,9 @@ def powerlog_framing(root): #define __le64 u64 #define EPIC_SUBTYPE_STD_SERVICE 0xc0 enum epic_subtype { REPORT = 0xc0 }; -struct apple_dcp {void *dev;int fw_compat;struct {int iomfb_method_profile;} hw;struct {u32 nits;} brightness;}; +struct work_struct { unsigned scheduled; }; +static void __attribute__((unused)) schedule_work(struct work_struct *w) { w->scheduled++; } +struct apple_dcp {void *dev;int fw_compat;struct {int iomfb_method_profile;} hw;struct {u32 nits,maximum;void *bl_dev;} brightness;struct work_struct bl_register_wq;}; struct apple_epic_service; struct apple_dcp_afkep {struct apple_dcp *dcp;unsigned endpoint;struct apple_epic_service *service;}; struct apple_epic_service_ops {int (*report)(struct apple_epic_service *,enum epic_subtype,const void *,size_t);}; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index 9b949a1a6ff1d..5caff361a38ca 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -62,6 +62,7 @@ def function(file,signature): #define DRM_CONNECTOR_REGISTERED 1 #define DPAVSERV_ENDPOINT_V2 0x2a #define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define DCP_IOMFB_METHODS_H17G 7 #define EPIC_SUBTYPE_STD_SERVICE 0xc0 #define EPIC_SUBTYPE_ANNOUNCE_H17P 0x11 #define to_platform_device(x) (x) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 2d7667dfc61e0..f42f8251fee56 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -741,6 +741,7 @@ def poweroff_replug(root): """An old disconnect's CRTC-off must not release a replacement link.""" poweroff = apple_function(root, 'dcp_poweroff') assert poweroff, 'dcp_poweroff' + panel_predicate = apple_function(root, 'dcp_h17g_panel') or '' iomfb = (root / 'drivers/gpu/drm/apple/iomfb.c').read_text() modeset = block(iomfb, iomfb.index('int dcp_crtc_atomic_modeset(')) assert 'dcp->crtc_unplugs = dcp->typec_unplugs;' in modeset @@ -764,7 +765,12 @@ def poweroff_replug(root): struct completion { int done; }; struct apple_connector { bool connected; }; struct dptx_port { bool enabled, connected, available; void *service; }; +#define DCP_FIRMWARE_H17P 1 +#define DCP_IOMFB_METHODS_H17G 7 +#define DRM_MODE_CONNECTOR_eDP 14 struct apple_dcp { + int fw_compat, connector_type; + struct { int iomfb_method_profile; } hw; void *dev, *avep, *hdmi_hpd; int external_lock, hpd_mutex; struct delayed_work typec_reconnect_wq, placeholder_edid_wq, swap_watchdog_wq; @@ -786,6 +792,9 @@ def poweroff_replug(root): { (void)d; (void)generation; return 0; } static void dcp_external_crtc_released(struct apple_dcp *d) { (void)d; } static bool dcp_uses_soft_dpms(struct apple_dcp *d) { return d->soft_dpms; } +static bool __attribute__((unused)) dcp_has_panel(struct apple_dcp *d) { return !d->typec_output; } +static void __attribute__((unused)) iomfb_blank_h17p(struct apple_dcp *d) +{ (void)d; assert(!"unexpected internal-panel blank in external-link test"); } static int dcp_backlight_dpms(struct apple_dcp *d, bool on) { (void)d; (void)on; return 0; } static void av_service_disconnect(struct apple_dcp *d) @@ -802,7 +811,7 @@ def poweroff_replug(root): { (void)d; (void)why; (void)error; (void)delay; retries++; } static bool gpiod_get_value_cansleep(void *gpio) { (void)gpio; return false; } static void disconnected_hpd_event(struct apple_connector *connector) { (void)connector; } -''' + poweroff + r''' +''' + panel_predicate + '\n' + poweroff + r''' static struct apple_dcp dcp; static struct apple_connector connector; static struct platform_device pdev = { .data = &dcp }; diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py index 124314ac83a5d..8c194aa176726 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py @@ -20,7 +20,7 @@ def body(source,marker): pre=r''' #![allow(dead_code,unused_variables)] use std::{cell::RefCell,rc::Rc,mem::{ManuallyDrop,size_of},ops::{Deref,DerefMut,Range},num::NonZeroUsize,sync::{Mutex as StdMutex,atomic::{AtomicBool as StdAtomicBool,AtomicU8,AtomicU32,AtomicU64,AtomicUsize,Ordering,fence}}}; -type Result = std::result::Result;const EIO:i32=5;const ETIMEDOUT:i32=110;const EOVERFLOW:i32=75; +type Result = std::result::Result;const EIO:i32=5;const ETIMEDOUT:i32=110;const EOVERFLOW:i32=75;const ENOTSUPP:i32=524; static TRIP_LOAD:AtomicUsize=AtomicUsize::new(0); struct AtomicBool(StdAtomicBool);impl AtomicBool{fn new(v:bool)->Self{Self(StdAtomicBool::new(v))}fn store(&self,v:bool,o:Ordering){self.0.store(v,o);}fn load(&self,o:Ordering)->bool{let n=TRIP_LOAD.load(Ordering::Relaxed);if n!=0&&TRIP_LOAD.fetch_sub(1,Ordering::Relaxed)==1{self.0.store(true,Ordering::Release);}self.0.load(o)}} const UAT_NUM_CTX:usize=64;const UAT_USER_CTX_START:usize=1;const TTBR_VALID:u64=1;const TTBR_ASID_SHIFT:usize=48;const UAT_PGSZ:usize=0x4000; @@ -34,6 +34,8 @@ def body(source,marker): #[derive(Clone,Copy,PartialEq,Eq)]enum UatFirmware{Handoff,PublishedRoots} struct Handoff{lock_ap:AtomicU8,lock_fw:AtomicU8,turn:AtomicU32,cur_slot:AtomicU32}impl Handoff{fn new()->Self{Self{lock_ap:AtomicU8::new(0),lock_fw:AtomicU8::new(0),turn:AtomicU32::new(0),cur_slot:AtomicU32::new(0xffff)}} ''' +# Keep the production firmware predicate in the legacy-firmware harness. +pre=pre.replace('struct Handoff{', 'impl UatFirmware{'+body(mm,'const fn is_no_peer(')+'}\nstruct Handoff{', 1) handoff='\n'.join(body(mm,m) for m in ['fn lock(&self) {\n self.lock_ap', 'fn unlock(&self) {\n self.turn', 'fn current_slot(']) mid=r''' }