diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 0118e3afa27af..843d7c4af8f9a 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -17,6 +17,7 @@ properties: - apple,agx-g13s - apple,agx-g14g - apple,agx-g14s + - apple,agx-t6040 - apple,agx-t8132 - apple,agx-t8140 - items: @@ -51,8 +52,12 @@ properties: - description: Region containing a shared handoff structure for VM management coordination - - description: Power calibration blob or firmware shared-L2 page - - description: GPU calibration blob or preserved firmware text + - description: + Power calibration blob, firmware shared-L2 page or firmware data + segment + - description: + GPU calibration blob, preserved firmware text or firmware + kernel-window page tables - description: Shared global variables or preserved firmware data memory-region-names: @@ -61,8 +66,8 @@ properties: - const: ttbs - const: pagetables - const: handoff - - enum: [hw-cal-a, shared-l2] - - enum: [hw-cal-b, fw-text] + - enum: [hw-cal-a, shared-l2, fw-data] + - enum: [hw-cal-b, fw-text, l2-pagetables] - enum: [globals, fw-data] apple,pmp: @@ -71,6 +76,26 @@ properties: operating-points-v2: true + apple,firmware-version: + $ref: /schemas/types.yaml#/definitions/uint32-array + maxItems: 3 + description: + macOS version (major, minor, patch) whose boot firmware loaded the GPU + firmware, which identifies the firmware build. Filled in by the boot + loader. + + apple,cs-opp: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Operating-point table of the GPU's CS clock domain, one voltage cell per + state. + + apple,afr-opp: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Operating-point table of the GPU's AFR clock domain, one voltage cell + per state. + apple,firmware-abi: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 @@ -121,20 +146,45 @@ allOf: else: properties: apple,pmp: false + - if: + properties: + compatible: + contains: + const: apple,agx-t8140 + then: + properties: + reg: + items: + - description: GFX coprocessor control registers + - description: GPU block MMIO registers + reg-names: + items: + - const: asc + - const: sgx + mboxes: + items: + - description: GFX coprocessor mailbox + - description: GFX1 coprocessor mailbox + memory-region: + items: + - description: Region containing GPU MMU TTBs + - description: Region containing GPU MMU page tables + - description: + Region containing a shared handoff structure for VM + management coordination memory-region-names: - minItems: 3 items: - const: ttbs - const: pagetables - const: handoff - - const: hw-cal-a - - const: hw-cal-b - - const: globals + required: + - power-domains + - operating-points-v2 - if: properties: compatible: contains: - const: apple,agx-t8140 + const: apple,agx-t6040 then: properties: reg: @@ -148,39 +198,64 @@ allOf: mboxes: items: - description: GFX coprocessor mailbox - - description: GFX1 coprocessor mailbox memory-region: - maxItems: 3 + items: + - description: Region containing GPU MMU TTBs + - description: Region containing GPU MMU page tables + - description: + Region containing a shared handoff structure for VM + management coordination + - description: + Data segment of the loaded GPU firmware, where the firmware + places its crash log + - description: + Region in which the GPU firmware builds the page tables of + its kernel window memory-region-names: - maxItems: 3 + items: + - const: ttbs + - const: pagetables + - const: handoff + - const: fw-data + - const: l2-pagetables required: - power-domains - operating-points-v2 - else: - if: + - apple,cs-opp + - apple,afr-opp + - if: + not: properties: compatible: - not: - contains: - const: apple,agx-t8132 - then: - properties: - reg: - items: - - description: GPU coprocessor control registers - - description: GPU block MMIO registers - reg-names: - items: - - const: asc - - const: sgx - mboxes: - maxItems: 1 - memory-region: - minItems: 6 - memory-region-names: - minItems: 6 - required: - - apple,firmware-abi + contains: + enum: + - apple,agx-t6040 + - apple,agx-t8132 + - apple,agx-t8140 + then: + properties: + reg: + items: + - description: GPU coprocessor control registers + - description: GPU block MMIO registers + reg-names: + items: + - const: asc + - const: sgx + mboxes: + maxItems: 1 + memory-region: + minItems: 6 + memory-region-names: + items: + - const: ttbs + - const: pagetables + - const: handoff + - const: hw-cal-a + - const: hw-cal-b + - const: globals + required: + - apple,firmware-abi additionalProperties: false diff --git a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml index 4798b62bd8430..09b3b45b4ed92 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -4,20 +4,23 @@ $id: http://devicetree.org/schemas/mailbox/apple,ascwrap-v6.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple T8140 ASCWrap v6 mailbox +title: Apple ASCWrap v6 GPU mailbox maintainers: - Ryan Murray description: - Mailbox interface of the T8140 GPU coprocessors. The wrapper resource - contains the coprocessor control registers and mailbox FIFOs. A separate - resource provides the coprocessor vector-base register. The firmware role - identifies which of the two GPU firmware instances uses the mailbox. + Mailbox interface of the GPU coprocessors of the T8140 and T6040. The + wrapper resource contains the coprocessor control registers and mailbox + FIFOs. A separate resource provides the coprocessor vector-base register. + The firmware role identifies which GPU firmware instance uses the mailbox; + the T6040 runs a single one. properties: compatible: - const: apple,t8140-ascwrap-v6 + enum: + - apple,t6040-ascwrap-v6 + - apple,t8140-ascwrap-v6 reg: items: @@ -57,6 +60,17 @@ required: - '#mbox-cells' - apple,firmware-role +allOf: + - if: + properties: + compatible: + contains: + const: apple,t6040-ascwrap-v6 + then: + properties: + apple,firmware-role: + const: GFX + additionalProperties: false examples: diff --git a/arch/arm64/boot/dts/apple/t6040-j616s.dts b/arch/arm64/boot/dts/apple/t6040-j616s.dts index d24ff77472e6c..76bc9e1adb406 100644 --- a/arch/arm64/boot/dts/apple/t6040-j616s.dts +++ b/arch/arm64/boot/dts/apple/t6040-j616s.dts @@ -26,6 +26,8 @@ /* used by the loader to hand over the display */ dcp = &dcp; disp0 = &display; + /* the boot loader finds the GPU node through this alias */ + gpu = &gpu; }; chosen { @@ -71,6 +73,40 @@ isp_heap: isp-heap { iommu-addresses = <&isp 0x100 0x0 0x0 0x2000000>; }; + + /* + * Data segment of the loaded GPU firmware (ADT sgx gfx-data-*); it + * places its crash log here. + */ + gpu_fw_data: gpu-fw-data@10001d44000 { + reg = <0x100 0x01d44000 0x0 0x138000>; + no-map; + }; + + /* + * The UAT ranges (ADT sgx gpu-region-*, gfx-shared-region-*, + * gfx-shared-l2-region-*, gfx-handoff-*). They stay no-map until + * the boot loader adds them to /memory. + */ + uat_ttbs: uat-ttbs@10bfffb8000 { + reg = <0x10b 0xfffb8000 0x0 0x4000>; + no-map; + }; + + uat_pagetables: uat-pagetables@10bfff78000 { + reg = <0x10b 0xfff78000 0x0 0x40000>; + no-map; + }; + + uat_l2_pagetables: uat-l2-pagetables@10bfff74000 { + reg = <0x10b 0xfff74000 0x0 0x4000>; + no-map; + }; + + uat_handoff: uat-handoff@10bfff70000 { + reg = <0x10b 0xfff70000 0x0 0x4000>; + no-map; + }; }; memory@0 { @@ -90,6 +126,158 @@ }; }; + /* + * GPU performance states, one opp-microvolt cell per cluster (ADT + * /arm-io/sgx perf-states and perf-states1). The operating points are + * binned per device: these are this unit's values until the boot + * loader fills them in. Entries keep the order of the GPU performance + * states, which is not sorted by frequency. + */ + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <125000 125000>; + }; + opp01 { + opp-hz = /bits/ 64 <338000000>; + opp-microvolt = <605000 585000>; + }; + opp02 { + opp-hz = /bits/ 64 <618000000>; + opp-microvolt = <655000 640000>; + }; + opp03 { + opp-hz = /bits/ 64 <796000000>; + opp-microvolt = <690000 690000>; + }; + opp04 { + opp-hz = /bits/ 64 <924000000>; + opp-microvolt = <715000 725000>; + }; + opp05 { + opp-hz = /bits/ 64 <952000000>; + opp-microvolt = <755000 770000>; + }; + opp06 { + opp-hz = /bits/ 64 <1056000000>; + opp-microvolt = <755000 770000>; + }; + opp07 { + opp-hz = /bits/ 64 <1062000000>; + opp-microvolt = <795000 810000>; + }; + opp08 { + opp-hz = /bits/ 64 <1182000000>; + opp-microvolt = <795000 810000>; + }; + opp09 { + opp-hz = /bits/ 64 <1182000000>; + opp-microvolt = <845000 860000>; + }; + opp10 { + opp-hz = /bits/ 64 <1312000000>; + opp-microvolt = <845000 860000>; + }; + opp11 { + opp-hz = /bits/ 64 <1242000000>; + opp-microvolt = <900000 915000>; + }; + opp12 { + opp-hz = /bits/ 64 <1380000000>; + opp-microvolt = <900000 915000>; + }; + opp13 { + opp-hz = /bits/ 64 <1326000000>; + opp-microvolt = <945000 960000>; + }; + opp14 { + opp-hz = /bits/ 64 <1470000000>; + opp-microvolt = <945000 960000>; + }; + opp15 { + opp-hz = /bits/ 64 <1578000000>; + opp-microvolt = <1030000 1045000>; + }; + }; + + /* The GPU's CS clock domain (ADT cs-perf-states) */ + gpu_cs_opp: opp-table-gpucs { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <24000000>; + opp-microvolt = <555000>; + }; + opp01 { + opp-hz = /bits/ 64 <338000000>; + opp-microvolt = <555000>; + }; + opp02 { + opp-hz = /bits/ 64 <618000000>; + opp-microvolt = <620000>; + }; + opp03 { + opp-hz = /bits/ 64 <796000000>; + opp-microvolt = <660000>; + }; + opp04 { + opp-hz = /bits/ 64 <992000000>; + opp-microvolt = <715000>; + }; + opp05 { + opp-hz = /bits/ 64 <1188000000>; + opp-microvolt = <780000>; + }; + opp06 { + opp-hz = /bits/ 64 <1380000000>; + opp-microvolt = <870000>; + }; + opp07 { + opp-hz = /bits/ 64 <1470000000>; + opp-microvolt = <910000>; + }; + }; + + /* The GPU's AFR clock domain (ADT afr-perf-states) */ + gpu_afr_opp: opp-table-gpuafr { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <24000000>; + opp-microvolt = <555000>; + }; + opp01 { + opp-hz = /bits/ 64 <338000000>; + opp-microvolt = <555000>; + }; + opp02 { + opp-hz = /bits/ 64 <618000000>; + opp-microvolt = <620000>; + }; + opp03 { + opp-hz = /bits/ 64 <796000000>; + opp-microvolt = <660000>; + }; + opp04 { + opp-hz = /bits/ 64 <992000000>; + opp-microvolt = <715000>; + }; + opp05 { + opp-hz = /bits/ 64 <1188000000>; + opp-microvolt = <780000>; + }; + opp06 { + opp-hz = /bits/ 64 <1380000000>; + opp-microvolt = <870000>; + }; + opp07 { + opp-hz = /bits/ 64 <1470000000>; + opp-microvolt = <910000>; + }; + }; + /* * CPU die temperature: SMC key TCMz (CPU die max), sensor 0 of * smc_hwmon. The PMP's fast die-temperature limit regulates (TCMz @@ -619,6 +807,26 @@ status = "okay"; }; +&gfx_mbox { + status = "okay"; +}; + +&gpu { + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, + <&gpu_fw_data>, <&uat_l2_pagetables>; + memory-region-names = "ttbs", "pagetables", "handoff", "fw-data", + "l2-pagetables"; + operating-points-v2 = <&gpu_opp>; + apple,cs-opp = <&gpu_cs_opp>; + apple,afr-opp = <&gpu_afr_opp>; + status = "okay"; +}; + +/* The GPU firmware runs from boot on (ps_gfx is always on): the PMP must see the GPU on. */ +&pmp_gfx { + apple,always-on; +}; + /* * The Type-C ports (ADT nub-spmi-a0 hpm0/hpm1, nub-spmi-a1 hpm2). The ADT * lists the event, sleep and wake interrupts; select is event + 2. diff --git a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi index 2e150ab28b6df..2d3ae512144bc 100644 --- a/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t6040-pmgr.dtsi @@ -7,6 +7,28 @@ * Copyright The Asahi Linux Contributors */ +&pmgr_gfx { + ps_afr: power-controller@100 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "afr"; + apple,always-on; + }; + + /* The GPU firmware keeps running and does not survive this domain going off. */ + ps_gfx: power-controller@110 { + compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + power-domains = <&ps_afr>; + apple,always-on; + }; +}; + &pmgr { ps_ispsens0: power-controller@0 { compatible = "apple,t6041-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; diff --git a/arch/arm64/boot/dts/apple/t6040.dtsi b/arch/arm64/boot/dts/apple/t6040.dtsi index 8c729672a3aac..e02d0edbd2320 100644 --- a/arch/arm64/boot/dts/apple/t6040.dtsi +++ b/arch/arm64/boot/dts/apple/t6040.dtsi @@ -609,6 +609,40 @@ power-domains = <&ps_avd_sys>; }; + /* Named after its sgx range: the coprocessor range is the mailbox node's too. */ + gpu: gpu@288000000 { + compatible = "apple,agx-t6040"; + reg = <0x2 0x8a600000 0x0 0x88000>, + <0x2 0x88000000 0x0 0x3758000>; + reg-names = "asc", "sgx"; + mboxes = <&gfx_mbox>; + power-domains = <&ps_gfx>; + status = "disabled"; + }; + + pmgr_gfx: power-management@288e80000 { + compatible = "apple,t6041-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x88e80000 0x0 0x4000>; + /* child nodes are added in t6040-pmgr.dtsi */ + }; + + gfx_mbox: mailbox@28a600000 { + compatible = "apple,t6040-ascwrap-v6"; + reg = <0x2 0x8a600000 0x0 0x88000>, + <0x2 0x8a050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupt-parent = <&aic>; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; + disp0_dart: iommu@402300000 { compatible = "apple,t6041-dart", "apple,t8110-dart"; reg = <0x4 0x02300000 0x0 0xc000>; @@ -869,8 +903,9 @@ * The GPU (soc-device 13). Without it the running PMP treats * the GPU as off and holds it at its lowest state, but claiming * it while nothing powers the GPU crashes the PMP firmware - * (SError) right after it starts. The GPU node takes it as a - * power domain, so it is on only while the GPU driver runs. + * (SError) right after it starts. It sits on the GPU's own + * power state, and a board that powers the GPU (ps_gfx stays on + * while the GPU firmware runs) claims it always-on. */ pmp_gfx: report@d { compatible = "apple,t6041-pmp-v2-report-entry", @@ -878,6 +913,7 @@ reg = <0xd>; label = "pmp-gfx"; #power-domain-cells = <0>; + power-domains = <&ps_gfx>; }; }; diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index d0dfc840265da..c26fe1671f7cf 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -146,6 +146,10 @@ kernel::of_device_table!( of::DeviceId::new(c"apple,agx-t8140"), ProbeConfig::G17(&hw::t8140::CONFIG) ), + ( + of::DeviceId::new(c"apple,agx-t6040"), + ProbeConfig::G17(&hw::t6040::CONFIG) + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) diff --git a/drivers/gpu/drm/asahi/g16/uat.rs b/drivers/gpu/drm/asahi/g16/uat.rs index 132d7efea6883..54104712c294c 100644 --- a/drivers/gpu/drm/asahi/g16/uat.rs +++ b/drivers/gpu/drm/asahi/g16/uat.rs @@ -74,7 +74,7 @@ fn with_table( // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on // unmap, and these VMs are retained until reboot). - unsafe { pgtable::with_root_entries(pa, count, f) } + unsafe { pgtable::with_root_entries(None, pa, count, f) } } /// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index c4b07c603fb82..8f3a03fbb8dba 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -556,6 +556,8 @@ pub(crate) struct Manager { buffer_id: u8, generation: u64, shrink: Shrink, + /// A tiling pass initialized this buffer manager (25G83 InitBM published). + initialized: bool, } /// Native tiler-heap shrink policy, matching the heap behaviour seen @@ -657,8 +659,18 @@ impl Manager { buffer_id: args.buffer_id, generation: args.generation, shrink: Shrink::default(), + initialized: false, }) } + + /// Whether a published tiling pass initialized this manager (25G83 InitBM). + pub(crate) fn initialized(&self) -> bool { + self.initialized + } + + pub(crate) fn mark_initialized(&mut self) { + self.initialized = true; + } pub(crate) fn buffer_id(&self) -> u8 { self.buffer_id } diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs index ed0f3c71a4c93..bfa64c0b27a6a 100644 --- a/drivers/gpu/drm/asahi/g17/buffer/target.rs +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -6,7 +6,7 @@ use super::compact_alias; use crate::g17::{ - fw::render::RenderPass, + fw::render::{RegisterPrograms, RenderPass}, object::{CpuMap, KernelObject}, }; use crate::{ @@ -25,6 +25,12 @@ fn aligned(value: usize, alignment: usize) -> Result { .ok_or(EOVERFLOW) } +/// The 25G83 tiler parameter block (the 25G83 Metal driver's default parameter-buffer sizes): +/// the TPC and seven sub-buffers the tiling registers name. Sizes are +/// [TPC, tile map × 64, 0x4000 for layered targets, 0, 0xc00, 0xa00, 0x600, 0x80]. +const GTP_ALIGNMENTS: [usize; 8] = [0x40, 0x40, 0x100, 1, 0x20, 0x20, 0x80, 0x40]; +const GTP_FIXED_SIZES: [usize; 4] = [0xc00, 0xa00, 0x600, 0x80]; + /// Per-command geometry, independent of the capacity already allocated. #[derive(Copy, Clone, Debug)] pub(crate) struct Layout { @@ -33,9 +39,18 @@ pub(crate) struct Layout { pub(crate) used: usize, pub(crate) tpc_size: usize, pub(crate) tvb_blocks: usize, + /// Offsets of the tiler parameter sub-buffers in the TPC object (25G83), `None` for an + /// empty one; the TPC is entry 0. + gtp: [Option; 8], + /// Bytes of the TPC object the command uses. + tpc_end: usize, } impl Layout { - pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { + pub(crate) fn new( + pass: &RenderPass, + clusters: u32, + programs: RegisterPrograms, + ) -> Result { let width = pass.width as usize; let height = pass.height as usize; let layers = pass.layers as usize; @@ -90,16 +105,41 @@ impl Layout { ); } tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); + let mut gtp = [None; 8]; + let mut tpc_end = tpc_size; + if programs == RegisterPrograms::V25G83 { + let sizes = [ + tpc_size, + tilemap_size.checked_mul(64).ok_or(EOVERFLOW)?, + if layers > 1 { 0x4000 } else { 0 }, + 0, + GTP_FIXED_SIZES[0], + GTP_FIXED_SIZES[1], + GTP_FIXED_SIZES[2], + GTP_FIXED_SIZES[3], + ]; + let mut position = 0; + for ((offset, size), alignment) in gtp.iter_mut().zip(sizes).zip(GTP_ALIGNMENTS) { + position = aligned(position, alignment)?; + if size != 0 { + *offset = Some(position); + } + position = position.checked_add(size).ok_or(EOVERFLOW)?; + } + tpc_end = position; + } Ok(Self { layer_offset, heap_offset, used, tpc_size, tvb_blocks, + gtp, + tpc_end, }) } pub(crate) fn tpc_capacity(&self) -> Result { - Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) + Ok(aligned(self.tpc_end, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) } pub(crate) fn record_stride(&self) -> Result { Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) @@ -169,6 +209,21 @@ impl Target { self.tpc.address } + /// GPU addresses of the tiler parameter sub-buffers of `layout` (25G83), zero for an empty + /// or absent one. The TPC object holds them after the TPC. + pub(crate) fn gtp_va(&self, layout: &Layout) -> Result<[u64; 8]> { + if layout.tpc_end > self.tpc.backing.size() { + return Err(ERANGE); + } + let mut addresses = [0; 8]; + for (address, offset) in addresses.iter_mut().zip(layout.gtp) { + if let Some(offset) = offset { + *address = self.tpc.address.checked_add(offset as u64).ok_or(EOVERFLOW)?; + } + } + Ok(addresses) + } + /// Only the span consumed by this render is cleared. A small render does /// not clear a previous large target's complete retained record capacity. pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 26785a326c6f0..c65f7de6ac915 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -351,6 +351,7 @@ impl Payload { pub(crate) fn validate( &self, window: &UscWindow, + variant: &super::fw::Variant, space: &impl AddressSpace, ) -> Result { match self { @@ -424,7 +425,7 @@ impl Payload { scissor_va: cmd.isp_scissor_base, depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, - few_primitives: cmd.flags & FEW_PRIMITIVES != 0, + few_primitives: cmd.flags & FEW_PRIMITIVES != 0 && variant.few_primitives, fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, @@ -457,6 +458,11 @@ impl Payload { rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; for stage in ray_query { + if (stage.intersection.enabled() || stage.scratch.enabled()) + && !variant.ray_traversal + { + return Err(refuse_ray_traversal()); + } if stage.intersection.enabled() { require_range(space, stage.intersection.pool, stage.intersection.pool_bytes, Access::Read)?; let code = stage.intersection.default_program & 0x0000_ffff_ffff_ffff; @@ -489,6 +495,10 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; + let scratch = ScratchRequest::new(cmd.scs_layout)?; + if (scratch.enabled() || intersection.enabled()) && !variant.ray_traversal { + return Err(refuse_ray_traversal()); + } if intersection.enabled() { require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; let code = intersection.default_program & 0x0000_ffff_ffff_ffff; @@ -497,7 +507,7 @@ impl Payload { Ok(Validated::Compute { long_job: super::timeout::select(intersection.enabled(), [false; 2], as_build), intersection: *intersection, - scratch: ScratchRequest::new(cmd.scs_layout)?, + scratch, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, @@ -509,3 +519,9 @@ impl Payload { } } } + +/// Ray-traversal work on a firmware build that does not run it ([`super::fw::Variant`]). +fn refuse_ray_traversal() -> Error { + kernel::pr_warn_once!("G17: ray-traversal work refused on this firmware\n"); + Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32)) +} diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs index 9e4e0253b7c81..9508a827be4b1 100644 --- a/drivers/gpu/drm/asahi/g17/event.rs +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -11,6 +11,8 @@ use kernel::prelude::*; /// The bounded shared-ring transport; recognized malformed entries return errors after consume. pub(super) trait Transport { + /// The firmware instances of this GPU: the primary, and the secondary where one exists. + fn roles(&self) -> &'static [Role]; fn trace(&mut self, role: Role); fn statistics(&mut self, role: Role); fn drain( @@ -81,10 +83,10 @@ pub(super) fn service(host: &mut H) -> Result { return Ok(Pending::default()); } host.observe_render_releases()?; - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.trace(role); } - for role in [Role::Primary, Role::Secondary] { + for &role in host.roles() { host.statistics(role); } let mut masks = [0; 2]; @@ -110,7 +112,7 @@ pub(super) fn service(host: &mut H) -> Result { return Err(EIO); } } - if !host.control_backpressured() { + if !host.control_backpressured() && host.roles().contains(&Role::Secondary) { batch(host, Role::Secondary, &mut masks)?; } host.retain_render_completions(masks); diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 3564c3d776a51..717e3d25a4962 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -152,13 +152,52 @@ const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; /// Offsets in the preempt buffer of the four state words the program names. const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; -/// Offsets in the operand-state buffer of the regions the program names. -const OPERAND_STATE_10229: u64 = 0x12800; -const OPERAND_STATE_140A8: u64 = 0x13000; -const OPERAND_STATE_10099: u64 = 0x9405; -const OPERAND_STATE_10091: u64 = 0x12400; -const OPERAND_STATE_0A5C1: u64 = 0x0005; -const OPERAND_STATE_0A5C9: u64 = 0x9000; +/// Layout of a queue's operand-state buffer, which depends on the GPU's core layout: offsets of +/// the regions the program names (two preemption banks of three 0x800-byte records per core, +/// tagged | 5 at their start, then their end, then the per-cluster tails) and its size. +pub(crate) struct OperandState { + r10229: u64, + r140a8: u64, + r10099: u64, + r10091: u64, + r0a5c1: u64, + r0a5c9: u64, + /// Size of the buffer. + pub(crate) size: usize, +} + +impl OperandState { + /// One cluster (the A18 Pro's six cores). + const ONE_BY_SIX: OperandState = OperandState { + r10229: 0x12800, + r140a8: 0x13000, + r10099: 0x9405, + r10091: 0x12400, + r0a5c1: 0x0005, + r0a5c9: 0x9000, + size: 0x14000, + }; + /// Two clusters of ten cores (M4 Pro): bank stride 0x1e400, tails 0x1000. + const TWO_BY_TEN: OperandState = OperandState { + r10229: 0x3c800, + r140a8: 0x3d800, + r10099: 0x1e405, + r10091: 0x3c400, + r0a5c1: 0x0005, + r0a5c9: 0x1e000, + size: 0x44000, + }; + + /// The layout for `clusters` clusters with `cores` enabled cores in all. + pub(crate) fn new(clusters: u32, cores: u32) -> Result<&'static OperandState> { + // shortcut: only the two measured layouts; add others when a GPU with them is supported. + match (clusters, cores) { + (1, _) => Ok(&Self::ONE_BY_SIX), + (2, 20) => Ok(&Self::TWO_BY_TEN), + _ => Err(ENODEV), + } + } +} /// Checkpoints into the main register array, as entry counts: after the /// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full @@ -283,6 +322,8 @@ pub(crate) struct ComputeArgs { pub(crate) preempt_va: u64, /// GPU address of the operand-state buffer, as seen by the work's VM. pub(crate) operand_state_va: u64, + /// Layout of the operand-state buffer. + pub(crate) operand: &'static OperandState, /// GPU address of the private-memory usage record, as seen by the work's /// VM. pub(crate) usage_va: u64, @@ -305,6 +346,7 @@ pub(crate) struct ComputeArgs { fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { let preempt = args.preempt_va; let operand = args.operand_state_va; + let state = args.operand; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); let intersection = args.intersection.words(); @@ -349,12 +391,12 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x10428, key), RegisterWrite::new(0x14028, u64::from(args.free_list_slot)), RegisterWrite::new(0x14070, args.usage_va | 1), - RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), - RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x10229, offset_va(operand, state.r10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, state.r140a8)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r0a5c9)?), RegisterWrite::new(0x1a440, UNK_1A440_FINAL), RegisterWrite::new(0x0a599, UNK_0A599), ]) @@ -364,11 +406,12 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS /// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { let operand = args.operand_state_va; + let state = args.operand; Ok([ - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), + RegisterWrite::new(0x10099, offset_va(operand, state.r0a5c1)?), + RegisterWrite::new(0x10091, offset_va(operand, state.r0a5c9)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, state.r10099)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, state.r10091)?), ]) } diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 2b61ecc118320..3092e599caa84 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -27,6 +27,8 @@ use kernel::prelude::*; use super::clear; +pub(crate) mod v25g83; + /// Interface version at root `+0x00`; the firmware rejects any other. pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; @@ -219,6 +221,21 @@ impl SecondaryRoot { config_va, } } + + /// Returns the root of a single instance that carries both roles: the primary root, with + /// status B, extended by the power-state and configuration pointers. + pub(crate) fn merged( + args: &RootArgs, + status_b_va: u64, + power_state_va: u64, + config_va: u64, + ) -> Self { + Self { + root: Root::new(Role::Primary, args, status_b_va), + power_state_va, + config_va, + } + } } /// Offset of the secondary root from the primary root. Both live in one @@ -591,31 +608,42 @@ pub(crate) struct RegisterWindow { pub(crate) phys: u64, /// GPU address of the window in the firmware MMIO aperture. pub(crate) va: u64, - /// Size in bytes; need not be a multiple of the page size. + /// Size in bytes of one instance; need not be a multiple of the page size. pub(crate) size: u32, /// Value of [`RegisterMapEntry::unk_18`]. pub(crate) unk_18: u64, /// Value of [`RegisterMapEntry::unk_20`]. pub(crate) unk_20: u32, + /// Number of instances of the block. The firmware finds instance `n` at + /// `va + n * size`; it lives at `phys + n * phys_stride`. + pub(crate) instances: u32, + /// Distance between the physical addresses of consecutive instances. + pub(crate) phys_stride: u64, } impl RegisterWindow { - /// Returns the page-aligned physical address, GPU address and length the - /// window must be mapped with, or `None` if its physical and GPU - /// addresses differ within a page or the mapped span overflows. - pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { + /// Returns the page-aligned physical address, GPU address and length that + /// instance `instance` of the window must be mapped with, or `None` if its + /// physical and GPU addresses differ within a page or the mapped span + /// overflows. + pub(crate) fn mapping(&self, instance: u32) -> Option<(u64, u64, u64)> { let mask = PAGE_SIZE as u64 - 1; - if self.phys & mask != self.va & mask { + let phys = self + .phys + .checked_add(self.phys_stride.checked_mul(u64::from(instance))?)?; + let va = self + .va + .checked_add(u64::from(self.size).checked_mul(u64::from(instance))?)?; + if phys & mask != va & mask { return None; } - let start = self.va & !mask; - let span = self - .va + let start = va & !mask; + let span = va .checked_add(u64::from(self.size))? .checked_sub(start)? .checked_add(mask)? & !mask; - Some((self.phys & !mask, start, span)) + Some((phys & !mask, start, span)) } } @@ -629,9 +657,9 @@ pub(crate) struct RegisterMapEntry { pub(crate) phys: u64, /// GPU address of the window. pub(crate) va: u64, - /// Window size in bytes. + /// Size in bytes of all instances of the window. pub(crate) total_size: u32, - /// Window size in bytes, again. + /// Size in bytes of one instance. pub(crate) element_size: u32, /// Per-window value, see [`RegisterWindow::unk_18`]. pub(crate) unk_18: u64, @@ -650,7 +678,7 @@ impl RegisterMapEntry { Self { phys: window.phys, va: window.va, - total_size: window.size, + total_size: window.size * window.instances, element_size: window.size, unk_18: window.unk_18, unk_20: window.unk_20, @@ -676,14 +704,19 @@ pub(crate) const PERF_STATES: usize = 11; /// Voltage lanes of a performance table. const PERF_LANES: usize = 16; +/// Most voltage cells of an operating point: one per GPU cluster, at most one per lane. +pub(crate) const OPP_CELLS: usize = PERF_LANES; + /// One GPU operating point, from the device tree, in performance-state /// order. #[derive(Copy, Clone, Debug)] pub(crate) struct Opp { /// Frequency in Hz. pub(crate) freq_hz: u64, - /// GPU core voltage in µV. - pub(crate) volt_uv: u32, + /// GPU core voltage in µV: one cell for the whole GPU, or one per cluster. + pub(crate) volt_uv: [u32; OPP_CELLS], + /// Number of valid cells in [`Self::volt_uv`]. + pub(crate) cells: usize, } /// Per-SoC parts of the performance tables that the operating points do not @@ -728,20 +761,23 @@ static_assert!(size_of::() == 0x704); #[derive(Copy, Clone)] struct PerfState { freq_mhz: u32, - core_mv: u32, - sram_mv: u32, + core_mv: [u32; OPP_CELLS], + sram_mv: [u32; OPP_CELLS], + cells: usize, } impl PerfGroup { - /// Fills the table. `lanes` is the number of voltage lanes holding each - /// state's voltages; the remaining lanes stay zero. + /// Fills the table. With one voltage cell, `lanes` is the number of voltage lanes + /// holding each state's voltages; with one cell per cluster, lane `n` holds cluster + /// `n`'s. The remaining lanes stay zero. fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { self.unk_0 = 10; for (index, state) in states.iter().enumerate() { self.freq_mhz[index] = state.freq_mhz; - for lane in 0..lanes { - self.core_mv[index][lane] = state.core_mv; - self.sram_mv[index][lane] = state.sram_mv; + for lane in 0..if state.cells == 1 { lanes } else { state.cells } { + let cell = if state.cells == 1 { 0 } else { lane }; + self.core_mv[index][lane] = state.core_mv[cell]; + self.sram_mv[index][lane] = state.sram_mv[cell]; } } } @@ -755,16 +791,18 @@ fn perf_states( ) -> Result<[PerfState; PERF_STATES]> { let mut states = [PerfState { freq_mhz: 0, - core_mv: 0, - sram_mv: 0, + core_mv: [0; OPP_CELLS], + sram_mv: [0; OPP_CELLS], + cells: 0, }; PERF_STATES]; for (state, &index) in states.iter_mut().zip(map) { let opp = opps.get(index as usize).ok_or(EINVAL)?; - let core_mv = opp.volt_uv / 1000; + let core_mv = opp.volt_uv.map(|uv| uv / 1000); *state = PerfState { freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, core_mv, - sram_mv: core_mv.max(sram_min_mv), + sram_mv: core_mv.map(|mv| mv.max(sram_min_mv)), + cells: opp.cells, }; } Ok(states) @@ -1169,6 +1207,35 @@ pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; /// scheduling pass. pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; +/// The values of region C that differ between SoCs and firmware builds. +pub(crate) struct RegionCConfig { + /// A18 Pro: 0xffff. + pub(crate) unk_54: u16, + /// A18 Pro: 1 in every entry. + pub(crate) unk_78: [u32; 3], + /// Highest GPU power, in mW (A18 Pro: 9550). + pub(crate) max_power_mw: u32, + /// A18 Pro: 0x3000_001d. + pub(crate) unk_9c: u32, + /// A18 Pro: 0x3100_0002. + pub(crate) unk_a0: u32, + /// A18 Pro: 0x7d0 (2000). + pub(crate) unk_7c4: u32, + /// A18 Pro: zero. + pub(crate) unk_9e0: [u32; 8], +} + +/// Region C of the A18 Pro firmware (26A428). +pub(crate) const REGION_C_26A428: RegionCConfig = RegionCConfig { + unk_54: 0xffff, + unk_78: [1; 3], + max_power_mw: 9550, + unk_9c: 0x3000_001d, + unk_a0: 0x3100_0002, + unk_7c4: 0x7d0, + unk_9e0: [0; 8], +}; + /// Region C: firmware policy settings shared by both instances. /// /// Three fields are live. The host updates [`Self::effort`] while it submits @@ -1197,7 +1264,7 @@ pub(crate) struct RegionC { /// Required: 0x78 (120). pub(crate) unk_3c: u32, pub(crate) unk_40: [u8; 0x14], - /// Required: 0xffff. + /// See [`RegionCConfig::unk_54`]. pub(crate) unk_54: u16, /// Required: 0x28 (40). pub(crate) unk_56: u16, @@ -1213,10 +1280,10 @@ pub(crate) struct RegionC { /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). pub(crate) context_switch_timer_multiplier: u8, pub(crate) unk_6b: [u8; 0xd], - /// Required: 1 in every entry. + /// See [`RegionCConfig::unk_78`]. pub(crate) unk_78: [u32; 3], - /// Required: 0x254e (9550). - pub(crate) unk_84: u32, + /// Highest GPU power, in mW. + pub(crate) max_power_mw: u32, /// Required: 0x3e8 (1000). pub(crate) unk_88: u32, /// Required: 0x3e8 (1000). @@ -1224,9 +1291,9 @@ pub(crate) struct RegionC { pub(crate) unk_90: [u8; 8], /// Required: 0x3e8 (1000). pub(crate) unk_98: u32, - /// Required: 0x3000_001d. + /// See [`RegionCConfig::unk_9c`]. pub(crate) unk_9c: u32, - /// Required: 0x3100_0002. + /// See [`RegionCConfig::unk_a0`]. pub(crate) unk_a0: u32, pub(crate) unk_a4: [u8; 0x40], /// Performance-floor request (live, see the type documentation). @@ -1234,7 +1301,7 @@ pub(crate) struct RegionC { /// Live; the host stores zero here whenever it stores [`Self::effort`]. pub(crate) unk_e8: u32, pub(crate) unk_ec: [u8; 0x6d8], - /// Required: 0x7d0 (2000). + /// See [`RegionCConfig::unk_7c4`]. pub(crate) unk_7c4: u32, /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). pub(crate) unk_7c8: u32, @@ -1285,7 +1352,10 @@ pub(crate) struct RegionC { pub(crate) compute_watchdog: u32, /// Required: 1. pub(crate) unk_9d0: u32, - pub(crate) unk_9d4: [u8; 0x449], + pub(crate) unk_9d4: [u8; 0xc], + /// See [`RegionCConfig::unk_9e0`]. + pub(crate) unk_9e0: [u32; 8], + pub(crate) unk_a00: [u8; 0x41d], /// Required: 1. pub(crate) unk_e1d: u32, pub(crate) unk_e21: [u8; 0x18], @@ -1319,28 +1389,28 @@ impl RegionC { pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); /// Initializes region C. - pub(crate) fn init(&mut self) { + pub(crate) fn init(&mut self, cfg: &RegionCConfig) { clear(self); self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; self.unk_24 = 0xbb8; self.idle_power_off = 1; self.unk_34 = 1; self.unk_3c = 0x78; - self.unk_54 = 0xffff; + self.unk_54 = cfg.unk_54; self.unk_56 = 0x28; self.unk_58 = 0xffff; self.unk_5e = 1; self.unk_62 = 1; self.unk_66 = 3; self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; - self.unk_78 = [1; 3]; - self.unk_84 = 0x254e; + self.unk_78 = cfg.unk_78; + self.max_power_mw = cfg.max_power_mw; self.unk_88 = 0x3e8; self.unk_8c = 0x3e8; self.unk_98 = 0x3e8; - self.unk_9c = 0x3000_001d; - self.unk_a0 = 0x3100_0002; - self.unk_7c4 = 0x7d0; + self.unk_9c = cfg.unk_9c; + self.unk_a0 = cfg.unk_a0; + self.unk_7c4 = cfg.unk_7c4; self.unk_7c8 = 0x3f80_0000; self.unk_7cc = 0x3f4c_cccd; self.unk_7d0 = 0x3e4c_cccd; @@ -1364,6 +1434,7 @@ impl RegionC { self.unk_9c8 = 0x28; self.compute_watchdog = 50; self.unk_9d0 = 1; + self.unk_9e0 = cfg.unk_9e0; self.unk_e1d = 1; self.unk_e39 = 3; self.unk_e3d = 1; @@ -1411,7 +1482,7 @@ impl StatusBlock { /// Size of the firmware-control page named by the primary status B. pub(crate) const FWCTL_SIZE: usize = 0x4000; /// Offset of the firmware-control ring in its page. -const FWCTL_RING: u64 = 0x40; +pub(crate) const FWCTL_RING: u64 = 0x40; /// One firmware-written recovery-info entry. The host reads each word separately after the /// recovery handshake reaches state 3; it never copies the live table as a structure. diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs new file mode 100644 index 0000000000000..c237896a49fb1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/initdata/v25g83.rs @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Initialization data of the 25G83 firmware build (macOS 26.6, `mBoot-18000.161.10`). +//! +//! The M4 Pro (T6040) runs this build. It has the interface version of the A18 Pro's +//! 26A428 build and the same objects, with these differences: +//! +//! - [`HwData25G83`]: the A18 Pro's hardware data up to `0x2554`, eight bytes inserted before +//! `0x2570` and four more before `0x2614`, and the CS and AFR performance tables; +//! - [`PowerConfig25G83`]: the power configuration with two blocks removed (`0xa8` and `0xa0` +//! bytes), at bundle `+0x5380`, followed by the auxiliary data; +//! - region C and the power state ([`POWER_STATE_WORDS`]) with other values; +//! - one firmware instance carrying both roles, with a larger status B ([`private`]), five +//! separately allocated views and a firmware area ([`kernel_window`]). +//! +//! The constant contents were derived from the G16 encoders of Gravity Linux, which run this +//! firmware on the M4 Pro, by evaluating them for this SoC; see the commit that added them. + +use super::{ + fill_words, perf_states, CompletionRingDescriptor, HwDataArgs, Opp, PerfGroup, RegisterMapEntry, + Words, COMPLETION_RING_ENTRIES, COMPLETION_RING_ENTRY_SIZE, COMPLETION_RING_INDEX, PERF_STATES, + REGISTER_MAP_SLOTS, +}; +use kernel::prelude::*; + +/// Kernel-window objects whose placement differs from the A18 Pro's, as offsets in the upper +/// kernel window. +pub(crate) mod kernel_window { + /// The power-state object named by the root (`0x6a8` bytes). + pub(crate) const POWER_STATE: u64 = 0x17_0000; + /// The five views named by the main configuration. + pub(crate) const VIEWS: [u64; 5] = [0x41_4000, 0x40_0000, 0x40_4000, 0x40_8000, 0x40_c000]; + /// The firmware area named twice by the main configuration (`0x88` bytes). + pub(crate) const FIRMWARE_AREA: u64 = 0x41_0000; + /// The AON temperature-source table named by the power configuration. + pub(crate) const AON_TABLE: u64 = 0xc050_0000; +} + +/// Offsets in the private cluster. +pub(crate) mod private { + /// Size of the private cluster: the state grid at `+0`, status B and status A. + pub(crate) const SIZE: usize = 0xd4000; + /// The primary status B ([`super::super::PrimaryStatusB`] up to its power state). + pub(crate) const STATUS_B: usize = 0x1_0000; + /// The primary status A, after the `0xf180` bytes of status B. + pub(crate) const STATUS_A: usize = 0x1_f180; + /// The configuration named by the root; its status follows at `+0x80`. + pub(crate) const CONFIG: usize = 0xd_3440; + /// The zeroed page named by the main configuration's `unk_18`. + pub(crate) const MAIN_HEADER: usize = 0x8000; +} + +/// Bundle offset of the [`PowerConfig25G83`]. +pub(crate) const POWER_CONFIG: usize = 0x5380; +/// Bundle offset of the completion-ring descriptors in the [`HwData25G83`]. +pub(crate) const COMPLETION_DESCRIPTORS: usize = 0x2620; +/// Bundle offset of the QoS pointer in the [`HwData25G83`]. +pub(crate) const QOS_POINTER: usize = 0x26b8; + +/// Most states of a CS or AFR performance table. +const AUX_PERF_STATES: usize = 16; + +/// The CS or AFR performance table of the hardware data. +#[repr(C, packed)] +pub(crate) struct AuxPerfTable { + /// Highest state, twice. + pub(crate) max_state: [u32; 2], + /// Frequency of each state in MHz. + pub(crate) freq_mhz: [u32; AUX_PERF_STATES], + /// Voltage of each state in mV, in the first word of each pair. + pub(crate) mv: [[u32; 2]; AUX_PERF_STATES], + /// SRAM voltage of each state in mV, in the first word of each pair. + pub(crate) sram_mv: [[u32; 2]; AUX_PERF_STATES], +} + +// SAFETY: `AuxPerfTable` consists of integers and arrays of integers only. +unsafe impl Zeroable for AuxPerfTable {} + +static_assert!(size_of::() == 0x148); + +impl AuxPerfTable { + /// Fills the table from its operating points (one voltage cell each), raising each SRAM + /// voltage to `sram_min_mv`. + fn fill(&mut self, opps: &[Opp], sram_min_mv: u32) -> Result { + if !(2..=AUX_PERF_STATES).contains(&opps.len()) { + return Err(EINVAL); + } + self.max_state = [opps.len() as u32 - 1; 2]; + for (index, opp) in opps.iter().enumerate() { + if opp.cells != 1 { + return Err(EINVAL); + } + let mv = opp.volt_uv[0] / 1000; + self.freq_mhz[index] = u32::try_from(opp.freq_hz / 1_000_000)?; + self.mv[index][0] = mv; + self.sram_mv[index][0] = mv.max(sram_min_mv); + } + Ok(()) + } +} + +/// Inputs of the 25G83 hardware data in addition to [`HwDataArgs`]. +pub(crate) struct HwDataArgs25G83<'a> { + /// Enabled GPU cores. + pub(crate) active_cores: u32, + /// Operating points of the CS clock domain. + pub(crate) cs_opps: &'a [Opp], + /// Operating points of the AFR clock domain. + pub(crate) afr_opps: &'a [Opp], +} + +/// The hardware data of the 25G83 build, at the base of the bundle. +#[repr(C, packed)] +pub(crate) struct HwData25G83 { + /// Required: 0x6f_0000_0000. + pub(crate) unk_0: u64, + /// Required: 0xffc0_0000. + pub(crate) unk_8: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_10: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_18: u64, + /// Required: 0x2ff_ffff_8000. + pub(crate) unk_20: u64, + /// Required: 0xffff_fc21_8140_0000. + pub(crate) unk_28: u64, + pub(crate) unk_30: [u8; 0x610], + /// Register windows the firmware maps. + pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], + pub(crate) unk_e88: [u8; 8], + /// Chip identifier. + pub(crate) chip_id: u32, + /// Required: 1. + pub(crate) unk_e94: u32, + /// Required: 1. + pub(crate) unk_e98: u32, + pub(crate) unk_e9c: [u8; 4], + /// Required: 1. + pub(crate) unk_ea0: u32, + /// Required: 1. + pub(crate) unk_ea4: u32, + pub(crate) unk_ea8: [u8; 0x10], + /// Required: 1. + pub(crate) unk_eb8: u32, + pub(crate) unk_ebc: [u8; 4], + /// Required: 1. + pub(crate) unk_ec0: u32, + pub(crate) unk_ec4: [u8; 4], + /// Required: 1. + pub(crate) unk_ec8: u32, + pub(crate) unk_ecc: [u8; 4], + /// Required: 0x5dc0 (24000). + pub(crate) unk_ed0: u32, + /// Required: 1. + pub(crate) unk_ed4: u32, + /// Power sample period in ms: 8. + pub(crate) unk_ed8: u32, + pub(crate) unk_edc: [u8; 4], + /// Required: 1 in every entry. + pub(crate) unk_ee0: [u32; 3], + pub(crate) unk_eec: [u8; 0x18], + /// Required: 0x1f (31). + pub(crate) unk_f04: u32, + pub(crate) unk_f08: [u8; 0x1c], + /// Required: 0x0700_2004. + pub(crate) unk_f24: u32, + /// Required: 6. + pub(crate) unk_f28: u32, + pub(crate) unk_f2c: [u8; 8], + /// Required: 1. + pub(crate) unk_f34: u32, + /// Required: 1. + pub(crate) unk_f38: u32, + pub(crate) unk_f3c: [u8; 0x10], + /// Required: 0x31 (49). + pub(crate) unk_f4c: u32, + pub(crate) unk_f50: [u8; 0x38], + /// Enabled GPU cores. + pub(crate) unk_f88: u32, + /// Required: 0xf (15). + pub(crate) unk_f8c: u32, + pub(crate) unk_f90: [u8; 0x28], + /// Required: 0x1c (28). + pub(crate) unk_fb8: u32, + /// Required: 4. + pub(crate) unk_fbc: u32, + /// Enabled GPU cores. + pub(crate) unk_fc0: u32, + /// Performance table A, one voltage lane per cluster. + pub(crate) perf_a: PerfGroup, + pub(crate) unk_16c8: [u8; 0x140], + /// Frequency of each state of table B in MHz. + pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], + pub(crate) unk_1834: [u8; 0x14], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_1848: [u32; 11], + pub(crate) unk_1874: [u8; 0x54], + /// Relative performance of each state of table A, in percent. + pub(crate) perf_a_relative: [u32; PERF_STATES], + pub(crate) unk_18f4: [u8; 0x14], + /// Relative performance of each state of table B, in percent. + pub(crate) perf_b_relative: [u32; PERF_STATES], + pub(crate) unk_1934: [u8; 0x94], + /// Operating point of each state of table A. + pub(crate) perf_a_states: [u32; PERF_STATES], + pub(crate) unk_19f4: [u8; 0x14], + /// Operating point of each state of table B. + pub(crate) perf_b_states: [u32; PERF_STATES], + pub(crate) unk_1a34: [u8; 0x14], + /// CS clock-domain performance table. + pub(crate) cs_perf: AuxPerfTable, + /// AFR clock-domain performance table. + pub(crate) afr_perf: AuxPerfTable, + /// Performance table B, one voltage lane per cluster. + pub(crate) perf_b: PerfGroup, + pub(crate) unk_23dc: [u8; 0x168], + /// Required: 5. + pub(crate) unk_2544: u32, + /// Required: 6. + pub(crate) unk_2548: u32, + /// Required: 3. + pub(crate) unk_254c: u32, + /// Required: 7. + pub(crate) unk_2550: u32, + /// Required: 7. + pub(crate) unk_2554: u32, + pub(crate) unk_2558: [u8; 0x18], + /// Enabled GPU cores. + pub(crate) unk_2570: u32, + pub(crate) unk_2574: [u8; 4], + /// Required: 1. + pub(crate) unk_2578: u32, + pub(crate) unk_257c: [u8; 0x1c], + /// Required: 0, 1, 1, 1. + pub(crate) unk_2598: [u32; 4], + /// Required: 0, 1, 1, 1. + pub(crate) unk_25a8: [u32; 4], + pub(crate) unk_25b8: [u8; 4], + /// Required: 0xffff_ffff in every entry, one per work channel. + pub(crate) unk_25bc: [u32; 12], + pub(crate) unk_25ec: [u8; 8], + /// Required: 2. + pub(crate) unk_25f4: u32, + pub(crate) unk_25f8: [u8; 4], + /// Required: 1. + pub(crate) unk_25fc: u32, + pub(crate) unk_2600: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2614: u32, + pub(crate) unk_2618: [u8; 4], + /// Required: 0x100. + pub(crate) unk_261c: u32, + /// Completion-ring descriptors. + pub(crate) completion_rings: [CompletionRingDescriptor; 4], + pub(crate) unk_26a0: [u8; 0x18], + /// QoS object, zeroed. + pub(crate) qos_va: u64, + /// Required: 1. + pub(crate) unk_26c0: u32, + /// Required: 1. + pub(crate) unk_26c4: u32, + pub(crate) unk_26c8: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_26f4: u32, + pub(crate) unk_26f8: [u8; 0x16bc], +} + +static_assert!(core::mem::offset_of!(HwData25G83, register_map) == 0x640); +static_assert!(core::mem::offset_of!(HwData25G83, chip_id) == 0xe90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_a) == 0xfc4); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b_freq_mhz) == 0x1808); +static_assert!(core::mem::offset_of!(HwData25G83, cs_perf) == 0x1a48); +static_assert!(core::mem::offset_of!(HwData25G83, afr_perf) == 0x1b90); +static_assert!(core::mem::offset_of!(HwData25G83, perf_b) == 0x1cd8); +static_assert!(core::mem::offset_of!(HwData25G83, unk_2570) == 0x2570); +static_assert!(core::mem::offset_of!(HwData25G83, completion_rings) == COMPLETION_DESCRIPTORS); +static_assert!(core::mem::offset_of!(HwData25G83, qos_va) == QOS_POINTER); +static_assert!(size_of::() == 0x3db4); +static_assert!(size_of::() <= POWER_CONFIG); + +// SAFETY: `HwData25G83` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwData25G83 {} + +impl HwData25G83 { + /// Initializes the hardware data. Table A and B carry one voltage lane per cluster. + pub(crate) fn init(&mut self, args: &HwDataArgs<'_>, ext: &HwDataArgs25G83<'_>) -> Result { + super::clear(self); + self.unk_0 = 0x6f_0000_0000; + self.unk_8 = 0xffc0_0000; + self.unk_10 = 0x10_0000_0000; + self.unk_18 = 0x10_0000_0000; + self.unk_20 = 0x2ff_ffff_8000; + self.unk_28 = 0xffff_fc21_8140_0000; + + let mut used = [false; REGISTER_MAP_SLOTS]; + for window in args.windows { + match used.get_mut(window.slot) { + Some(used) if !*used => *used = true, + _ => return Err(EINVAL), + } + self.register_map[window.slot] = RegisterMapEntry::window(window); + } + if !args.reserved_slots.is_empty() { + return Err(EINVAL); + } + + self.chip_id = args.chip_id; + self.unk_e94 = 1; + self.unk_e98 = 1; + self.unk_ea0 = 1; + self.unk_ea4 = 1; + self.unk_eb8 = 1; + self.unk_ec0 = 1; + self.unk_ec8 = 1; + self.unk_ed0 = 0x5dc0; + self.unk_ed4 = 1; + self.unk_ed8 = 8; + self.unk_ee0 = [1; 3]; + self.unk_f04 = 0x1f; + self.unk_f24 = 0x0700_2004; + self.unk_f28 = 6; + self.unk_f34 = 1; + self.unk_f38 = 1; + self.unk_f4c = 0x31; + self.unk_f88 = ext.active_cores; + self.unk_f8c = 0xf; + self.unk_fb8 = 0x1c; + self.unk_fbc = 4; + self.unk_fc0 = ext.active_cores; + + let perf = args.perf; + let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; + let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; + if states_a[0].cells < 2 { + return Err(EINVAL); + } + self.perf_a.fill(&states_a, 1); + self.perf_b.fill(&states_b, 1); + for (index, state) in states_b.iter().enumerate() { + self.perf_b_freq_mhz[index] = state.freq_mhz; + self.perf_a_states[index] = perf.states_a[index].into(); + self.perf_b_states[index] = perf.states_b[index].into(); + } + self.perf_a_relative = perf.relative_a; + self.perf_b_relative = perf.relative_b; + self.unk_1848 = [0x3f82_8f5c; 11]; + self.cs_perf.fill(ext.cs_opps, perf.sram_min_mv)?; + self.afr_perf.fill(ext.afr_opps, perf.sram_min_mv)?; + + self.unk_2544 = 5; + self.unk_2548 = 6; + self.unk_254c = 3; + self.unk_2550 = 7; + self.unk_2554 = 7; + self.unk_2570 = ext.active_cores; + self.unk_2578 = 1; + self.unk_2598 = [0, 1, 1, 1]; + self.unk_25a8 = [0, 1, 1, 1]; + self.unk_25bc = [u32::MAX; 12]; + self.unk_25f4 = 2; + self.unk_25fc = 1; + self.unk_2614 = 1; + self.unk_261c = 0x100; + for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { + self.completion_rings[index as usize] = CompletionRingDescriptor { + low_va: ring.low_va, + high_va: ring.high_va, + entries: COMPLETION_RING_ENTRIES, + entry_size: COMPLETION_RING_ENTRY_SIZE, + index, + unk_1c: 0, + }; + } + self.qos_va = args.qos_va; + self.unk_26c0 = 1; + self.unk_26c4 = 1; + self.unk_26f4 = 1; + Ok(()) + } +} + +/// Power configuration control period, in 24 MHz clocks (8 ms). +const PERIOD_CLOCKS: u32 = 200_000; +/// Highest GPU power, in mW. +pub(crate) const MAX_POWER_MW: u32 = 69_116; +/// [`MAX_POWER_MW`] as an IEEE-754 single. +const MAX_POWER_MW_F32: u32 = 0x4786_fe00; +/// Temperature sensors read by the die and MTR temperature limiters. +const TEMP_SENSOR_MASK: u32 = 0x2a22_2aa2; + +/// Leakage coefficients, IEEE-754 singles: three arrays of sixteen clusters, then four arrays +/// of two dies. Not found in the ADT; constant for the SoC. +const LEAKAGE: [u32; 56] = { + let mut words = [0; 56]; + words[0] = 0x44f0_3000; // 1921.5 + words[1] = 0x44f3_0000; // 1944.0 + words[32] = 0x4373_0000; // 243.0 + words[33] = 0x4373_0000; // 243.0 + words[52] = 0x437b_0000; // 251.0 + words[54] = 0x43bd_c000; // 379.5 + words +}; +/// Offset of [`LEAKAGE`] in the power configuration. +const LEAKAGE_OFFSET: usize = 0x642c; +/// Offset in the power configuration of the GPU address of the configuration's status (the +/// configuration named by the root, `+0x80`). +const CONFIG_STATUS_POINTER: usize = 0x3b50; +/// Offset in the power configuration of the GPU address of the AON table. +const AON_POINTER: usize = 0x6318; +/// Offset in the power configuration of a random word the firmware expects. +const ENTROPY: usize = 0x63d4; + +/// Nonzero words of the power configuration and the auxiliary data that follows it, offsets +/// from the start of the power configuration. +const POWER_CONFIG_WORDS: Words = &[ + (0x4, PERIOD_CLOCKS, 2), + (0x10, 0x4, 1), + (0x14, 0x3f800000, 1), + (0x2c, 0x1, 2), + (0x40, 0x64, 1), + (0x44, 0x1, 1), + (0x48, 0x3e8, 1), + (0x50, 0x64, 1), + (0x60, 0x4, 1), + (0x64, 0x3f800000, 1), + (0x6c, 0x1, 2), + (0x80, 0x64, 1), + (0x84, 0x1, 1), + (0x88, 0x3e8, 1), + (0x90, 0x64, 1), + (0xc0, 0x3f828f5c, 11), + (0x9b8, 0x271, 1), + (0x9c4, 0x3f7f2e9f, 1), + (0x9cc, 0x3b516154, 1), + (0x9d4, 0x3ca59586, 1), + (0x9e0, MAX_POWER_MW_F32, 1), + (0x9e4, 0x40a90fdb, 1), + (0x9ec, 0xbc6d0cfe, 1), + (0x9f0, 0x64, 1), + (0x9f4, 0x3e8, 2), + (0xa04, MAX_POWER_MW, 1), + (0xa38, 0x3e8, 1), + (0xa70, 0x8, 1), + (0xa7c, 0x3f000000, 1), + (0xa84, 0x3f000000, 1), + (0xa8c, 0x3e3bd7b3, 1), + (0xa98, 0x47800000, 1), + (0xa9c, 0x3ecf2e49, 1), + (0xaa8, 0x64, 1), + (0xaac, 0x3e8, 1), + (0xabc, MAX_POWER_MW, 1), + (0xac8, 0x10, 1), + (0xad0, 0x5dc00, 1), + (0xb28, 0x50, 1), + (0xb30, 0x55, 1), + (0xb34, 0x32, 1), + (0xb38, 0x6, 1), + (0xb40, 0x6, 1), + (0xb44, 0x1, 1), + (0xb4c, 0x3f4ccccd, 1), + (0xb50, 0x3f7eb852, 1), + (0xb54, 0x3e4ccccd, 1), + (0xb58, 0x3ba3d70a, 1), + (0xb5c, 0x3f64c986, 2), + (0xb68, 0x42be0000, 1), + (0xb6c, 0x403ea7f0, 2), + (0xb74, 0x41179436, 1), + (0xb78, 0x64, 1), + (0xb7c, 0x3e8, 1), + (0xb80, 0x64, 1), + (0xb8c, 0x50, 1), + (0xbc0, 0x64, 1), + (0x1bec, 0xff000000, 1), + (0x1bf0, 0xff7fff7f, 255), + (0x1fec, 0x7fff7f, 1), + (0x21f0, 0x47800000, 1), + (0x21f4, 0x42c80000, 1), + (0x21f8, 0x447a0000, 1), + (0x21fc, 0x62a, 1), + (0x2210, 0x64, 1), + (0x2218, 0x42c80000, 1), + (0x221c, 0x3e8, 1), + (0x2224, 0x3f4ccccd, 1), + (0x2228, 0x3e4ccccd, 1), + (0x2268, TEMP_SENSOR_MASK, 1), + (0x2270, 0x1f40, 1), + (0x2278, 0x4, 1), + (0x228c, 0x3f800000, 1), + (0x2294, 0x414eeeee, 1), + (0x22a0, 0x47800000, 1), + (0x22a4, 0x42140000, 1), + (0x22b0, 0x64, 1), + (0x22b4, 0x3e8, 2), + (0x22c4, 0x2af8, 1), + (0x22f8, 0x3e8, 1), + (0x2728, 0xb8520000, 1), + (0x272c, 0x3f7e, 1), + (0x2730, 0xd70a0000, 1), + (0x2734, 0x3ba3, 1), + (0x273c, 0x43c8, 1), + (0x2748, 0x4780, 1), + (0x274c, 0x42c8, 1), + (0x2754, 0x384bac8, 1), + (0x2758, 0x3e80000, 2), + (0x2764, 0x80000000, 1), + (0x2768, 0x26ac453b, 1), + (0x279c, 0x3e80000, 1), + (0x27b8, 0xc80000, 1), + (0x2870, 0x3e8, 1), + (0x28c0, 0x1, 1), + (0x28c8, MAX_POWER_MW, 3), + (0x28e0, 0x3f800000, 1), + (0x28e4, 0x4, 1), + (0x28e8, 0x8, 1), + (0x28ec, 0x2ee00, 1), + (0x29bc, 0x7c, 1), + (0x29c8, 0x3f77bdef, 1), + (0x29d0, 0x3d042108, 1), + (0x29d8, 0x3d25bea1, 1), + (0x29e4, 0x47800000, 1), + (0x29e8, 0x3f4f2e49, 1), + (0x29f4, 0x28, 1), + (0x29f8, 0x3e8, 2), + (0x2a04, MAX_POWER_MW_F32, 1), + (0x2a08, MAX_POWER_MW, 1), + (0x2a14, 0xf8, 1), + (0x2a1c, 0x5ad200, 1), + (0x2a3c, 0x3e8, 1), + (0x2b4c, 0x32, 1), + (0x2b50, 0x1, 1), + (0x2b58, 0x3f638e39, 1), + (0x2b5c, 0x3f2aaaab, 1), + (0x2b60, 0x3de38e39, 1), + (0x2b64, 0x3eaaaaab, 1), + (0x2b68, 0xbecccccd, 2), + (0x2b74, 0x47800000, 1), + (0x2b78, 0xc0a00000, 2), + (0x2b84, 0x64, 1), + (0x2b88, 0x3e8, 1), + (0x2b8c, 0x64, 1), + (0x2b94, 0x45fa0000, 1), + (0x2b98, 0x578, 1), + (0x2ba4, 0x48, 1), + (0x2ba8, 0x18, 1), + (0x2bac, 0x1a5e00, 1), + (0x2bb4, 0x8ca00, 1), + (0x2bc0, 0x47800000, 1), + (0x2be8, 0x1f40, 1), + (0x2bec, 0x4, 1), + (0x2bf0, 0x9c4, 1), + (0x2bf4, 0x258, 1), + (0x2bf8, 0x2, 1), + (0x2bfc, 0x4, 1), + (0x2c00, 0x32, 1), + (0x2c30, 0x47800000, 1), + (0x2c40, 0x64, 1), + (0x2c44, 0x3e8, 1), + (0x2e2c, 0x4, 1), + (0x2e38, 0x3f780000, 1), + (0x2e40, 0x3d000000, 1), + (0x2e48, 0x40a00000, 1), + (0x2e54, 0x47800000, 1), + (0x2e58, 0x41700000, 1), + (0x2e64, 0x384, 1), + (0x2e68, 0x3e8, 2), + (0x2e78, 0x3e8, 1), + (0x2eac, 0x3e8, 1), + (0x2eec, 0x64, 1), + (0x2f08, TEMP_SENSOR_MASK, 1), + (0x2f10, 0x7d, 1), + (0x2f14, 0x1, 1), + (0x3b58, TEMP_SENSOR_MASK, 1), + (0x3b60, 0x1, 1), + (0x5b64, 0x40a00000, 1), + (0x6320, u32::from_be_bytes(*b"g0TR"), 1), + (0x6324, u32::from_be_bytes(*b"g1TR"), 1), + (0x6360, 0x3, 1), + (0x6368, u32::from_be_bytes(*b"r0TR"), 1), + (0x6370, 0x1, 1), + (0x6420, 0x1, 1), +]; + +/// The power configuration and auxiliary data of the 25G83 build, at bundle +/// [`POWER_CONFIG`]. The main configuration names it. +#[repr(C)] +pub(crate) struct PowerConfig25G83(pub(crate) [u8; 0x6510]); + +// SAFETY: `PowerConfig25G83` is a byte array. +unsafe impl Zeroable for PowerConfig25G83 {} + +static_assert!(LEAKAGE_OFFSET + size_of::<[u32; 56]>() <= size_of::()); + +impl PowerConfig25G83 { + /// Pointer and random words, which [`Self::hash`] leaves out. + const RUNTIME: [(usize, usize); 3] = + [(CONFIG_STATUS_POINTER, 8), (AON_POINTER, 8), (ENTROPY, 4)]; + + /// Initializes the power configuration. + pub(crate) fn init(&mut self, config_va: u64, aon_va: u64, entropy: u32) { + fill_words(&mut self.0, POWER_CONFIG_WORDS); + for (index, word) in LEAKAGE.iter().enumerate() { + let at = LEAKAGE_OFFSET + 4 * index; + self.0[at..at + 4].copy_from_slice(&word.to_le_bytes()); + } + self.0[CONFIG_STATUS_POINTER..CONFIG_STATUS_POINTER + 8] + .copy_from_slice(&(config_va + 0x80).to_le_bytes()); + self.0[AON_POINTER..AON_POINTER + 8].copy_from_slice(&aon_va.to_le_bytes()); + self.0[ENTROPY..ENTROPY + 4].copy_from_slice(&entropy.to_le_bytes()); + } + + /// Hash of the contents without the pointers and the random word. + pub(crate) fn hash(&self) -> u64 { + hash_without(&self.0, &Self::RUNTIME) + } +} + +/// The AON temperature-source rows the power configuration names: the GPU cluster 0, GPU +/// cluster 1 and SoC sensors. +pub(crate) const AON_TABLE: [u32; 9] = [ + u32::from_be_bytes(*b"g0TR"), + 0x006b_0001, + 0x0000_1401, + u32::from_be_bytes(*b"g1TR"), + 0x00b5_0001, + 0x0000_1401, + u32::from_be_bytes(*b"r0TR"), + 0x00c1_0001, + 0x0000_1401, +]; + +/// Nonzero words of the power-state object. The fast-die limiter reads its target +/// (`+0xc8`, 101.00 °C) and release (`+0xcc`, 96.00 °C) temperatures in 0.01 °C; left at zero +/// the limit is 0 °C and the GPU stays at its lowest state whenever the PMP runs. +pub(crate) const POWER_STATE_WORDS: Words = &[(0xc8, 10_100, 1), (0xcc, 9_600, 1)]; + +impl HwData25G83 { + /// What [`Self::hash`] leaves out: the runtime words (completion descriptors, QoS pointer) + /// and what comes from the device: the performance tables built from its operating points, + /// which are binned per device, and its enabled core count. + const UNHASHED: [(usize, usize); 10] = [ + (COMPLETION_DESCRIPTORS, 0x80), + (QOS_POINTER, 8), + (core::mem::offset_of!(Self, unk_f88), size_of::()), + (core::mem::offset_of!(Self, unk_fc0), size_of::()), + (core::mem::offset_of!(Self, perf_a), size_of::()), + (core::mem::offset_of!(Self, perf_b_freq_mhz), size_of::<[u32; PERF_STATES]>()), + (core::mem::offset_of!(Self, cs_perf), size_of::()), + (core::mem::offset_of!(Self, afr_perf), size_of::()), + (core::mem::offset_of!(Self, perf_b), size_of::()), + (core::mem::offset_of!(Self, unk_2570), size_of::()), + ]; + + /// Hash of the constant contents (everything but [`Self::UNHASHED`]). + pub(crate) fn hash(&self) -> u64 { + // SAFETY: `HwData25G83` consists of integers only and has no padding. + let bytes = unsafe { + core::slice::from_raw_parts((self as *const Self).cast::(), size_of::()) + }; + hash_without(bytes, &Self::UNHASHED) + } +} + +/// FNV-1a 64 of `bytes` with the `(offset, length)` ranges in `skip` left out. +pub(crate) fn hash_without(bytes: &[u8], skip: &[(usize, usize)]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for (offset, byte) in bytes.iter().enumerate() { + if skip.iter().any(|&(at, len)| (at..at + len).contains(&offset)) { + continue; + } + hash = (hash ^ u64::from(*byte)).wrapping_mul(0x100_0000_01b3); + } + hash +} + +/// Hashes ([`hash_without`]) of the hardware data, power configuration and region C that the +/// Gravity Linux G16 encoders produce for the J616s, which this build's objects must equal. +pub(crate) mod expected { + pub(crate) const HW_DATA: u64 = 0xfd3b_a26f_570f_57cd; + pub(crate) const POWER_CONFIG: u64 = 0xb669_5b84_2ad2_36fe; + pub(crate) const REGION_C: u64 = 0xd2db_fa6b_8976_47b4; +} diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 8f08531097f87..1cb46fce2d602 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -23,6 +23,9 @@ pub(crate) mod queue; pub(crate) mod render; pub(crate) mod render_rt; pub(crate) mod qos; +pub(crate) mod variant; + +pub(crate) use variant::Variant; /// Clears a firmware structure in place. /// diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index ac46fda33eb5f..5d9a6202943f3 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -20,6 +20,8 @@ use core::sync::atomic::{AtomicU32, Ordering}; use kernel::prelude::*; +use super::variant::PointerLayout; + use super::kick::{KickTimestamp, QID_MAX}; /// Firmware scheduling profile of a queue. @@ -94,6 +96,10 @@ pub(crate) enum CommandTag { QueueConfig = 0x0f, /// [`KickPredecessor`]. KickPredecessor = 0x10, + /// [`FragmentBarrier`] (25G83 firmware). + FragmentBarrier = 4, + /// [`InitBufferManager`] (25G83 firmware). + InitBufferManager = 5, } /// The engine that executes a queue's work. @@ -115,26 +121,29 @@ pub(crate) enum DataMaster { /// /// - creation: the whole block, [`PointerBlock::new`], before the queue is /// first published; -/// - publication: `producer`, with one aligned 32-bit store per publication +/// - publication: the producer, with one aligned 32-bit store per publication /// step, after a barrier that orders the work pointers and commands before /// it; /// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. +/// +/// Where the producer and the recovery target live depends on the firmware +/// build ([`PointerLayout`]). #[repr(C)] #[derive(Debug)] pub(crate) struct PointerBlock { /// Item-ring position up to which the firmware has finished commands. - /// Free ring space is measured from it, so it bounds `producer`. + /// Free ring space is measured from it, so it bounds the producer. pub(crate) done: AtomicU32, pub(crate) unk_04: [u32; 11], /// Written by the firmware. The host treats the queue as idle only when - /// this cursor, `done` and `producer` are equal. + /// this cursor, `done` and the producer are equal. pub(crate) unk_30: AtomicU32, pub(crate) unk_34: [u32; 3], - /// Item-ring position after the last work pointer the host published. - pub(crate) producer: AtomicU32, + /// 26A428: item-ring position after the last work pointer the host published. + word_40: AtomicU32, pub(crate) unk_44: [u32; 3], - /// On recovery, the host copies producer here before acknowledging resume. - pub(crate) recovery_target: u32, + /// 26A428: on recovery, the host copies the producer here before acknowledging resume. + word_50: AtomicU32, pub(crate) unk_54: [u32; 3], /// Number of entries in the item ring. pub(crate) entries: u32, @@ -143,29 +152,40 @@ pub(crate) struct PointerBlock { static_assert!(size_of::() == 0x80); static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); -static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_40) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, word_50) == 0x50); static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); // SAFETY: PointerBlock contains only integers and integer atomics. unsafe impl Zeroable for PointerBlock {} impl PointerBlock { - pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); - pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); - /// Returns the cursors of an empty item ring. - pub(crate) const fn new() -> Self { - Self { + pub(crate) fn new(layout: &PointerLayout) -> Self { + let block = Self { done: AtomicU32::new(0), unk_04: [0; 11], unk_30: AtomicU32::new(0), unk_34: [0; 3], - producer: AtomicU32::new(0), + word_40: AtomicU32::new(0), unk_44: [0; 3], - recovery_target: u32::MAX, + word_50: AtomicU32::new(0), unk_54: [0; 3], entries: ITEM_RING_ENTRIES, unk_64: [0; 7], + }; + if let Some(target) = layout.recovery_target { + block.word(target).store(u32::MAX, Ordering::Relaxed); + } + block + } + + /// The host word at `offset`, one of the two [`PointerLayout`] names. + fn word(&self, offset: usize) -> &AtomicU32 { + if offset == core::mem::offset_of!(Self, word_50) { + &self.word_50 + } else { + &self.word_40 } } @@ -174,12 +194,12 @@ impl PointerBlock { /// Used when a render queue pair that was kicked before is handed to a new /// owner, with `base` the position the host tracks for the new owner's /// queues. Only valid while the firmware does not run the queue, with all - /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this + /// three cursors zero. Writes `done`, `unk_30` and the producer, in this /// order, with single stores; the caller issues a full barrier afterwards. - pub(crate) fn rebase(&self, base: u32) { + pub(crate) fn rebase(&self, layout: &PointerLayout, base: u32) { self.done.store(base, Ordering::Relaxed); self.unk_30.store(base, Ordering::Relaxed); - self.producer.store(base, Ordering::Relaxed); + self.word(layout.producer).store(base, Ordering::Relaxed); } } @@ -525,6 +545,95 @@ impl QueueConfig { } } +/// The 25G83 firmware's barrier before every fragment descriptor (tag 4): the fragment work +/// waits for the paired tiling stamp, in addition to the kick-entry dependency, as the 25G83 +/// Metal driver writes it ("emitBarrierCommands"). +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FragmentBarrier { + /// [`CommandTag::FragmentBarrier`]. + pub(crate) tag: u32, + /// GPU address of the tiling queue's completion stamp word. + pub(crate) stamp_va: u64, + /// Timestamp of the tiling kick. + pub(crate) tiling_kick: u64, + /// QID of the tiling queue. + pub(crate) tiling_qid: u32, + /// Tiling stamp value waited for. + pub(crate) wait: u32, + pub(crate) unk_1c: [u8; 8], + /// Event of the wait: the tiling QID. + pub(crate) event: u32, + /// Fragment stamp value. + pub(crate) stamp_self: u32, + /// Current object ID of the fragment stage. + pub(crate) fragment_id: u64, + pub(crate) unk_34: u32, + pub(crate) unk_38: [u8; 0x10], +} + +static_assert!(size_of::() == 0x48); +static_assert!(core::mem::offset_of!(FragmentBarrier, wait) == 0x18); +static_assert!(core::mem::offset_of!(FragmentBarrier, event) == 0x24); +static_assert!(core::mem::offset_of!(FragmentBarrier, fragment_id) == 0x2c); + +impl FragmentBarrier { + /// The barrier of a pass whose tiling stage, kicked at `tiling_kick` on `tiling_qid`, + /// signals `stamp_value` at `stamp_va`. + pub(crate) fn new( + stamp_va: u64, + tiling_kick: KickTimestamp, + tiling_qid: u32, + stamp_value: u32, + fragment_id: u32, + ) -> Self { + Self { + tag: CommandTag::FragmentBarrier as u32, + stamp_va, + tiling_kick: tiling_kick.get(), + tiling_qid, + wait: stamp_value, + unk_1c: [0; 8], + event: tiling_qid, + stamp_self: stamp_value, + fragment_id: fragment_id.into(), + unk_34: 1, + unk_38: [0; 0x10], + } + } +} + +/// The 25G83 firmware's buffer-manager initialization (tag 5), before the first tiling +/// descriptor that uses a parameter buffer. The block count is read from the manager. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct InitBufferManager { + /// [`CommandTag::InitBufferManager`]. + pub(crate) tag: u32, + /// UAT context of the pass. + pub(crate) context_id: u32, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// Tiling stamp value of the pass. + pub(crate) stamp_value: u32, +} + +static_assert!(size_of::() == 0x18); + +impl InitBufferManager { + pub(crate) fn new(context_id: u32, pb_slot: u32, pb_va: u64, stamp_value: u32) -> Self { + Self { + tag: CommandTag::InitBufferManager as u32, + context_id, + pb_slot, + pb_va, + stamp_value, + } + } +} + /// Announcement of a newly written kick entry of a queue (tag 14). /// /// A record slot larger than this structure must be zero beyond it. diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 33c42a5a2aca7..f28ef376b887d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -48,10 +48,59 @@ pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; const USER_BASE: u64 = 0x10_0000_0000; /// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. const VDM_OFFSET_LIMIT: u64 = 1 << 39; -/// The deflake buffer holds three sub-buffers: at its start and at these -/// offsets. -const DEFLAKE_20: u64 = 0x20; -const DEFLAKE_2A0: u64 = 0x2a0; +/// The deflake (context) buffer holds three sub-buffers: at its start and at two offsets +/// that follow the control words of each cluster. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Deflake { + /// Offsets of the second and third sub-buffers. + pub(crate) sub: [u64; 2], + /// The host writes the viewport words at `+0x900`; with more clusters that offset lies in + /// the third sub-buffer and the buffer is left to the GPU. + pub(crate) viewport: bool, +} + +impl Deflake { + /// The layout for a GPU with `clusters` clusters. + pub(crate) fn new(clusters: u32) -> Result { + match clusters { + 1 => Ok(Self { + sub: [0x20, 0x2a0], + viewport: true, + }), + // shortcut: only one and two clusters were measured; add other counts when a GPU + // with them is supported. + 2 => Ok(Self { + sub: [0xa0, 0x820], + viewport: false, + }), + _ => Err(EINVAL), + } + } +} + +/// The register programs of a firmware build's render descriptors. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum RegisterPrograms { + /// The A18 Pro firmware's (26A428). + G17P, + /// The 25G83 build's: other tiling and fragment arrays, another tile-state base, and the + /// tiler parameter sub-buffers in the TPC object. + V25G83, +} + +/// The 25G83 tiling and main fragment arrays. +const TA_REGISTERS_25G83: usize = 83; +const FRAGMENT_REGISTERS_25G83: usize = 91; +const TA_CHECKPOINTS_25G83: [u16; 3] = [81, 83, 83]; +const FRAGMENT_CHECKPOINTS_25G83: [u16; 3] = [87, 88, 88]; +/// Index of the free-list slot register (0x14308) in the 25G83 tiling array. +const TA_SLOT_INDEX_25G83: u8 = 79; +/// Index of the free-list slot register (0x14048) in the 25G83 fragment array. +const FRAGMENT_SLOT_INDEX_25G83: u8 = 84; +/// A 25G83 stage with shader-context scratch (ray queries) also writes the SCS enable 0x017d9 = 1, +/// which the compute program and both A18 Pro render programs carry for such work: first in the +/// tiling array, after 0x01739 in the fragment array. Every count and index after it moves by one. +const SCS_ENABLE: u32 = 0x017d9; const TA_REGISTERS: usize = 73; const TA_REGISTERS_MAX: usize = TA_REGISTERS + 1; @@ -104,6 +153,10 @@ const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; /// 32, the utile size bits of the tile-buffer configuration in bits 43:40, /// tiles in X less one from bit 44 and tiles in Y less one from bit 53. const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; +/// The 25G83 build's low tile-state bits, the ones the M4 Pro has run with. With the A18 Pro's, +/// a pass more than 80 tiles wide and with more than 4096 tiles never completes its fragment +/// stage (the 3456x2234 panel is 108 x 70 tiles). +const TILE_STATE_BASE_25G83: u64 = 0x36011 | 1 << 37; const TILE_STATE_LAYERED: u64 = 1 << 32; const TILE_STATE_UTILE_SHIFT: u32 = 28; const TILE_STATE_TILES_X_SHIFT: u32 = 44; @@ -262,6 +315,9 @@ pub(crate) struct RenderResources { pub(crate) tpc_va: u64, /// Size of the tail-pointer cache the target needs. pub(crate) tpc_size: u64, + /// GPU addresses of the tiler parameter sub-buffers (entry 0 is the TPC); zero for + /// an empty one and on other GPUs. + pub(crate) gtp_va: [u64; 8], /// GPU address of the deflake buffer. pub(crate) deflake_va: u64, /// GPU address of the tiling status word. @@ -314,6 +370,10 @@ pub(crate) struct RenderArgs<'a> { pub(crate) ta: RenderStage, /// Fragment stage. pub(crate) fragment: RenderStage, + /// Register programs of the firmware build. + pub(crate) programs: RegisterPrograms, + /// Deflake-buffer layout of the GPU. + pub(crate) deflake: Deflake, } impl RenderArgs<'_> { @@ -491,9 +551,10 @@ fn ta_registers( let layermeta = user_offset(r.layermeta_va)?; let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; let tpc = user_offset(r.tpc_va)?; + let [deflake_20, deflake_2a0] = args.deflake.sub; let deflake = u64::from(user_offset32(r.deflake_va)?); - let deflake_20 = u64::from(user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)?); - let deflake_2a0 = u64::from(user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)?); + let deflake_20 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_20)?)?); + let deflake_2a0 = u64::from(user_offset32(offset_va(r.deflake_va, deflake_2a0)?)?); let vdm = user_offset(p.vdm_va)?; if vdm >= VDM_OFFSET_LIMIT { return Err(EINVAL); @@ -581,6 +642,129 @@ fn ta_registers( ]) } +/// The 25G83 tiling array: the A18 Pro's without 0x1748 and 0x1740 (and 0x17d9 unless the stage +/// has SCS), with the tiler parameter sub-buffer registers and 0x10131 = 0x88, in the order the +/// M4 Pro has run with. Returns the array and its used entry count. +fn ta_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; TA_REGISTERS_25G83 + 1], usize)> { + let a = ta_registers(args, g, w)?; + let r = args.resources; + let gtp = |index: usize| -> Result { + match r.gtp_va[index] { + 0 => Ok(0), + va => user_offset(va), + } + }; + let find = |reg: u32, nth: usize| -> Result { + a.iter() + .filter(|write| write.reg == reg) + .nth(nth) + .copied() + .ok_or(EINVAL) + }; + let gtp1 = gtp(1)?; + let gtp2 = gtp(2)?; + let gtp6 = gtp(6)? & !127; + let scs = usize::from(args.ray_query[0].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); TA_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut push = |write: RegisterWrite| { + out[n] = write; + n += 1; + }; + if scs != 0 { + push(find(SCS_ENABLE, 0)?); + } + for reg in [0x10141, 0x1c039, 0x1c9c8] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x1c041, gtp1)); + push(RegisterWrite::new(0x1c9d0, gtp1)); + for reg in [ + 0x1c0a1, 0x1c031, 0x1c9c0, 0x1c051, 0x1c061, 0x10149, 0x10139, 0x10111, 0x1c9b0, 0x10119, + 0x1c9b8, 0x1c958, 0x1c950, 0x1c930, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 0)?); + push(RegisterWrite::new(0x1c948, gtp(4)?)); + push(RegisterWrite::new(0x1c888, gtp(5)?)); + push(RegisterWrite::new(0x1c890, 0)); + push(RegisterWrite::new(0x1c918, 0)); + push(find(0x1c079, 0)?); + push(find(0x1c9d8, 0)?); + push(RegisterWrite::new(0x1c089, gtp2)); + push(RegisterWrite::new(0x1c9e0, gtp2)); + push(RegisterWrite::new(0x16c41, gtp6)); + push(RegisterWrite::new(0x1ca40, gtp6)); + push(RegisterWrite::new(0x1c9a8, 0)); + push(RegisterWrite::new(0x1c920, gtp(7)?)); + for reg in [0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, 0x1c8f8, 0x1c0b1, 0x1c850] { + push(find(reg, 0)?); + } + push(RegisterWrite::new(0x10131, 0x88)); + for reg in [ + 0x10121, 0x10129, 0x101b9, 0x1c069, 0x1c071, 0x1c081, 0x1c0a9, 0x10171, 0x10169, 0x0a309, + 0x1c8e0, 0x1c8e8, + ] { + push(find(reg, 0)?); + } + push(find(0x1c898, 0)?); + for reg in [ + 0x101e1, 0x1c9e8, 0x1a099, 0x1a0a1, 0x1a069, 0x1a071, 0x1a0c9, 0x1a0d1, 0x101c9, 0x0d471, + 0x1a0f1, 0x10799, 0x1c830, 0x1ca30, 0x16c39, 0x1c910, 0x0a5a1, 0x0d419, 0x1ca10, 0x014a1, + 0x0a349, 0x10209, 0x1c9f0, 0x14320, 0x14308, 0x14318, + ] { + push(find(reg, 0)?); + } + push(find(0x1c880, 1)?); + push(find(0x1c898, 1)?); + if n != TA_REGISTERS_25G83 + scs || out[TA_SLOT_INDEX_25G83 as usize + scs].reg != 0x14308 { + return Err(EINVAL); + } + Ok((out, n)) +} + +/// The 25G83 main fragment array: the A18 Pro's (without 0x17d9 unless the stage has SCS), with +/// 0x160b8 = 0 after the tile state and 0x160a8 = 0x20 before the closing 0x16068, as the M4 +/// Pro has run with. Returns the array and its used entry count. +fn fragment_registers_25g83( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<([RegisterWrite; FRAGMENT_REGISTERS_25G83 + 1], usize)> { + let a = fragment_registers(args, g, w); + let scs = usize::from(args.ray_query[1].scratch.enabled()); + let mut out = [RegisterWrite::new(0, 0); FRAGMENT_REGISTERS_25G83 + 1]; + let mut n = 0; + let mut tile_state = false; + for (index, write) in a.iter().enumerate() { + if write.reg == SCS_ENABLE && scs == 0 { + continue; + } + if index == a.len() - 1 { + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160a8, 0x20); + n += 1; + } + *out.get_mut(n).ok_or(EINVAL)? = *write; + n += 1; + if write.reg == 0x16068 && !tile_state { + tile_state = true; + *out.get_mut(n).ok_or(EINVAL)? = RegisterWrite::new(0x160b8, 0); + n += 1; + } + } + if n != FRAGMENT_REGISTERS_25G83 + scs + || out[FRAGMENT_SLOT_INDEX_25G83 as usize + scs].reg != 0x14048 + { + return Err(EINVAL); + } + Ok((out, n)) +} + fn fragment_registers( args: &RenderArgs<'_>, g: &Geometry, @@ -588,7 +772,12 @@ fn fragment_registers( ) -> [RegisterWrite; FRAGMENT_REGISTERS_MAX] { let p = args.pass; let r = args.resources; - let mut tile_state = TILE_STATE_BASE + let tile_state_base = if args.programs == RegisterPrograms::V25G83 { + TILE_STATE_BASE_25G83 + } else { + TILE_STATE_BASE + }; + let mut tile_state = tile_state_base | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; @@ -902,7 +1091,8 @@ impl TaDescriptor { let geometry = Geometry::new(pass)?; let words = PassWords::new(pass)?; let registers = ta_registers(args, &geometry, &words)?; - let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; + let deflake_2a0_offset = + user_offset32(offset_va(resources.deflake_va, args.deflake.sub[1])?)?; let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; let sampler_max = sampler_max(pass.sampler_count)?; let stamp_value = args.stamp_value(); @@ -919,7 +1109,11 @@ impl TaDescriptor { let scs_register = args.ray_query[0].scratch.enabled(); self.checkpoints = TA_CHECKPOINTS.map(|count| count + u16::from(scs_register)); self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + self.checkpoints = TA_CHECKPOINTS_25G83.map(|count| count + u16::from(scs_register)); + let (registers, count) = ta_registers_25g83(args, &geometry, &words)?; + self.registers.set(array_va, ®isters[..count])?; + } else if scs_register { self.registers.set(array_va, ®isters)?; } else { let mut trimmed = [registers[0]; TA_REGISTERS]; @@ -945,7 +1139,10 @@ impl TaDescriptor { self.current_id = resources.ta_ids.current.into(); self.ordinal = args.ordinal as u32; self.timestamp_start_va = resources.timestamp_va; - self.unk_932 = Self::UNK_932; + self.unk_932 = match args.programs == RegisterPrograms::V25G83 { + true => TA_SLOT_INDEX_25G83 + u8::from(scs_register), + false => Self::UNK_932, + }; self.free_list_control_va = resources.free_list_control_va; self.unk_93c = Self::UNK_93C; if args.ordinal == 0 { @@ -980,11 +1177,14 @@ impl TaDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { - let array = RegisterArrayBinding::new( - offset_va(descriptor_va, Self::ARRAY_OFFSET)?, - TA_CHECKPOINTS[0] as u8 + u8::from(scs_register), - )?; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => TA_CHECKPOINTS_25G83[0] as u8, + false => TA_CHECKPOINTS[0] as u8, + } + u8::from(scs_register); + let array = + RegisterArrayBinding::new(offset_va(descriptor_va, Self::ARRAY_OFFSET)?, checkpoint)?; Ok([Some(array), None, None, None]) } } @@ -1171,13 +1371,18 @@ impl FragmentDescriptor { pub(crate) fn register_bindings( descriptor_va: u64, scs_register: bool, + programs: RegisterPrograms, ) -> Result<[Option; 4]> { let array = |index, checkpoint: usize| -> Result> { let va = offset_va(descriptor_va, Self::array_offset(index))?; Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) }; + let checkpoint = match programs == RegisterPrograms::V25G83 { + true => FRAGMENT_CHECKPOINTS_25G83[0] as usize, + false => FRAGMENT_CHECKPOINTS[0] as usize, + } + usize::from(scs_register); Ok([ - array(0, FRAGMENT_CHECKPOINTS[0] as usize + usize::from(scs_register))?, + array(0, checkpoint)?, array(1, PARTIAL_STORE_REGISTERS)?, array(2, PARTIAL_RESUME_REGISTERS)?, array(3, PARTIAL_LOAD_REGISTERS)?, @@ -1249,7 +1454,12 @@ impl FragmentSlot { desc.checkpoints = FRAGMENT_CHECKPOINTS.map(|count| count + extra); desc.entries = FRAGMENT_REGISTERS as u16 + extra; let registers = fragment_registers(args, &geometry, &words); - if scs_register { + if args.programs == RegisterPrograms::V25G83 { + desc.checkpoints = FRAGMENT_CHECKPOINTS_25G83.map(|count| count + extra); + desc.entries = FRAGMENT_REGISTERS_25G83 as u16 + extra; + let (registers, count) = fragment_registers_25g83(args, &geometry, &words)?; + desc.arrays[0].set(arrays[0], ®isters[..count])?; + } else if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; @@ -1294,7 +1504,10 @@ impl FragmentSlot { desc.ordinal = args.ordinal as u32; desc.timestamp_start_va = resources.timestamp_va; desc.timestamp_end_va = timestamp_end_va; - desc.unk_21cc = Self::UNK_21CC; + desc.unk_21cc = match args.programs == RegisterPrograms::V25G83 { + true => FRAGMENT_SLOT_INDEX_25G83 + u8::from(args.ray_query[1].scratch.enabled()), + false => Self::UNK_21CC, + }; desc.free_list_control_va = resources.free_list_control_va; desc.unk_21d6 = Self::UNK_21D6; desc.status_va = resources.fragment_status_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/variant.rs b/drivers/gpu/drm/asahi/g17/fw/variant.rs new file mode 100644 index 0000000000000..bb30925f54160 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/variant.rs @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware builds. +//! +//! Firmware builds that share the interface version ([`super::initdata::INTERFACE_VERSION`]) +//! still differ in a few shared layouts and queue rules. A [`Variant`] names those differences +//! for one build; each SoC's [`crate::g17::Config`] points at the variant of the firmware it +//! runs. + +/// Where a firmware build keeps the host words of a queue's pointer block +/// ([`super::queue::PointerBlock`]). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct PointerLayout { + /// Offset of the item-ring producer. + pub(crate) producer: usize, + /// Offset of the word the host copies the producer to before it acknowledges a recovery, + /// if the build has one. + pub(crate) recovery_target: Option, +} + +/// How a firmware build acknowledges the initialization root the host hands it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitAck { + /// With an `INITDATA_ACK` message on the firmware endpoint. + Message, + /// By setting word `+0x14` of the primary status A to 1; it sends no message. + Status, +} + +/// The initialization graph of a firmware build. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum InitGraph { + /// Two instances, as the A18 Pro firmware (26A428) runs. + G17P, + /// One instance carrying both roles, with the 25G83 layouts ([`super::initdata::v25g83`]). + V25G83, +} + +/// The layouts and rules of one G17 firmware build. +pub(crate) struct Variant { + /// The initialization graph. + pub(crate) graph: InitGraph, + /// How the firmware acknowledges its initialization. + pub(crate) init_ack: InitAck, + /// The host ends the power-on transition with `POWER_ON_DONE` once every coprocessor + /// is initialized. + pub(crate) power_on_done: bool, + /// First QID the host may use; the firmware keeps the lower ones for itself. + pub(crate) qid_floor: usize, + /// Host words of a queue's pointer block. + pub(crate) pointers: PointerLayout, + /// The host registers each render pair through the GPU's MMIO queue ports. + pub(crate) mmio_pair_registration: bool, + /// Render kick entries may name an MCache table. + pub(crate) mcache: bool, + /// Renders may set the few-primitives auxiliary framebuffer flag. + pub(crate) few_primitives: bool, + /// The firmware runs ray-traversal work (ray queries and intersection programs). + pub(crate) ray_traversal: bool, + /// Render item rings carry lead commands: InitBufferManager before the first tiling pass + /// of a buffer manager, and FragmentBarrier before every fragment descriptor. + pub(crate) lead_commands: bool, + /// Register programs of the render descriptors. + pub(crate) programs: super::render::RegisterPrograms, +} + +/// A firmware build a SoC supports, identified by the macOS release it ships with. +pub(crate) struct Build { + /// macOS version (major, minor, patch). + pub(crate) os_version: [u32; 3], + /// Name, for the log. + pub(crate) name: &'static str, + /// Layouts and rules of the build. + pub(crate) variant: &'static Variant, +} + +/// How a SoC's firmware build is determined. +pub(crate) enum Firmware { + /// The SoC runs a single build, which is not identified (A18 Pro). + Fixed(&'static Variant), + /// One of these builds, selected by the macOS version that the boot loader reports in + /// `apple,firmware-version`: the release whose boot firmware loaded the GPU firmware, and + /// with it this firmware build. Any other version is refused. + Identified(&'static [Build]), +} + +/// The A18 Pro firmware (26A428). +pub(crate) const V26A428: Variant = Variant { + graph: InitGraph::G17P, + init_ack: InitAck::Message, + power_on_done: true, + qid_floor: 0, + pointers: PointerLayout { + producer: 0x40, + recovery_target: Some(0x50), + }, + mmio_pair_registration: true, + mcache: true, + few_primitives: true, + ray_traversal: true, + lead_commands: false, + programs: super::render::RegisterPrograms::G17P, +}; + +/// The M4 Pro firmware (25G83, `mBoot-18000.161.10`). +pub(crate) const V25G83: Variant = Variant { + graph: InitGraph::V25G83, + init_ack: InitAck::Status, + // Never sent on the M4 Pro: the firmware has retired its opening record when it accepts + // its initialization. + power_on_done: false, + // QIDs below 32 belong to the firmware's bootstrap protocol. + qid_floor: 32, + pointers: PointerLayout { + producer: 0x50, + recovery_target: None, + }, + // Whether this build tolerates the MMIO registration is unknown; the M4 Pro has only run + // without it. + mmio_pair_registration: false, + mcache: false, + // A fragment stage with the few-primitives flag hangs. + few_primitives: false, + // A stalled ray traversal wedges the GPU without a firmware recovery. + ray_traversal: false, + lead_commands: true, + programs: super::render::RegisterPrograms::V25G83, +}; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 6b2f7c18584ef..6057e7d0ac957 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -16,6 +16,8 @@ use core::sync::atomic::{fence, AtomicU32, Ordering}; use kernel::{c_str, prelude::*}; use t8140::{kernel_window as high, low}; +mod v25g83; + const EVENT_RING_RECHECK: u32 = 256; const INDEX_SIZE: usize = 0x10000; @@ -155,6 +157,15 @@ pub(super) struct InitData { boot_control: usize, boot_state: usize, metrics_fw_va: u64, + /// Client VA of the parameter-buffer metrics object (`Config::pm_metrics`). + pm_metrics_va: u64, + /// `Config::recovery_slot_keys`. + recovery_slot_keys: Option, + /// Primary status A and status B in the cluster. + status_a: usize, + status_b: usize, + /// Separately allocated views; `None` when they are packed into the bundle. + views: Option<[usize; 5]>, compute_shared: Option, compute_scheduler: Option, compute_active: core::cell::Cell, @@ -235,6 +246,11 @@ impl InitData { boot_control: 0, boot_state: 0, metrics_fw_va: 0, + pm_metrics_va: 0, + recovery_slot_keys: None, + status_a: abi::private::PRIMARY_STATUS_A, + status_b: abi::private::STATUS_B, + views: None, compute_shared: None, compute_scheduler: None, compute_active: core::cell::Cell::new(false), @@ -247,6 +263,10 @@ impl InitData { } } + pub(super) fn pm_metrics_va(&self) -> u64 { + self.pm_metrics_va + } + pub(super) fn metrics_fw_va(&self) -> Result { if self.metrics_fw_va == 0 { return Err(EIO); @@ -502,7 +522,7 @@ impl InitData { core::mem::size_of_val(record), )?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -522,7 +542,7 @@ impl InitData { ) -> Result<[u32; 3]> { use super::fw::channels; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let mut values = [0; 3]; for (value, offset) in values.iter_mut().zip([ @@ -601,7 +621,7 @@ impl InitData { }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) + self.state_offset(Role::Primary) + abi::work_state_offset(channel) + abi::ChannelState::PRODUCER, )?; @@ -694,7 +714,11 @@ impl InitData { uat: &mmu::Uat, render: &mmu::Vm, metrics: &KernelObject, + active_cores: u32, + variant: &super::fw::Variant, ) -> Result { + self.pm_metrics_va = cfg.pm_metrics.va; + self.recovery_slot_keys = cfg.recovery_slot_keys; let range = uat.geometry().kernel_range(); let env = BuildEnv { dev, @@ -704,8 +728,11 @@ impl InitData { base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, }; - self.map_register_windows(&env)?; - let tables = self.build_descriptor_tables(&env)?; + if variant.graph == super::fw::variant::InitGraph::V25G83 { + return self.build_25g83(&env, cfg, metrics, active_cores); + } + self.map_register_windows(&env, cfg.windows)?; + let tables = self.build_descriptor_tables(&env, abi::private::SIZE)?; self.build_render_free_lists(&env)?; let first_block = self.build_render_pool(&env)?; self.build_compute_pools(&env, first_block)?; @@ -713,31 +740,41 @@ impl InitData { self.initialize_firmware_objects(&env, cfg, &tables, &bundle) } - fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { - for window in &t8140::REGISTER_WINDOWS { - let (phys, va, size) = window.mapping().ok_or(EINVAL)?; - self.aliases.push( - env.vm.map_io( - env.dev.as_ref().as_ref(), - va, - phys.try_into()?, - size.try_into()?, - mmu::PROT_FW_MMIO_RW, - )?, - GFP_KERNEL, - )?; + fn map_register_windows( + &mut self, + env: &BuildEnv<'_>, + windows: &[abi::RegisterWindow], + ) -> Result { + for window in windows { + for instance in 0..window.instances { + let (phys, va, size) = window.mapping(instance).ok_or(EINVAL)?; + self.aliases.push( + env.vm.map_io( + env.dev.as_ref().as_ref(), + va, + phys.try_into()?, + size.try_into()?, + mmu::PROT_FW_MMIO_RW, + )?, + GFP_KERNEL, + )?; + } } Ok(()) } /// The private cluster, zero page and the three descriptor tables with their low aliases. - fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { + fn build_descriptor_tables( + &mut self, + env: &BuildEnv<'_>, + cluster_size: usize, + ) -> Result { use mmu::{ PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, }; use CpuMap::{WriteBack as WB, WriteCombined as WC}; - self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, cluster_size, FS, WC)?)?; self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? @@ -988,13 +1025,13 @@ impl InitData { bundle: &BundleObjects, ) -> Result { let base = env.base; - let opps = Self::opps(env.dev)?; + let opps = Self::opps(env.dev, cfg.opps)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, - windows: &t8140::REGISTER_WINDOWS, - reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, opps: &opps, - perf: &t8140::PERF, + perf: cfg.perf, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; @@ -1002,7 +1039,7 @@ impl InitData { .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; self.object_mut(self.region_c)? .initialize::(0, |region| { - region.init(); + region.init(cfg.region_c); Ok(()) })?; self.object_mut(self.cluster)? @@ -1050,8 +1087,8 @@ impl InitData { let slot = role as usize; let main_offset = abi::bundle::MAIN_CONFIG[slot]; let main_va = base + high::BUNDLE + main_offset as u64; - let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; - let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; + let state_va = base + high::PRIVATE_CLUSTER + self.state_offset(role) as u64; + let status_va = base + high::PRIVATE_CLUSTER + self.status_offset(role) as u64; let args = abi::MainConfigArgs { bundle_va: base + high::BUNDLE, main_config_va: main_va, @@ -1107,7 +1144,7 @@ impl InitData { (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), ] { self.object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .store(value, Ordering::Relaxed); } } @@ -1150,7 +1187,7 @@ impl InitData { )?; } mappings.push( - metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, + metrics.map_alias(vm, self.pm_metrics_va, mmu::PROT_GPU_FW_SHARED_RW)?, GFP_KERNEL, )?; Ok(mappings) @@ -1180,37 +1217,53 @@ impl InitData { .load(Ordering::Relaxed)) } - fn opps(dev: &AsahiDrmDevice) -> Result> { + fn opps(dev: &AsahiDrmDevice, count: usize) -> Result> { + let opps = Self::opp_table(dev, c_str!("operating-points-v2"))?; + if opps.len() != count { + return Err(EINVAL); + } + Ok(opps) + } + + /// The operating points of the table that the GPU node's property `name` names; all have + /// the same number of voltage cells. + fn opp_table(dev: &AsahiDrmDevice, name: &kernel::str::CStr) -> Result> { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; - let table = node - .parse_phandle(c_str!("operating-points-v2"), 0) - .ok_or(EIO)?; + let table = node.parse_phandle(name, 0).ok_or(EIO)?; let mut opps = KVec::new(); for opp in table.children() { + // One cell for the whole GPU, or one per cluster. + let cells: KVec = opp.get_property(c_str!("opp-microvolt"))?; + if cells.is_empty() || cells.len() > abi::OPP_CELLS { + return Err(EINVAL); + } + let mut volt_uv = [0; abi::OPP_CELLS]; + volt_uv[..cells.len()].copy_from_slice(&cells); opps.push( abi::Opp { freq_hz: opp.get_property(c_str!("opp-hz"))?, - volt_uv: opp.get_property(c_str!("opp-microvolt"))?, + volt_uv, + cells: cells.len(), }, GFP_KERNEL, )?; } - if opps.len() != t8140::OPPS { + if opps.iter().any(|opp| opp.cells != opps[0].cells) { return Err(EINVAL); } Ok(opps) } - fn state_offset(role: Role) -> usize { + fn state_offset(&self, role: Role) -> usize { match role { Role::Primary => abi::private::PRIMARY_STATE, Role::Secondary => abi::private::SECONDARY_STATE, } } - fn status_offset(role: Role) -> usize { + fn status_offset(&self, role: Role) -> usize { match role { - Role::Primary => abi::private::PRIMARY_STATUS_A, + Role::Primary => self.status_a, Role::Secondary => abi::private::SECONDARY_STATUS_A, } } @@ -1218,7 +1271,7 @@ impl InitData { /// Recheck the host-authored event pointer pair after a bounded number of drains. /// The device mutex serializes the cache and all event consumption. pub(super) fn validate_event_ring(&self, role: Role) -> Result { - let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; + let status = self.object(self.cluster)?.gpu_va() + self.status_offset(role) as u64; let expected = ( status + abi::status::EVENT_STATE, status + abi::status::EVENT_RING, @@ -1247,9 +1300,14 @@ impl InitData { Ok(()) } + /// Whether a firmware that acknowledges in status A accepted the graph (`+0x14` = 1). + pub(super) fn init_accepted(&self) -> Result { + Ok(self.status_word(Role::Primary, 0x14)?.load(Ordering::Acquire) == 1) + } + pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { self.object(self.cluster)?.word( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1261,7 +1319,7 @@ impl InitData { offset: usize, ) -> Result<[u32; N]> { self.object(self.cluster)?.read_words( - Self::status_offset(role) + self.status_offset(role) .checked_add(offset) .ok_or(EINVAL)?, ) @@ -1336,7 +1394,7 @@ impl InitData { for offset in [0, 0x10, abi::ChannelState::PRODUCER] { if self .object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .word(self.state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? .load(Ordering::SeqCst) != abi::OPENING_PRODUCER { @@ -1351,8 +1409,12 @@ impl InitData { /// Validate the serialized view before reading its retained host mapping. fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { let bundle = self.object(self.bundle)?; - let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + // Separately allocated views live at offset 0 of their own object. + let (object, base) = match self.views { + Some(views) => (self.object(*views.get(view).ok_or(EINVAL)?)?, 0), + None => (bundle, *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?), + }; let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) .ok_or(EOVERFLOW)?; if end > extent { @@ -1363,20 +1425,20 @@ impl InitData { fence(Ordering::Acquire); let pointer = bundle.read_words::<2>(pointer_at)?; let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; - if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { + if actual != object.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { return Err(EINVAL); } - bundle.read_words(base + offset) + object.read_words(base + offset) } } impl super::recovery::Memory for InitData { fn fault_sources(&self) -> Result { let mask = self.object(self.cluster)? - .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .dword(self.status_b + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot)?; + let key = super::recovery::slot_key(slot, self.recovery_slot_keys)?; let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) .ok().map(|words| words[0])?; @@ -1391,21 +1453,21 @@ impl super::recovery::Memory for InitData { fence(Ordering::Acquire); Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)? .load(Ordering::Relaxed)) } fn host_recovery(&self) -> Result { Ok(self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? + .word(self.status_b + abi::PrimaryStatusB::HOST_RECOVERY)? .load(Ordering::Relaxed)) } fn transition(&self, expected: u32, next: u32) -> Result { let state = self .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; + .word(self.status_b + abi::PrimaryStatusB::RECOVERY_STATE)?; fence(Ordering::Acquire); if state.load(Ordering::Relaxed) != expected { return Err(EBUSY); @@ -1417,17 +1479,21 @@ impl super::recovery::Memory for InitData { } fn clear_timestamps(&self) -> Result { - let bundle = self.object(self.bundle)?; + // The per-QID timestamp table is view 4; separately allocated views live at offset 0 of + // their own object. + let (table, offset) = match self.views { + Some(views) => (self.object(*views.get(4).ok_or(EINVAL)?)?, 0), + None => (self.object(self.bundle)?, abi::bundle::VIEWS[4]), + }; // Validate the complete table before publishing its first live word. const ENTRIES: usize = 128; const STRIDE: usize = 0x10; - let offset = abi::bundle::VIEWS[4]; - bundle.pointer(offset, ENTRIES * STRIDE)?; + table.pointer(offset, ENTRIES * STRIDE)?; for qid in 0..ENTRIES { - bundle + table .word(offset + qid * STRIDE)? .store(0, Ordering::Relaxed); - bundle + table .dword(offset + qid * STRIDE + 8)? .store(0, Ordering::Relaxed); } @@ -1437,7 +1503,7 @@ impl super::recovery::Memory for InitData { fn entries(&self) -> Result { let cluster = self.object(self.cluster)?; - let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; + let base = self.status_b + abi::PrimaryStatusB::RECOVERY_INFO; fence(Ordering::Acquire); let mut entries = super::recovery::Entries::default(); for index in 0..256 { diff --git a/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs new file mode 100644 index 0000000000000..bfb7cf1af3ecd --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata/v25g83.rs @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The initialization graph of the 25G83 firmware build. +//! +//! One coprocessor carries both roles: there is one main configuration, with the +//! secondary-only words, and one merged root. The cluster, the hardware data, the power +//! configuration, region C, the views and the firmware area follow the 25G83 layouts +//! ([`abi::v25g83`]); free lists, pools, descriptor tables, rings and regions are the A18 Pro's. + +use super::{abi, low, BuildEnv, BundleObjects, CpuMap, DescriptorTables, InitData, KernelObject}; +use crate::g17::{Config, Role}; +use crate::mmu; +use abi::v25g83 as v; +use core::sync::atomic::{fence, Ordering}; +use kernel::{c_str, prelude::*}; + +/// Object indices of the 25G83-specific objects. +struct Objects { + views: [usize; 5], + firmware_area: usize, + power_state: usize, + aon: usize, +} + +impl InitData { + /// Constructs the 25G83 graph after the firmware has published its root entries. + pub(super) fn build_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + metrics: &KernelObject, + active_cores: u32, + ) -> Result { + self.status_a = v::private::STATUS_A; + self.status_b = v::private::STATUS_B; + self.map_register_windows(env, cfg.windows)?; + let tables = self.build_descriptor_tables(env, v::private::SIZE)?; + self.build_render_free_lists(env)?; + let first_block = self.build_render_pool(env)?; + self.build_compute_pools(env, first_block)?; + let bundle = self.build_bundle(env, cfg, metrics)?; + let objects = self.build_25g83_objects(env)?; + self.views = Some(objects.views); + self.initialize_25g83(env, cfg, active_cores, &tables, &bundle, &objects)?; + if cfg.uat_quirks.clean_tables { + // Write back every host-built byte before the firmware may read it. + for object in self.objects.iter_mut() { + let bytes = object.bytes_mut(); + crate::pgtable::clean_poc(bytes.as_ptr(), bytes.len()); + } + crate::pgtable::poc_barrier(); + } + Ok(()) + } + + /// The views, the firmware area, the power-state object and the AON table. + fn build_25g83_objects(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS}; + use CpuMap::WriteCombined as WC; + let mut views = [0; 5]; + for (view, offset) in views.iter_mut().zip(v::kernel_window::VIEWS) { + *view = self.add(env.fixed(offset, mmu::UAT_PGSZ, FP, WC)?)?; + } + let firmware_area = + self.add(env.fixed(v::kernel_window::FIRMWARE_AREA, mmu::UAT_PGSZ, FP, WC)?)?; + let power_state = + self.add(env.fixed(v::kernel_window::POWER_STATE, mmu::UAT_PGSZ, FS, WC)?)?; + let aon = self.add(env.fixed(v::kernel_window::AON_TABLE, mmu::UAT_PGSZ, FP, WC)?)?; + Ok(Objects { + views, + firmware_area, + power_state, + aon, + }) + } + + /// Writes the 25G83 firmware objects, checks them against the expected contents, then + /// stages the opening control-ring state. + fn initialize_25g83( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + active_cores: u32, + tables: &DescriptorTables, + bundle: &BundleObjects, + objects: &Objects, + ) -> Result { + let bundle_va = self.object(self.bundle)?.gpu_va(); + let cluster_va = self.object(self.cluster)?.gpu_va(); + let config_va = cluster_va + v::private::CONFIG as u64; + let status_a_va = cluster_va + v::private::STATUS_A as u64; + let status_b_va = cluster_va + v::private::STATUS_B as u64; + let aon_va = self.object(objects.aon)?.gpu_va(); + let power_state_va = self.object(objects.power_state)?.gpu_va(); + let firmware_area_va = self.object(objects.firmware_area)?.gpu_va(); + let mut view_va = [0; 5]; + for (va, index) in view_va.iter_mut().zip(objects.views) { + *va = self.object(index)?.gpu_va(); + } + + let opps = Self::opps(env.dev, cfg.opps)?; + let cs_opps = Self::opp_table(env.dev, c_str!("apple,cs-opp"))?; + let afr_opps = Self::opp_table(env.dev, c_str!("apple,afr-opp"))?; + let hw_args = abi::HwDataArgs { + chip_id: cfg.chip_id, + windows: cfg.windows, + reserved_slots: cfg.reserved_slots, + opps: &opps, + perf: cfg.perf, + completion_rings: bundle.completion_rings, + qos_va: self.object(bundle.qos)?.gpu_va(), + }; + let hw_args_25g83 = v::HwDataArgs25G83 { + active_cores, + cs_opps: &cs_opps, + afr_opps: &afr_opps, + }; + // SAFETY: Plain C entropy source. + let entropy = unsafe { kernel::bindings::get_random_u32() }; + let mut hashes = [0; 3]; + self.object_mut(self.bundle)? + .initialize::(0, |hw| { + hw.init(&hw_args, &hw_args_25g83)?; + hashes[0] = hw.hash(); + Ok(()) + })?; + self.object_mut(self.bundle)? + .initialize::(v::POWER_CONFIG, |power| { + power.init(config_va, aon_va, entropy); + hashes[1] = power.hash(); + Ok(()) + })?; + self.object_mut(self.region_c)? + .initialize::(0, |region| { + region.init(cfg.region_c); + Ok(()) + })?; + hashes[2] = v::hash_without(self.object_mut(self.region_c)?.bytes_mut(), &[]); + // The constant contents must equal what the Gravity Linux encoders produce on the proven + // setup; the operating points and the core count come from the device and are not hashed. + let expected = [ + v::expected::HW_DATA, + v::expected::POWER_CONFIG, + v::expected::REGION_C, + ]; + if hashes != expected { + dev_err!( + env.dev.as_ref(), + "25G83 initialization data differs: {:#x?}, expected {:#x?}\n", + hashes, + expected + ); + return Err(EINVAL); + } + for (index, word) in v::AON_TABLE.iter().enumerate() { + self.object_mut(objects.aon)?.write(index * 4, *word)?; + } + + // Status A and B: only the boot words and the firmware-control pointers. + self.object_mut(self.cluster)? + .write(v::private::STATUS_A, abi::StatusBlock::new())?; + self.object_mut(self.cluster)? + .write(v::private::STATUS_B, abi::StatusBlock::new())?; + // Status B keeps the A18 Pro's field offsets up to its recovery words. + let fwctl_va = self.object(tables.fwctl)?.gpu_va(); + let fwctl = + v::private::STATUS_B + core::mem::offset_of!(abi::PrimaryStatusB, fwctl_state_va); + let cluster = self.object(self.cluster)?; + cluster.dword(fwctl)?.store(fwctl_va, Ordering::Relaxed); + cluster.dword(fwctl + 8)?.store(fwctl_va + abi::FWCTL_RING, Ordering::Relaxed); + + // One main configuration carrying both roles. + let main_offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize]; + let main_va = bundle_va + main_offset as u64; + let args = abi::MainConfigArgs { + bundle_va, + main_config_va: main_va, + state_grid_va: cluster_va + self.state_offset(Role::Primary) as u64, + status_a_va, + pb_descriptors_low_va: low::PB_DESCRIPTORS, + page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, + }; + let views = abi::PrimaryViews { + zero_page_va: self.object(bundle.zero)?.gpu_va(), + pb_descriptors_va: self.object(tables.pb)?.gpu_va(), + page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), + }; + self.object_mut(self.bundle)? + .initialize::(main_offset, |main| { + main.init_primary(&args, &views); + main.view_va = view_va; + main.firmware_area_va = [firmware_area_va; 2]; + main.unk_18 = (cluster_va + v::private::MAIN_HEADER as u64).to_le_bytes(); + // The secondary role's words, on the single instance. + main.unk_300 = 4; + main.power_config_va = bundle_va + v::POWER_CONFIG as u64; + main.unk_32c = 0; + main.unk_344 = 0; + Ok(()) + })?; + self.object_mut(self.bundle)?.write( + main_offset + abi::MainConfig::CONTROL_RING, + abi::OpeningRecord::new(abi::Role::Primary), + )?; + + // The merged root. + let root = abi::RootArgs { + region_a_va: self.object(bundle.region_a)?.gpu_va(), + region_c_va: self.object(self.region_c)?.gpu_va(), + main_config_va: main_va, + status_a_va, + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + self.object_mut(self.roots)?.write( + 0, + abi::SecondaryRoot::merged(&root, status_b_va, power_state_va, config_va), + )?; + for &(offset, value, repeat) in v::POWER_STATE_WORDS { + for index in 0..repeat { + self.object_mut(objects.power_state)?.write(offset + 4 * index, value)?; + } + } + + // The opening control record is staged in slot 0. + for (offset, value) in [ + (0, 0), + (0x10, 0), + (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), + ] { + self.object(self.cluster)? + .word(self.state_offset(Role::Primary) + abi::CONTROL_STATE_OFFSET + offset)? + .store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 7aaca95992f8d..f223f967b00dd 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -30,7 +30,15 @@ pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; // channels are shared after exact retirement, rather than pinned per client. const COMPUTE_RENDER_RESERVE: usize = 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2; pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT - COMPUTE_RENDER_RESERVE; +/// Logical queues that can each hold a render pair: what the render reserve leaves beside the +/// two bootstrap reservations once the QIDs below `floor`, which the firmware keeps, are gone +/// (every execution context with no floor). +pub(super) const fn queue_limit(floor: usize) -> u32 { + ((COMPUTE_RENDER_RESERVE - 2 - floor) / 2) as u32 +} + const _: () = { + assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); assert!(COMPUTE_CHANNELS >= 4); assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); }; @@ -65,15 +73,19 @@ pub(crate) struct QueueIds { initial_render: [Option; 2], compute_publication: u64, leased: usize, + /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). + floor: usize, } impl QueueIds { - pub(crate) fn new() -> impl Init { + pub(crate) fn new(floor: usize) -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), initial_render: [None; 2], compute_publication: 0, leased: 0, + // The reserve must keep render capacity above the floor. + floor: if floor < COMPUTE_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } @@ -117,7 +129,11 @@ impl QueueIds { Err(EINVAL) }; } - let initial = if engine == DataMaster::Compute && self.leased == QID_COUNT { + // The QIDs below the floor are never the host's; the reserve shrinks + // with them, so the compute channel count stays the same. + let total = QID_COUNT - self.floor; + let reserve = COMPUTE_RENDER_RESERVE - self.floor; + let initial = if engine == DataMaster::Compute && self.leased == total { self.initial_render .iter() .copied() @@ -145,16 +161,22 @@ impl QueueIds { // Include that transition when protecting render's minimum, and // count the replacement as available compute capacity. let replacement = usize::from(initial.is_some()); - if QID_COUNT - self.leased + replacement - <= COMPUTE_RENDER_RESERVE.saturating_sub(render - replacement) + if total - self.leased + replacement + <= reserve.saturating_sub(render - replacement) { return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); } } let index = preferred - .map(usize::from) + .map(|index| usize::from(index) + self.floor) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) - .or_else(|| self.entries.iter().position(Option::is_none)) + .or_else(|| { + self.entries + .iter() + .skip(self.floor) + .position(Option::is_none) + .map(|index| index + self.floor) + }) .or_else(|| initial.map(|(_, id)| usize::from(id.qid))) .ok_or_else(|| crate::g17::capacity::refused(crate::g17::capacity::Site::PhysicalQids))?; let id = Id { @@ -220,7 +242,7 @@ mod accounting_tests { use super::*; fn table() -> KBox { - KBox::init(QueueIds::new(), GFP_KERNEL).unwrap() + KBox::init(QueueIds::new(0), GFP_KERNEL).unwrap() } #[test] diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 953f57aaef7df..9f92a74da53cc 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -59,6 +59,7 @@ use kernel::{ device::{self, Core}, drm::gem::shmem, io::{ + self, IoBase, IoSysMap, Region, // @@ -148,12 +149,50 @@ pub(crate) struct Config { pub(crate) uat_ias: u32, /// Output (physical) address bits of the UAT. pub(crate) uat_oas: u32, - /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. - pub(crate) secondary_root_offset: u64, - /// Crash log buffer of each [`Role`]. - pub(crate) crash_buffers: [FixedBuffer; 2], + /// Offset of the secondary coprocessor's kernel root in the `pagetables` region; `None` for + /// a single-instance firmware. + pub(crate) secondary_root_offset: Option, + /// Where the coprocessors' crash log buffers live. + pub(crate) crash_buffers: CrashBuffers, /// Parameter buffer page metrics shared with the firmware. pub(crate) pm_metrics: FixedBuffer, + /// Register windows the firmware maps through the hardware data. + pub(crate) windows: &'static [fw::initdata::RegisterWindow], + /// Register-map slots that are reserved but have no window. + pub(crate) reserved_slots: &'static [usize], + /// Per-SoC parts of the performance tables. + pub(crate) perf: &'static fw::initdata::PerfConfig, + /// Number of operating points the device tree lists. + pub(crate) opps: usize, + /// UAT policy where it differs from the A18 Pro's. + pub(crate) uat_quirks: mmu::UatQuirks, + /// Snapshot the firmware root's own top-level entries after the firmware booted, clear the + /// others and confirm the snapshot after building (A18 Pro). Otherwise the root is used as + /// the firmware left it, which is the only way the M4 Pro has run. + pub(crate) adopt_root_entries: bool, + /// Enable the GPU-to-fabric bridge and dynamic core power gating before the firmware + /// starts (A18 Pro). The M4 Pro has only run with neither written. + pub(crate) fabric_bridge: bool, + /// The SoC's values of region C. + pub(crate) region_c: &'static fw::initdata::RegionCConfig, + /// Physical address of the firmware's per-slot recovery keys, read while the firmware is + /// halted to attribute a fault; `None` where it is unknown (recovery then relies on the + /// firmware's own blame). + pub(crate) recovery_slot_keys: Option, + /// GPU generation reported to userspace. + pub(crate) gpu_generation: hw::GpuGen, + /// The firmware builds this SoC runs. + pub(crate) firmware: fw::variant::Firmware, +} + +/// Where a G17 target's crash log buffers live. +pub(crate) enum CrashBuffers { + /// One buffer per [`Role`] that the firmware requests at a fixed address of the kernel + /// lower root, mapped by the driver before the coprocessors start. + Fixed([FixedBuffer; 2]), + /// The firmware places its crash log in its own data segment, the `fw-data` reserved + /// region, and asks for it there by physical address. + FirmwareData, } /// A firmware coprocessor instance. @@ -196,6 +235,9 @@ const RECEIVER_RETRY_MS: i64 = 20; /// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. const OPENING_RETIRE_POLL_MS: i64 = 1; const OPENING_RETIRE_POLLS: usize = 100; +/// One-millisecond polls for a firmware that acknowledges its initialization only in status A +/// ([`fw::variant::InitAck::Status`]). +const INIT_ACCEPT_POLLS: usize = 3000; /// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in /// its buffer requests. @@ -206,7 +248,19 @@ const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; /// A crash log buffer handed to RTKit. pub(crate) struct CrashLog { iova: usize, - map: shmem::VMapOwned, + map: CrashLogMap, +} + +/// The CPU mapping of a crash log buffer. +enum CrashLogMap { + /// The driver's buffer ([`CrashBuffers::Fixed`]). + Object(shmem::VMapOwned), + /// `size` bytes at `offset` of the firmware's data segment ([`CrashBuffers::FirmwareData`]). + FirmwareData { + mem: io::mem::Mem, + offset: usize, + size: usize, + }, } impl rtkit::Buffer for CrashLog { @@ -215,7 +269,16 @@ impl rtkit::Buffer for CrashLog { } fn buf(&mut self) -> Result> { - Ok(IoSysMap::from((&self.map).as_view())) + match &mut self.map { + CrashLogMap::Object(map) => Ok(IoSysMap::from((&*map).as_view())), + CrashLogMap::FirmwareData { mem, offset, size } => { + // SAFETY: shmem_map() checked that the range lies inside the mapping. + let base = unsafe { mem.ptr().add(*offset) }; + let ptr = Region::<0>::ptr_try_from_raw_parts_mut(base, *size)?; + // SAFETY: The range is mapped RAM for as long as this borrow of the mapping. + Ok(IoSysMap::Sys(unsafe { io::SysMem::new(ptr) })) + } + } } } @@ -224,8 +287,8 @@ impl rtkit::Buffer for CrashLog { struct RoleState { role: Role, shared: Arc, - crash_log: FixedBuffer, - crash_map: shmem::VMapOwned, + /// The fixed crash log buffer and its mapping; `None` where the firmware places its own. + crash_log: Option<(FixedBuffer, shmem::VMapOwned)>, crashed: AtomicBool, #[pin] ack: Opaque, @@ -304,13 +367,28 @@ impl rtkit::Operations for RoleOps { iova: usize, size: usize, ) -> Result { - if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size - { + if let Some((buffer, map)) = data.crash_log.as_ref() { + if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != buffer.va || size > buffer.size { + return Err(EINVAL); + } + return Ok(CrashLog { + iova, + map: CrashLogMap::Object(map.owner().owned_vmap()?), + }); + } + // The firmware places its buffers in its data segment, at their physical address. + let node = data.shared.dev.of_node().ok_or(EINVAL)?; + let res = node.reserved_mem_region_to_resource_byname(c_str!("fw-data"))?; + let offset = iova.checked_sub(res.start().try_into()?).ok_or(EINVAL)?; + if offset.checked_add(size).ok_or(EINVAL)? > res.size().try_into()? { return Err(EINVAL); } + // SAFETY: The boot loader reserved this RAM for the loaded GPU firmware, which keeps + // it until reboot; only the requested subrange is exposed. + let mem = unsafe { io::mem::Mem::try_new(res, io::mem::MemFlag::WB.into())? }; Ok(CrashLog { iova, - map: data.crash_map.owner().owned_vmap()?, + map: CrashLogMap::FirmwareData { mem, offset, size }, }) } } @@ -323,7 +401,7 @@ struct Coprocessor { running: bool, rtkit: rtkit::RtKit, state: Arc, - _crash_buffer: KernelObject, + _crash_buffer: Option, } impl Coprocessor { @@ -337,27 +415,32 @@ impl Coprocessor { role: Role, ) -> Result { let index = role as usize; - let buffer = &cfg.crash_buffers[index]; - let mut crash_buffer = KernelObject::new( - dev, - uat.kernel_lower_vm(), - Placement::At(buffer.va), - buffer.size, - mmu::PROT_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); - - let crash_map = crash_buffer.kernel_map()?; + let (crash_buffer, crash_log) = match &cfg.crash_buffers { + CrashBuffers::Fixed(buffers) => { + let buffer = &buffers[index]; + let mut crash_buffer = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(buffer.va), + buffer.size, + mmu::PROT_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); + let crash_map = crash_buffer.kernel_map()?; + let fixed = FixedBuffer { + va: buffer.va, + size: buffer.size, + }; + (Some(crash_buffer), Some((fixed, crash_map))) + } + CrashBuffers::FirmwareData => (None, None), + }; let state = Arc::pin_init( pin_init!(RoleState { role, shared: shared.clone(), - crash_log: FixedBuffer { - va: buffer.va, - size: buffer.size, - }, - crash_map, + crash_log, crashed: AtomicBool::new(false), ack <- Opaque::ffi_init(|ptr| { // SAFETY: ptr is valid uninitialized completion storage that is pinned @@ -403,8 +486,9 @@ impl Coprocessor { Pin::new(&mut self.rtkit).boot() } - /// Starts the firmware endpoints and hands the firmware its initialization root. - fn init(&mut self, root_va: u64) -> Result { + /// Starts the firmware endpoints and hands the firmware its initialization root. An + /// acknowledgment message is awaited here; one in status A is the caller's to poll. + fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { let mut rtkit = Pin::new(&mut self.rtkit); rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; rtkit.as_mut().start_endpoint(EP_DOORBELL)?; @@ -412,7 +496,10 @@ impl Coprocessor { EP_FIRMWARE, MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), )?; - self.state.wait_for_ack() + match ack { + fw::variant::InitAck::Message => self.state.wait_for_ack(), + fw::variant::InitAck::Status => Ok(()), + } } /// Sends a host notification to the firmware. @@ -451,6 +538,14 @@ impl Bridge { Ok(bridge) } + /// Keeps the register block for a target that runs without the bridge. + fn unused(regs: regs::G17Resources) -> Bridge { + Bridge { + regs, + enabled: false, + } + } + fn disable(&mut self) { if self.enabled { self.regs.disable_g17_bridge(); @@ -471,13 +566,16 @@ impl Drop for Bridge { /// dropped, the bridge is disabled once they are stopped, and the firmware objects and their /// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. struct Firmware { - secondary: Coprocessor, + /// The secondary instance, on GPUs whose firmware runs on two (A18 Pro). A single-instance + /// firmware (M4 Pro) carries both roles on the primary. + secondary: Option, primary: Coprocessor, bridge: Bridge, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, registration_power_release_pending: bool, + variant: &'static fw::Variant, queues: runtime::Registry, pool: Arc, /// Render kick ring storage, kept and reused for the life of the device. @@ -491,7 +589,9 @@ struct Firmware { impl Drop for Firmware { fn drop(&mut self) { - self.secondary.stop(); + if let Some(secondary) = self.secondary.as_mut() { + secondary.stop(); + } self.primary.stop(); self.pool.stop_after_firmware(); // RTKit destruction releases the mailbox power references. Clear the bridge while @@ -501,6 +601,15 @@ impl Drop for Firmware { } impl Firmware { + /// The firmware instances this GPU runs. + fn roles(&self) -> &'static [Role] { + if self.secondary.is_some() { + &[Role::Primary, Role::Secondary] + } else { + &[Role::Primary] + } + } + /// Publishes controls only while the restart handshake is closed. The device mutex remains /// held through publication and the following mailbox notification. fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { @@ -611,6 +720,9 @@ struct Shared { drm: Mutex>, clusters: u32, active_cores: u32, + variant: &'static fw::Variant, + /// Layout of compute queues' operand-state buffers for this GPU's cores. + operand_state: &'static fw::compute::OperandState, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, @@ -846,6 +958,7 @@ impl Gpu { cfg: &'static Config, regs: regs::G17Resources, ) -> Result> { + let variant = Self::firmware_variant(dev, &cfg.firmware)?; let id = regs.get_g17_id()?; if id.family != cfg.identity.family || id.variant != cfg.identity.variant @@ -870,12 +983,15 @@ impl Gpu { let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + let operand_state = fw::compute::OperandState::new(id.num_clusters, active_cores)?; let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), dev: dev.as_ref().as_ref().into(), clusters: id.num_clusters, active_cores, + variant, + operand_state, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, @@ -917,6 +1033,42 @@ impl Gpu { Ok(gpu) } + /// Returns the variant of the loaded firmware build. An identified SoC refuses a build it + /// does not know, before anything is written to the GPU. + fn firmware_variant( + dev: &AsahiDrmDevice, + firmware: &fw::variant::Firmware, + ) -> Result<&'static fw::Variant> { + let builds = match firmware { + fw::variant::Firmware::Fixed(variant) => return Ok(variant), + fw::variant::Firmware::Identified(builds) => builds, + }; + let version = Self::os_firmware_version(dev); + if let Some(build) = builds.iter().find(|build| Some(build.os_version) == version) { + dev_info!(dev.as_ref(), "GPU firmware: {}\n", build.name); + return Ok(build.variant); + } + dev_err!( + dev.as_ref(), + "Unsupported GPU firmware: macOS version {:?} (apple,firmware-version)\n", + version + ); + for build in builds.iter() { + dev_err!(dev.as_ref(), "Supported GPU firmware: {}\n", build.name); + } + Err(ENODEV) + } + + /// The macOS version the boot loader reports for the boot firmware that loaded the GPU + /// firmware: `apple,firmware-version` of the GPU node, which m1n1 writes from the ADT. + fn os_firmware_version(dev: &AsahiDrmDevice) -> Option<[u32; 3]> { + let node = dev.as_ref().as_ref().of_node()?; + let cells = node + .get_property::>(c_str!("apple,firmware-version")) + .ok()?; + cells.as_slice().try_into().ok() + } + /// Returns the highest advertised operating-point frequency without changing table order. fn max_frequency_khz(dev: &AsahiDrmDevice) -> Result { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; @@ -948,6 +1100,7 @@ impl Gpu { mmu::UatFirmware::PublishedRoots { secondary_root_offset: cfg.secondary_root_offset, }, + cfg.uat_quirks, )?, GFP_KERNEL, )?; @@ -964,11 +1117,20 @@ impl Gpu { )?; let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; - let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; + let secondary = match cfg.secondary_root_offset { + Some(_) => Some( + Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary).inspect_err(|e| { + dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e) + })?, + ), + None => None, + }; let render_global = uat.new_vm(0, geometry.user_range())?; - let bridge = Bridge::enable(regs)?; + let bridge = match cfg.fabric_bridge { + true => Bridge::enable(regs)?, + false => Bridge::unused(regs), + }; // Install all session ownership before either CPU starts. Error unwinding then runs // Firmware::drop, stopping both CPUs before releasing shared objects or translations. @@ -981,7 +1143,12 @@ impl Gpu { recovery: recovery::State::new(), effort: power::Effort::new(), registration_power_release_pending: false, - queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), + variant: shared.variant, + queues <- runtime::Registry::new( + shared.clusters, + shared.descriptor_flags, + shared.variant, + ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, @@ -994,45 +1161,94 @@ impl Gpu { )?; let session = &mut *firmware; session.primary.boot()?; - session.secondary.boot()?; - let root_entries = session.uat.adopt_firmware_root_entries()?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.boot()?; + } + // The firmware built its root and kernel-window tables while booting. + session.uat.invalidate_reserved_tables(); + let root_entries = match cfg.adopt_root_entries { + true => Some(session.uat.adopt_firmware_root_entries()?), + false => None, + }; session.init.build( dev, cfg, &session.uat, &session._render_global, &session._pm_metrics, + shared.active_cores, + shared.variant, )?; - session.uat.confirm_firmware_root_entries(root_entries)?; + if let Some(root_entries) = root_entries { + session.uat.confirm_firmware_root_entries(root_entries)?; + } session.uat.mirror_secondary_root()?; - session.bridge.regs.enable_g17_gating()?; - session.primary.init(session.init.root_va(Role::Primary)?)?; - session - .secondary - .init(session.init.root_va(Role::Secondary)?)?; - session.primary.notify(MSG_POWER_ON_DONE)?; - session.secondary.notify(MSG_POWER_ON_DONE)?; - Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; + if cfg.fabric_bridge { + session.bridge.regs.enable_g17_gating()?; + } + let ack = shared.variant.init_ack; + session.primary.init(session.init.root_va(Role::Primary)?, ack)?; + if ack == fw::variant::InitAck::Status { + Self::wait_for_init_accepted(dev, &session.init, &session.primary)?; + } + if let Some(secondary) = session.secondary.as_mut() { + secondary.init(session.init.root_va(Role::Secondary)?, ack)?; + } + if shared.variant.power_on_done { + session.primary.notify(MSG_POWER_ON_DONE)?; + if let Some(secondary) = session.secondary.as_mut() { + secondary.notify(MSG_POWER_ON_DONE)?; + } + } + Self::wait_for_opening(dev, &session.init, &session.primary, session.secondary.as_ref())?; session .uat .activate_render_context(&session._render_global)?; Ok(firmware) } + /// Waits for a firmware that sends no acknowledgment message to accept its + /// initialization in status A. + fn wait_for_init_accepted( + dev: &AsahiDrmDevice, + init: &initdata::InitData, + primary: &Coprocessor, + ) -> Result { + for poll in 0..=INIT_ACCEPT_POLLS { + if primary.state.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if init.init_accepted()? { + return Ok(()); + } + if poll < INIT_ACCEPT_POLLS { + fsleep(Delta::from_millis(1)); + } + } + dev_err!( + dev.as_ref(), + "Firmware did not accept its initialization: status A {:#x?}\n", + init.status_read_words::<4>(Role::Primary, 0x10)? + ); + Err(ETIMEDOUT) + } + /// Waits for both coprocessors to retire their opening device-control records. fn wait_for_opening( dev: &AsahiDrmDevice, init: &initdata::InitData, primary: &Coprocessor, - secondary: &Coprocessor, + secondary: Option<&Coprocessor>, ) -> Result { for poll in 0..=OPENING_RETIRE_POLLS { if primary.state.crashed.load(Ordering::Acquire) - || secondary.state.crashed.load(Ordering::Acquire) + || secondary.is_some_and(|s| s.state.crashed.load(Ordering::Acquire)) { return Err(EIO); } - if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { + if init.opening_retired(Role::Primary)? + && (secondary.is_none() || init.opening_retired(Role::Secondary)?) + { return Ok(()); } if poll < OPENING_RETIRE_POLLS { @@ -1100,11 +1316,18 @@ impl gpu::Gpu for Gpu { params.features |= u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); - params.gpu_generation = hw::GpuGen::G17 as u32; + let variant = self.shared.variant; + if variant.few_primitives { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES); + } + if variant.ray_traversal { + params.features |= + u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY); + } + params.gpu_generation = self.cfg.gpu_generation as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; params.chip_id = self.cfg.chip_id; @@ -1144,7 +1367,7 @@ impl gpu::Gpu for Gpu { fn queue_limits(&self) -> Option { Some(uapi::drm_asahi_queue_limits { - max_queues: mmu::MAX_EXECUTION_CONTEXTS, + max_queues: kick::queue_limit(self.shared.variant.qid_floor), max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, ..Default::default() }) @@ -1183,8 +1406,12 @@ impl gpu::Gpu for Gpu { ) -> Result> { let dev = self.shared.drm()?; let owner = self.ids.queue.next(); - let frontend = - queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let frontend = queue::Frontend::new( + priority, + usc_exec_base, + self.geometry.user_usable_range(), + self.shared.variant, + )?; let policy = scheduling_policy(priority)?; let admission_started = now_ns(); let (backend, slot) = loop { diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 7fc4d79a5f95c..dd908aac48c01 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -360,14 +360,20 @@ pub(crate) struct Pool { #[pin] released: CondVar, queues: AtomicU32, + /// Logical queues admitted at most: each must be able to hold a render pair. + max_queues: u32, } impl Pool { - pub(crate) fn new() -> Result> { + pub(crate) fn new(max_queues: u32) -> Result> { + if max_queues > MAX_EXECUTION_CONTEXTS { + return Err(EINVAL); + } Arc::pin_init( pin_init!(Self { state <- new_mutex!(State::new(), "G17 render admission"), released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), + max_queues, }), GFP_KERNEL, ) @@ -379,7 +385,7 @@ impl Pool { pub(crate) fn reserve_queue(self: &Arc) -> Result { let mut count = self.queues.load(Ordering::Acquire); loop { - if count >= MAX_EXECUTION_CONTEXTS { + if count >= self.max_queues { return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::LogicalQueues)); } match self.queues.compare_exchange( @@ -615,7 +621,7 @@ mod accounting_tests { #[test] fn logical_queue_count_returns_on_drop() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); let mut leases = KVec::with_capacity(MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); for _ in 0..MAX_EXECUTION_CONTEXTS { leases.push(pool.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); @@ -732,7 +738,7 @@ impl Pool { impl Pool { pub(crate) fn write_capacity(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - writeln!(f, "logical_queues used={} total={}", self.queues.load(Ordering::Acquire), MAX_EXECUTION_CONTEXTS)?; + writeln!(f, "logical_queues used={} total={}", self.queues.load(Ordering::Acquire), self.max_queues)?; let state = self.state.lock(); let retained = state.owners.iter().enumerate().fold(0u64, |bits, (slot, owner)| { bits | if *owner != 0 { 1u64 << slot } else { 0 } @@ -751,7 +757,7 @@ mod fault_capacity_tests { #[test] fn closed_faulting_clients_return_leases_after_drain() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); for owner in 1..=512 { let queue = pool.reserve_queue().unwrap(); let lease = pool.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); @@ -774,7 +780,7 @@ mod fault_capacity_tests { #[test] fn published_exit_kill_holds_survive_close_and_drain() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); let lease = pool.acquire_render(1, None, u64::MAX, || Ok(())).unwrap(); let bit = 1u64 << lease.slot(); pool.release_owner(1); @@ -816,7 +822,7 @@ mod deadline_tests { #[test] fn all_pairs_busy_no_progress_expires_without_release() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); { let mut state = pool.state.lock(); // No pair has room, including the requesting owner's retained pair. @@ -835,7 +841,7 @@ mod deadline_tests { #[test] fn busy_pair_observes_cancellation_without_a_release_event() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); { let mut state = pool.state.lock(); for slot in 1..cfg::RENDER_SLOTS { @@ -855,7 +861,7 @@ mod deadline_tests { #[test] fn cancellation_precedes_available_capacity() { - let pool = Pool::new().unwrap(); + let pool = Pool::new(MAX_EXECUTION_CONTEXTS).unwrap(); assert!(matches!(pool.acquire_render(1, None, u64::MAX, || Err(ECANCELED)), Err(error) if error == ECANCELED)); assert_eq!(pool.state.lock().active, 0); diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 9ccdd703106b7..bf8d7ffa6afb4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -9,7 +9,12 @@ pub(crate) mod retirement; use crate::g17::{ channel::Rings, context::Context, - fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + fw::{ + channels::ControlRecord, + compute::{OperandState, COMPUTE_DESCRIPTOR_SIZE}, + queue::*, + variant::PointerLayout, + }, object::{Allocator, CpuMap, KernelObject}, }; use crate::{hw::t8140::dynamic, mmu}; @@ -113,17 +118,26 @@ struct Graph { records: KernelObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, + /// Host words of the pointer block in this firmware build. + pointers: PointerLayout, + /// Layout of the operand-state buffer for the GPU's cores. + operand_state: &'static OperandState, } impl Graph { - fn new(alloc: &Allocator<'_>, context: &Context) -> Result { + fn new( + alloc: &Allocator<'_>, + context: &Context, + pointers: PointerLayout, + operand_state: &'static OperandState, + ) -> Result { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; let ring = DescriptorRing::new(alloc)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; - let operand = alloc.lower(0x14000, page, prot, cpu)?; + let operand = alloc.lower(operand_state.size, page, prot, cpu)?; let mut support = alloc.kernel(0xc000, page, prot, cpu)?; let support_va = support.gpu_va(); support.write(0, support_va + page)?; @@ -141,7 +155,7 @@ impl Graph { owner_pid: context.owner_pid(), policy: context.policy(), }); - view.pointers = PointerBlock::new(); + view.pointers = PointerBlock::new(&pointers); view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; @@ -160,6 +174,8 @@ impl Graph { records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, + pointers, + operand_state, }) } @@ -221,7 +237,7 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), + producer: self.graph.word(POINTERS + self.pointers.producer)?.load(Ordering::Relaxed), consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), }) @@ -276,7 +292,9 @@ impl Graph { self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); } fence(Ordering::SeqCst); - self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); + self.graph + .word(POINTERS + self.pointers.producer)? + .store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); Ok(()) @@ -530,9 +548,11 @@ impl Queue { backing: Backing, owner: u64, context: Arc, + pointers: PointerLayout, + operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context)?; + let mut graph = Graph::new(alloc, &context, pointers, operand_state)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; @@ -1607,14 +1627,17 @@ impl Queue { } /// Firmware is halted and publication is closed while its target is copied. pub(crate) fn publish_recovery_target(&self) -> Result { + let Some(target) = self.graph.pointers.recovery_target else { + return Ok(()); + }; let producer = self .graph .graph - .word(POINTERS + PointerBlock::PRODUCER)? + .word(POINTERS + self.graph.pointers.producer)? .load(Ordering::Relaxed); self.graph .graph - .word(POINTERS + PointerBlock::RECOVERY_TARGET)? + .word(POINTERS + target)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -1706,7 +1729,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(POINTERS + self.pointers.producer)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -1812,6 +1835,7 @@ impl Queue { kick: kick_timestamp, preempt_va: addresses.preempt, operand_state_va: addresses.operand, + operand: self.graph.operand_state, usage_va: addresses.usage, usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), @@ -2130,7 +2154,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + 0x40)?; + let producer = self.graph.word(POINTERS + self.pointers.producer)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 84d1bd5a2fd71..6a9b15789339f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -56,10 +56,16 @@ pub(crate) struct Frontend { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, } impl Frontend { - pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { + pub(crate) fn new( + priority: u32, + usc_base: u64, + range: Range, + variant: &'static crate::g17::fw::Variant, + ) -> Result { if priority > 3 { return Err(EINVAL); } @@ -71,6 +77,7 @@ impl Frontend { allocation_failure, range, window, + variant, }) } } @@ -85,6 +92,7 @@ pub(crate) struct Queue { allocation_failure: Fence, range: Range, window: UscWindow, + variant: &'static crate::g17::fw::Variant, proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], @@ -109,6 +117,7 @@ impl Queue { allocation_failure, range, window, + variant, } = frontend; let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); Ok(Self { @@ -121,6 +130,7 @@ impl Queue { allocation_failure, range, window, + variant, proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), @@ -334,6 +344,7 @@ impl Queue { self.context.vm(), self.range.clone(), &self.window, + self.variant, &command.payload, )?; let engine = match &payload { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index b69cd4d565736..66c515f6f59de 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -155,6 +155,7 @@ pub(crate) struct Pair { graph_started: bool, clusters: u32, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], configs: [Option; 2], cursors: [u32; 2], @@ -200,6 +201,7 @@ impl Pair { first: &Arc, clusters: u32, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, ) -> Result> { if slot == 0 || usize::from(slot) >= queues::RENDER_SLOTS @@ -224,7 +226,7 @@ impl Pair { if word == u32::MAX { return Err(EINVAL); } - let layout = target::Layout::new(pass, clusters)?; + let layout = target::Layout::new(pass, clusters, variant.programs)?; let storage = KBox::::new_uninit(GFP_KERNEL)?; let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm(), rings)?; let (fragment, fragment_alias) = @@ -240,6 +242,7 @@ impl Pair { pm_generation, 0, word, + variant, )?; storage.write_init(kernel::try_init!(Self { kick_aliases: Some([ta_alias, fragment_alias]), @@ -256,6 +259,7 @@ impl Pair { graph_started: false, clusters, descriptor_flags, + variant, installed: [false; 2], configs: [None; 2], cursors: [0; 2], @@ -356,14 +360,17 @@ impl Pair { let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; + let deflake_layout = abi::Deflake::new(self.clusters)?; let owners = self.memory.owners(); self.memory .target .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; let target = self.memory.target.prepare(ordinal, &layout)?; let scene = self.memory.manager.prepare_scene(ordinal)?; - self.memory.generation(ordinal)?.prepare(ordinal, pass)?; + self.memory + .generation(ordinal)? + .prepare(ordinal, pass, deflake_layout.viewport)?; let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = self.memory.generation(ordinal)?.addresses(); let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { @@ -393,6 +400,7 @@ impl Pair { heapmeta_va: target.heap, tpc_va: self.memory.target.tpc_va(), tpc_size: layout.tpc_size as u64, + gtp_va: self.memory.target.gtp_va(&layout)?, deflake_va: deflake, ta_status_va: ta_status, fragment_status_va: fragment_status, @@ -424,11 +432,18 @@ impl Pair { stamp_va: addresses.stamps[1][0], aux_stamp_va: addresses.stamps[1][1], }, + programs: self.variant.programs, + deflake: deflake_layout, }; self.memory.graph.claim(ordinal)?; + // Without an MCache table kick-entry bit 58 stays clear. + let attachments = match self.variant.mcache { + true => attachments[1].as_slice(), + false => &[][..], + }; let result = self.memory.write_descriptors( &args, - attachments[1].as_slice(), + attachments, packet.context.id().try_into()?, ); let mcache = match result { @@ -547,6 +562,13 @@ impl Pair { let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); self.memory .completion_links(ordinal, kicks, payload, !covered[1])?; + let leads = match self.variant.lead_commands { + true => { + let init_manager = !self.memory.manager.initialized(); + self.memory.write_leads(ordinal, kicks[0], init_manager)? + } + false => [false; 2], + }; let context = &packet.context; let ring = 1u8 << policy.priority(); let qos = context.qos_id(); @@ -649,6 +671,7 @@ impl Pair { abi::TaDescriptor::register_bindings( self.memory.graph.descriptor_client(0, ordinal), ray_query[0].scratch.enabled(), + self.variant.programs, )?, None, ); @@ -658,6 +681,7 @@ impl Pair { abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), ray_query[1].scratch.enabled(), + self.variant.programs, )?, mcache, ); @@ -679,6 +703,7 @@ impl Pair { configs[0], present[0], kicks[0], + leads[0], )?, publication::Lane::new( &memory.graph, @@ -688,6 +713,7 @@ impl Pair { configs[1], present[1], kicks[1], + leads[1], )?, ]; let windows = [lanes[0].window, lanes[1].window]; @@ -774,7 +800,12 @@ impl Pair { ) })(); let outer_next = match result { - Ok(next) => next, + Ok(next) => { + if leads[0] { + self.memory.manager.mark_initialized(); + } + next + } Err(error) => { self.quarantined = true; if !packet.completion.spared() && self.terminal.is_none() { @@ -987,7 +1018,7 @@ impl Pair { let Validated::Render { pass, .. } = &packet.command else { return Err(EINVAL); }; - let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + let required = target::Layout::new(pass, self.clusters, self.variant.programs)?.tvb_blocks; self.memory.manager.note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); @@ -1703,7 +1734,7 @@ impl Pair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ self.kicks[0].prepare_client_alias(context.vm())?, self.kicks[1].prepare_client_alias(context.vm())?, @@ -1719,6 +1750,7 @@ impl Pair { pm_generation, self.ordinal, first.completion.work_state()?.word(), + self.variant, )?; (Some(memory), Some(persistent), Some(aliases)) }; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 53bc3cfbd6aeb..09ccb689b6632 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -31,6 +31,9 @@ pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; +/// The lead command of a stage (InitBM before tiling, the barrier before fragment), after the +/// stage's configuration record in its slot ([`crate::g17::fw::Variant::lead_commands`]). +pub(super) const LEAD: usize = 0xc0; const DEPTH: usize = queues::RENDER_DEPTH as usize; /// Offsets of the priority pair and the policy word in a queue record. const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); @@ -94,6 +97,8 @@ pub(super) struct Graph { pub(super) config: PooledObject, pub(super) support: KernelObject, pub(super) owners: [Option; RECORDS], + /// Host words of the pointer blocks in this firmware build. + pub(super) pointers: fw::variant::PointerLayout, } impl Graph { pub(super) fn new( @@ -101,6 +106,7 @@ impl Graph { pool: &Arc, context: &Context, qids: [u8; 2], + pointers: fw::variant::PointerLayout, ) -> Result { let prot = mmu::PROT_GPU_FW_SHARED_RW; let descriptors = pool.allocate( @@ -154,7 +160,7 @@ impl Graph { }), )?; queues.initialize::(POINTERS[stage], |view| { - *view = queue::PointerBlock::new(); + *view = queue::PointerBlock::new(&pointers); Ok(()) })?; } @@ -166,6 +172,7 @@ impl Graph { config, support, owners: [None; RECORDS], + pointers, }) } pub(super) fn offset(stage: usize, ordinal: u64) -> usize { @@ -188,11 +195,13 @@ impl Graph { self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 } pub(super) fn publish_recovery_targets(&self) -> Result { + let Some(target) = self.pointers.recovery_target else { + return Ok(()); + }; for at in POINTERS { - let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? + let producer = self.queues.word(at + self.pointers.producer)? .load(Ordering::Relaxed); - self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? - .store(producer, Ordering::Relaxed); + self.queues.word(at + target)?.store(producer, Ordering::Relaxed); } Ok(()) } @@ -208,7 +217,7 @@ impl Graph { values[stage] = Cursors { done: self.queues.word(at)?.load(Ordering::Relaxed), read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), - write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), + write: self.queues.word(at + self.pointers.producer)?.load(Ordering::Relaxed), }; } Ok(values) @@ -310,24 +319,36 @@ impl Generation { ) .map_err(Into::into) } - pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { + /// With `viewport` false the deflake buffer has no room for the viewport words + /// ([`abi::Deflake::viewport`]). + pub(super) fn prepare( + &mut self, + ordinal: u64, + pass: &abi::RenderPass, + viewport: bool, + ) -> Result { if self.ordinal.is_some_and(|old| old != ordinal) { return Err(EBUSY); } - let viewport = storage::Viewport::new(pass.width, pass.height)?; + let viewport = match viewport { + true => Some(storage::Viewport::new(pass.width, pass.height)?), + false => None, + }; self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] .fill(0); - let destination = self - .state - .pointer(storage::DEFLAKE + 0x900, size_of::())? - .cast::(); - if destination.align_offset(align_of::()) != 0 { - return Err(EINVAL); + if let Some(viewport) = viewport { + let destination = self + .state + .pointer(storage::DEFLAKE + 0x900, size_of::())? + .cast::(); + if destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + // SAFETY: The complete typed integer view is in the retained CPU mapping + // and naturally aligned. This command generation is exclusively owned + // and unpublished; no prior command can access it until reassigned. + unsafe { viewport.apply(&mut *destination) }; } - // SAFETY: The complete typed integer view is in the retained CPU mapping - // and naturally aligned. This command generation is exclusively owned - // and unpublished; no prior command can access it until reassigned. - unsafe { viewport.apply(&mut *destination) }; self.fragment.bytes_mut().fill(0); self.ordinal = Some(ordinal); Ok(()) @@ -380,10 +401,11 @@ impl Memory { pm_generation: u64, ordinal: u64, word: u32, + variant: &'static fw::Variant, ) -> Result<(KBox, KVec)> { let storage = KBox::::new_uninit(GFP_KERNEL)?; let vm = context.vm(); - let mut graph = Graph::new(alloc, pool, &context, qids)?; + let mut graph = Graph::new(alloc, pool, &context, qids, variant.pointers)?; let mut target = target::Target::new(alloc.dev, vm, layout)?; target.prepare(ordinal, layout)?; let state = Generation::allocate_state(alloc, vm)?; @@ -509,6 +531,9 @@ const _: () = { assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); assert!(RECORDS * CONFIG_STRIDE <= 0x4000); + assert!(size_of::() <= LEAD); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); + assert!(LEAD + size_of::() <= CONFIG_STRIDE); }; impl Memory { @@ -602,6 +627,52 @@ impl Memory { } Ok(()) } + + /// Writes the lead commands of `ordinal` from its linked descriptors: InitBM before the + /// tiling descriptor when `init_manager`, and the fragment barrier. Returns which stages + /// have one. + pub(super) fn write_leads( + &mut self, + ordinal: u64, + tiling_kick: fw::kick::KickTimestamp, + init_manager: bool, + ) -> Result<[bool; 2]> { + if !self.owns(ordinal) { + return Err(EIO); + } + let ta = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + // SAFETY: Both views were checked during preparation and the ordinal still owns them; + // only integer fields are read. + let (ta, fragment_id) = unsafe { (&*ta, (*fragment).descriptor.current_id) }; + let slot = ordinal as usize % RECORDS * CONFIG_STRIDE + LEAD; + let barrier = queue::FragmentBarrier::new( + ta.stamp_va, + tiling_kick, + ta.qid, + ta.stamp_value, + fragment_id as u32, + ); + self.graph.config.write(CONFIG[1] + slot, barrier)?; + if init_manager { + let init = queue::InitBufferManager::new( + ta.context_id, + ta.pb_slot, + ta.pb_va, + ta.stamp_value, + ); + self.graph.config.write(CONFIG[0] + slot, init)?; + } + Ok([init_manager, true]) + } } impl Memory { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 5b3f2a00bad6d..ce77d870de641 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -26,6 +26,7 @@ pub(crate) struct ParkedPair { ordinal: u64, clusters: u32, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, installed: [bool; 2], recovery_generation: u64, registered: Option, @@ -91,6 +92,7 @@ impl Pair { ordinal: self.ordinal, clusters: self.clusters, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, recovery_generation: self.recovery_generation, registered: self.registered, @@ -151,7 +153,7 @@ impl ParkedPair { return Err(EINVAL); } alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters)?; + let layout = target::Layout::new(pass, self.clusters, self.variant.programs)?; let aliases = [ kicks[0].prepare_client_alias(context.vm())?, kicks[1].prepare_client_alias(context.vm())?, @@ -167,6 +169,7 @@ impl ParkedPair { pm_generation, self.ordinal, word, + self.variant, )?; Ok(Prepared { memory: Some(memory), @@ -230,6 +233,7 @@ impl ParkedPair { initial_packet: next.initial_packet, clusters: self.clusters, descriptor_flags: self.descriptor_flags, + variant: self.variant, installed: self.installed, configs: [None; 2], cursors: [0; 2], diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 72d36afcf5b87..9f4cb05b6bede 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -28,9 +28,9 @@ pub(super) fn required(previous: Option, config: QueueConfig) -> bo } pub(super) struct Lane<'a> { - items: [&'a AtomicU64; 3], + items: [&'a AtomicU64; 4], producer: &'a AtomicU32, - values: [u64; 3], + values: [u64; 4], config: QueueConfig, config_destination: *mut u64, config_present: bool, @@ -49,8 +49,9 @@ impl<'a> Lane<'a> { config: QueueConfig, present: bool, timestamp: KickTimestamp, + lead: bool, ) -> Result { - let count = 2 + u32::from(present); + let count = 2 + u32::from(present) + u32::from(lead); let window = Window::new(base, count, ITEM_RING_ENTRIES)?; let offset = ordinal as usize % memory::RECORDS; let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; @@ -72,21 +73,28 @@ impl<'a> Lane<'a> { graph.queues.dword(index(0))?, graph.queues.dword(index(1))?, graph.queues.dword(index(2))?, + graph.queues.dword(index(3))?, ]; let announce_va = graph.queues.gpu_va() + announce_offset as u64; let descriptor = graph.descriptor_va(stage, ordinal); - let values = if present { - [ - descriptor, - graph.config.gpu_va() + config_offset as u64, - announce_va, - ] - } else { - [descriptor, announce_va, 0] + // Item order: [lead command], descriptor, [configuration], announcement. + let mut values = [0; 4]; + let mut n = 0; + let mut push = |value: u64| { + values[n] = value; + n += 1; }; + if lead { + push(graph.config.gpu_va() + (config_offset + memory::LEAD) as u64); + } + push(descriptor); + if present { + push(graph.config.gpu_va() + config_offset as u64); + } + push(announce_va); Ok(Self { items, - producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, + producer: graph.queues.word(memory::POINTERS[stage] + graph.pointers.producer)?, values, config, config_destination, diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs index 4db471f254ec6..f718cd47f90cf 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -36,10 +36,11 @@ impl Pair { } *block = pointer.cast(); } + let layout = self.memory.graph.pointers; for block in blocks { // SAFETY: Both complete pointer blocks were checked above. This fresh // graph is unpublished; only the live atomic cursors are accessed. - unsafe { &*block }.rebase(0); + unsafe { &*block }.rebase(&layout, 0); } fence(Ordering::SeqCst); self.cursors = [0; 2]; diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index c23a5f0c13d07..89f8d33bdb80c 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -13,8 +13,8 @@ mod safety; pub(crate) use attribution::Sources; use attribution::Attribution; -pub(super) fn slot_key(slot: usize) -> Option { - slot::key(slot).ok() +pub(super) fn slot_key(slot: usize, slot_keys: Option) -> Option { + slot::key(slot, slot_keys?).ok() } use super::status::VmStatus; diff --git a/drivers/gpu/drm/asahi/g17/recovery/slot.rs b/drivers/gpu/drm/asahi/g17/recovery/slot.rs index 238b269747511..bd9d4c96be14f 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/slot.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/slot.rs @@ -2,7 +2,7 @@ //! Bounded read-only slot identity sampling while firmware is halted. -use crate::{hw::t8140, mmu}; +use crate::mmu; use core::{ ffi::c_void, ptr::NonNull, @@ -19,11 +19,11 @@ impl Drop for Mapping { } } -pub(super) fn key(slot: usize) -> Result { +pub(super) fn key(slot: usize, slot_keys: u64) -> Result { if slot >= super::attribution::SLOTS { return Err(EINVAL); } - let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; + let physical = slot_keys + slot as u64 * size_of::() as u64; let offset = physical as usize & (mmu::UAT_PGSZ - 1); // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here // for mapping. Only the selected aligned eight-byte word is read below. diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index a9622d5909499..cc3e4b3044b03 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -72,16 +72,17 @@ impl Registry { pub(super) fn new( clusters: u32, descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, ) -> impl Init { kernel::try_init!(Self { - qids <- kick::QueueIds::new().chain(|qids| { + qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { qids.reserve_initial_render() }), buffers <- BufferIds::new(), accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::new()?, + admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, render: render::RenderState::new(clusters, descriptor_flags)?, identities: queue::Identities::new(), compute: { @@ -410,9 +411,10 @@ pub(super) enum Rebind { #[kunit_tests(asahi_g17_uninstalled_cleanup)] mod uninstalled_cleanup_tests { use super::*; + use crate::g17::fw::variant::V26A428; #[test] fn empty_reservation_prevents_reuse_until_explicit_cancellation() { - let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut registry = KBox::init(Registry::new(1, [0; 2], &V26A428), GFP_KERNEL).unwrap(); let first = registry.reserve_compute(1).unwrap(); assert!(registry.reserved(first).is_ok()); assert!(registry.reserve_compute(1).is_err()); @@ -871,7 +873,14 @@ impl Backend { // The global scheduler is created after pool/kick backing and // retained even if this queue's later descriptor graph fails. self.compute_scheduler(&alloc, &lease, admission)?; - let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; + let queue = compute::Queue::new( + &alloc, + backing, + self.owner, + self.context.clone(), + self.shared.variant.pointers, + self.shared.operand_state, + )?; Ok(KBox::new(queue, GFP_KERNEL)?) })(); let (mut built, build_error) = match built { @@ -1466,7 +1475,9 @@ impl super::Shared { { let mut state = self.state.lock(); if let Some(firmware) = state.as_deref_mut() { - firmware.secondary.stop(); + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } firmware.primary.stop(); } } @@ -1490,7 +1501,9 @@ impl super::Shared { let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { - firmware.secondary.stop(); + if let Some(secondary) = firmware.secondary.as_mut() { + secondary.stop(); + } firmware.primary.stop(); } state.take() @@ -1639,10 +1652,11 @@ impl Backend { #[kunit_tests(asahi_g17_shared_capacity)] mod shared_capacity_tests { use super::*; + use crate::g17::fw::variant::V26A428; #[test] fn all_logical_render_owners_fit_beside_shared_compute() { - let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut registry = KBox::init(Registry::new(1, [0; 2], &V26A428), GFP_KERNEL).unwrap(); let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); // Physical compute channels are device-owned and bounded independently // of how many logical clients have queued unsignalled work. @@ -1660,9 +1674,30 @@ mod shared_capacity_tests { // published identity was cancelled or retyped to achieve the fit. } + #[test] + fn a_qid_floor_admits_only_the_render_pairs_it_leaves() { + use crate::g17::fw::variant::V25G83; + let limit = super::super::kick::queue_limit(V25G83.qid_floor); + assert_eq!(limit, 42); + let mut registry = KBox::init(Registry::new(2, [0; 2], &V25G83), GFP_KERNEL).unwrap(); + for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + registry.reserve_compute(owner).unwrap(); + } + let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); + for owner in 1..=u64::from(limit) { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); + registry.ensure_render_ids(lease.slot(), owner).unwrap(); + leases.push(lease, GFP_KERNEL).unwrap(); + } + // The next logical queue is refused when it is created, not left waiting for a pair. + assert!(matches!(registry.admission.reserve_queue(), Err(ENOSPC))); + } + #[test] fn four_clients_have_capacity_before_any_owner_closes() { - let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut registry = KBox::init(Registry::new(1, [0; 2], &V26A428), GFP_KERNEL).unwrap(); let mut leases = KVec::with_capacity(36, GFP_KERNEL).unwrap(); for client in 0..4u64 { for queue in 0..9u64 { diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index f8fcf253b41a5..4fd6e56741986 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -24,6 +24,9 @@ struct Service<'a> { } impl event::Transport for Service<'_> { + fn roles(&self) -> &'static [Role] { + self.firmware.roles() + } fn trace(&mut self, role: Role) { let firmware = &mut *self.firmware; firmware @@ -445,7 +448,8 @@ impl crate::g17::Shared { } else if !recover && !polling && (channel::Events::pending(&firmware.init, Role::Primary)? - || channel::Events::pending(&firmware.init, Role::Secondary)?) + || (firmware.secondary.is_some() + && channel::Events::pending(&firmware.init, Role::Secondary)?)) { // A full bounded pass must not depend on a fresh notification for // records that arrived while it was consuming the same ring. diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 88340442bbcae..a399a1532550f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -527,6 +527,11 @@ impl crate::g17::queue::render::Host for Host<'_> { /// check let the firmware gate the cores between the host's configuration /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + // Firmware builds without MMIO registration install the pair from its queue + // configuration alone. + if !self.firmware.variant.mmio_pair_registration { + return Ok(()); + } if !self.registration_powered { self.firmware.acquire_registration_power()?; self.registration_powered = true; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 2acd464bbeea4..18c6abf294fc4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -36,7 +36,7 @@ impl BuildResources { metrics_ids: firmware.queues.metrics.clone(), metrics_bases: crate::g17::fw::buffer::MetricsAddresses { firmware: firmware.init.metrics_fw_va()?, - client: crate::hw::t8140::CONFIG.pm_metrics.va, + client: firmware.init.pm_metrics_va(), }, config: firmware.queues.render.config, }) @@ -275,6 +275,7 @@ impl super::super::Backend { first, resources.config.clusters, resources.config.descriptor_flags, + self.shared.variant, ) })(); let (mut built, error) = match built { diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index da43c10cd3bef..b2a6cf6e8bfe3 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -90,10 +90,11 @@ impl Proof { vm: &mmu::Vm, range: Range, window: &UscWindow, + variant: &crate::g17::fw::Variant, payload: &Payload, ) -> Result { let requests = RefCell::new(Requests::new()); - let optimistic = payload.validate(window, &Space::Record(&range, &requests)); + let optimistic = payload.validate(window, variant, &Space::Record(&range, &requests)); let requests = requests.into_inner(); if optimistic.is_ok() && !requests.overflow { if self.covers(vm.mapping_validation_epoch(), &requests) { @@ -107,7 +108,7 @@ impl Proof { self.epoch = None; } // Preserve per-range validation's failure and concurrent-mutation behavior. - payload.validate(window, &Space::Checked(&range, vm)) + payload.validate(window, variant, &Space::Checked(&range, vm)) } } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index e9e207f0ad3d8..b68381ce60b30 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -747,6 +747,7 @@ impl GpuManager::ver { }, map_kernel_to_user, mmu::UatFirmware::Handoff, + mmu::UatQuirks::DEFAULT, )?, GFP_KERNEL, )?) diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index deffe3fe47d0b..2650131a4a293 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -13,6 +13,7 @@ const MAX_POWERZONES: usize = 5; pub(crate) mod t600x; pub(crate) mod t602x; +pub(crate) mod t6040; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; @@ -23,6 +24,7 @@ pub(crate) mod t8140; pub(crate) enum GpuGen { G13 = 13, G14 = 14, + G16 = 16, G17 = 17, } diff --git a/drivers/gpu/drm/asahi/hw/t6040.rs b/drivers/gpu/drm/asahi/hw/t6040.rs new file mode 100644 index 0000000000000..db0c33fff8485 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6040.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t6040 platforms (M4 Pro). +//! +//! The M4 Pro runs the macOS 26.6 GPU firmware (25G83), which has the A18 Pro firmware's +//! interface; [`fw::variant::V25G83`] describes where it differs. + +use crate::g17::fw::{ + self, + initdata::{v25g83, PerfConfig, RegionCConfig, RegisterWindow, PERF_STATES}, +}; +use crate::g17::{Config, CrashBuffers, FixedBuffer, Identity}; +use crate::hw::{GpuGen, GpuVariant}; +use crate::mmu::UatQuirks; + +/// Static configuration of the T6040 GPU. +pub(crate) const CONFIG: Config = Config { + chip_id: CHIP_ID, + identity: Identity { + family: 0xa, + // ID register 0x0a03_1100: family 0xa, variant 3, revision 0x11 (B1). + variant: 3, + num_dies: 1, + gpu_variant: GpuVariant::S, + usc_generation: 3, + hal_generation: 200, + }, + uat_ias: 42, + uat_oas: 42, + // One coprocessor carries both firmware roles; `pagetables` holds only its root. + secondary_root_offset: None, + crash_buffers: CrashBuffers::FirmwareData, + // Read by the firmware through the kernel lower root, whose top-level entries it only + // samples when it starts. + pm_metrics: FixedBuffer { + va: 0x10_0008_0000, + size: 0x8000, + }, + windows: ®ISTER_WINDOWS, + reserved_slots: &[], + perf: &PERF, + opps: OPPS, + // The no-map tables are needed until the boot loader adds the UAT ranges to /memory. The + // rest is the policy the M4 Pro has run with; none of it was tested otherwise. + uat_quirks: UatQuirks { + reserved_tables: true, + clean_tables: true, + page_tlbi: true, + handoff_lock: false, + handoff_slot: 0xffff_ffff, + native_gpu_permissions: true, + }, + region_c: ®ION_C, + adopt_root_entries: false, + fabric_bridge: false, + // The firmware's per-slot recovery keys are not known for this build; recovery relies on + // the firmware's own blame. + recovery_slot_keys: None, + // Userspace drives this GPU as a G16 part. + gpu_generation: GpuGen::G16, + // The macOS 26.6.2 boot firmware (iBoot mBoot-18000.161.10) loads the 25G83 GPU firmware. + firmware: fw::variant::Firmware::Identified(&[fw::variant::Build { + os_version: [26, 6, 2], + name: "macOS 26.6.2 (25G83, iBoot mBoot-18000.161.10)", + variant: &fw::variant::V25G83, + }]), +}; + +/// Chip identifier written to the hardware data. +pub(crate) const CHIP_ID: u32 = 0x6040; + +/// Base of the firmware MMIO aperture in the kernel window. +const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; + +const fn window( + slot: usize, + phys: u64, + offset: u64, + size: u32, + unk_18: u64, + unk_20: u32, +) -> RegisterWindow { + RegisterWindow { + slot, + phys, + va: MMIO_APERTURE + offset, + size, + unk_18, + unk_20, + instances: 1, + phys_stride: 0, + } +} + +/// `window` repeated `instances` times, `phys_stride` apart in physical memory. +const fn instances(window: RegisterWindow, instances: u32, phys_stride: u64) -> RegisterWindow { + RegisterWindow { + instances, + phys_stride, + ..window + } +} + +/// Register windows the firmware maps, by register-map slot. Slot 3 holds the four memory +/// controllers (ADT /arm-io/mcc), slot 20 the four AFR blocks. +pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 25] = [ + window(0, 0x5_0241_4000, 0x20_0000, 0x4000, 0, 2), + instances(window(3, 0x2_201c_4000, 0x20_8000, 0x1_8000, 0, 2), 4, 0x200_0000), + window(4, 0x5_0244_c000, 0x26_c000, 0x4000, 0, 0), + window(9, 0x5_003d_0000, 0x27_4000, 0x4000, 0, 2), + window(10, 0x5_003c_0000, 0x27_c000, 0x4000, 0, 0), + window(11, 0x5_003d_8000, 0x28_4000, 0x4000, 0, 2), + window(12, 0x4_8565_c000, 0x28_c000, 0x4000, 0, 2), + window(14, 0x5_0040_0000, 0x29_4000, 0x1_0000, 0, 0), + window(15, 0x5_0cdc_0000, 0x2a_8000, 0x4000, 0, 0), + window(17, 0x2_8800_0000, 0x2b_0000, 0x2_1400, 0, 2), + instances(window(20, 0x2_8b00_0000, 0x2d_8000, 0x4_0000, 0, 2), 4, 0x20_0000), + window(22, 0x2_8900_0000, 0x3d_c000, 0x8000, 0, 2), + window(26, 0x2_88d0_4000, 0x3e_8000, 0x8000, 0xd0_4000, 2), + window(28, 0x2_88d5_8000, 0x3f_4000, 0x8000, 0xd5_8000, 2), + window(29, 0x2_88d1_0000, 0x40_0000, 0x4000, 0xd1_0000, 2), + window(31, 0x2_88d4_0000, 0x40_8000, 0x4000, 0xd4_0000, 2), + window(32, 0x2_88d6_0000, 0x41_0000, 0x2_0000, 0xd6_0000, 2), + window(33, 0x2_88d4_4000, 0x43_4000, 0x4000, 0xd4_4000, 2), + window(34, 0x2_88d4_c000, 0x43_c000, 0x200, 0xd4_c000, 2), + window(35, 0x2_88e0_0000, 0x44_4000, 0x4000, 0xe0_0000, 2), + window(37, 0x2_88e4_0000, 0x44_c000, 0x4000, 0xe4_0000, 2), + window(39, 0x2_88e0_8000, 0x45_4000, 0x8000, 0, 2), + window(40, 0x2_88e1_c000, 0x46_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x2_88e1_f800, 0x46_b800, 0x4000, 0, 2), + window(43, 0x2_88e6_0000, 0x47_4000, 0x58, 0xe6_0000, 2), +]; + +/// Performance-table configuration. The device tree lists the sixteen operating points in +/// performance-state order, with one voltage per cluster. +pub(crate) const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 6, 8, 10, 12, 14, 15], + states_b: [0, 1, 2, 3, 4, 6, 7, 9, 11, 13, 15], + relative_a: [0, 9, 16, 23, 29, 38, 49, 60, 69, 79, 100], + relative_b: [0, 0, 22, 36, 47, 57, 68, 78, 84, 91, 100], + sram_min_mv: 810, +}; + +/// Operating points the performance tables reference. +pub(crate) const OPPS: usize = 16; + +const _: () = { + let mut index = 0; + while index < PERF_STATES { + assert!((PERF.states_a[index] as usize) < OPPS); + assert!((PERF.states_b[index] as usize) < OPPS); + index += 1; + } +}; + +/// Region C. +const REGION_C: RegionCConfig = RegionCConfig { + unk_54: 0x0fa0, + unk_78: [1, 0, 0], + max_power_mw: v25g83::MAX_POWER_MW, + unk_9c: 0, + unk_a0: 0, + unk_7c4: 1500, + unk_9e0: [ + 1, + 0x0102_0200, + 0x010a_2801, + 0x2802_0101, + 0x0101_010a, + 0x010a_2802, + 0x2801_0101, + 0x0001_010a, + ], +}; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 7c6975b71a4c8..f5ed7bc80963b 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -4,10 +4,11 @@ use crate::g17::{ Config, + CrashBuffers, FixedBuffer, Identity, // }; -use crate::hw::GpuVariant; +use crate::hw::{GpuGen, GpuVariant}; /// Static configuration of the T8140 GPU. pub(crate) const CONFIG: Config = Config { @@ -23,10 +24,10 @@ pub(crate) const CONFIG: Config = Config { uat_ias: 42, uat_oas: 42, // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. - secondary_root_offset: 0x40000, + secondary_root_offset: Some(0x40000), // The coprocessors request their crash log buffers at these addresses, in this order, and // expect them to be mapped before they start. - crash_buffers: [ + crash_buffers: CrashBuffers::Fixed([ FixedBuffer { va: 0x01e1_c000, size: 0x8000, @@ -35,13 +36,24 @@ pub(crate) const CONFIG: Config = Config { va: 0x01f5_0000, size: 0x8000, }, - ], + ]), // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. pm_metrics: FixedBuffer { va: 0x10_0008_0000, size: 0x8000, }, + recovery_slot_keys: Some(recovery::SLOT_KEYS), + gpu_generation: GpuGen::G17, + windows: ®ISTER_WINDOWS, + reserved_slots: &RESERVED_REGISTER_SLOTS, + perf: &PERF, + opps: OPPS, + uat_quirks: crate::mmu::UatQuirks::DEFAULT, + region_c: &crate::g17::fw::initdata::REGION_C_26A428, + adopt_root_entries: true, + fabric_bridge: true, + firmware: crate::g17::fw::variant::Firmware::Fixed(&crate::g17::fw::variant::V26A428), }; use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; @@ -67,6 +79,8 @@ const fn window( size, unk_18, unk_20, + instances: 1, + phys_stride: 0, } } diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 31c41221967fa..91fd5092aaefc 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -15,9 +15,35 @@ use kernel::page::PAGE_SIZE; use crate::debug::*; use crate::mmu; +use core::sync::atomic::{AtomicBool, Ordering}; type Asid = u8; +/// Invalidate ranges page by page (`mmu::UatQuirks::page_tlbi`): never range operations. +static PAGE_TLBI: AtomicBool = AtomicBool::new(false); + +/// Ranges above this many pages are invalidated by ASID in page mode. +const PAGE_TLBI_MAX_PAGES: usize = 2048; + +/// Selects page-by-page range invalidation for this GPU. +pub(crate) fn set_page_tlbi(enabled: bool) { + PAGE_TLBI.store(enabled, Ordering::Relaxed); +} + +/// Invalidates `len` bytes at `va` one 16 KiB page at a time, or the whole ASID above +/// [`PAGE_TLBI_MAX_PAGES`]. +fn tlbi_pages(asid: Asid, va: usize, len: usize) { + let start = va >> mmu::UAT_PGBIT; + let end = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; + if end - start > PAGE_TLBI_MAX_PAGES { + tlbi_asid(asid); + return; + } + for page in start..end { + tlbi_page(asid, page << mmu::UAT_PGBIT); + } +} + /// Invalidate the entire GPU TLB. #[inline(always)] pub(crate) fn tlbi_all() { @@ -78,6 +104,10 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } let start_pg = va >> mmu::UAT_PGBIT; let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; @@ -167,6 +197,10 @@ pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { if len == 0 { return; } + if PAGE_TLBI.load(Ordering::Relaxed) { + tlbi_pages(asid, va, len); + return; + } if va == 0 || len >= TLBI_RANGE_MAX { tlbi_asid(asid); diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 7fa207c9ff5e4..f981b1c88496c 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -154,20 +154,59 @@ pub(crate) enum UatFirmware { /// One coprocessor that publishes its magic number in the handoff region, arbitrates the /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). Handoff, - /// Two coprocessor instances that never touch the handoff region (G17). + /// One or two coprocessor instances that never touch the handoff region (G17). /// /// Each instance has its own kernel root and writes the top-level entries below and including /// the driver-managed kernel window into it while it boots: the lower ones are private to the /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, the secondary one the root at - /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel - /// context. + /// primary instance uses the `pagetables` root, a secondary one (A18 Pro) the root at + /// `secondary_root_offset` in the same region; a single-instance firmware (M4 Pro) has none. + /// TTBAT context 1 starts as an alias of the kernel context. PublishedRoots { /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: u64, + secondary_root_offset: Option, }, } +/// UAT policy of a SoC where it differs from the A18 Pro's. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UatQuirks { + /// The firmware builds its root and kernel-window tables in the no-map `pagetables` and + /// `l2-pagetables` regions: reach tables there through a write-back mapping instead of + /// `struct page`. + /// + /// Temporary: once the boot loader adds these regions to `/memory`, as m1n1 does, the + /// regions lose `no-map` and this path goes away. + pub(crate) reserved_tables: bool, + // The following settings keep a SoC on the only policy it was tested with. Each one is + // a candidate for the A18 Pro's value once a test on that SoC shows it is not needed. + /// Write back every page-table and context-table change: the GPU table walker is not + /// assumed to snoop the CPU caches. + pub(crate) clean_tables: bool, + /// Invalidate ranges one page at a time, never with range TLBI operations. + pub(crate) page_tlbi: bool, + /// Take the handoff lock around context-table edits. Where the firmware leaves its half of + /// the handoff page alone, its lock bytes are not initialized and are never taken. + pub(crate) handoff_lock: bool, + /// Handoff "current slot" written at initialization. + pub(crate) handoff_slot: u32, + /// Store GPU-only mappings of client and kernel-lower tables as AP 0 with the access in + /// PXN (read) and UXN (write), the form the firmware itself uses, instead of AP 2. + pub(crate) native_gpu_permissions: bool, +} + +impl UatQuirks { + /// The A18 Pro and earlier policy. + pub(crate) const DEFAULT: UatQuirks = UatQuirks { + reserved_tables: false, + clean_tables: false, + page_tlbi: false, + handoff_lock: true, + handoff_slot: 0, + native_gpu_permissions: false, + }; +} + const TTBR_VALID: u64 = 0x1; // BIT(0) const TTBR_ASID_SHIFT: usize = 48; @@ -1022,7 +1061,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -1030,8 +1069,9 @@ impl ExecutionRoot { if owned { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); + inner.publish_slot(id); } - inner.handoff().unlock(); + inner.handoff_unlock(); owned || (low == 0 && high == 0) }; if release { @@ -1371,10 +1411,10 @@ impl KernelMapping { let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff().lock(); + uat_inner.handoff_lock(); let cur_slot = uat_inner.handoff().current_slot(); let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff().unlock(); + uat_inner.handoff_unlock(); if cur_slot == Some(slot) && ttb_cur == ttb { slot } else { @@ -1590,9 +1630,33 @@ struct UatShared { map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, + quirks: UatQuirks, } impl UatShared { + /// Takes the handoff lock, where this SoC uses it. + fn handoff_lock(&self) { + if self.quirks.handoff_lock { + self.handoff().lock(); + } + } + + /// Releases the handoff lock taken by [`Self::handoff_lock`]. + fn handoff_unlock(&self) { + if self.quirks.handoff_lock { + self.handoff().unlock(); + } + } + + /// Writes back context-table slot `idx` after it changed, where this SoC needs it. + fn publish_slot(&self, idx: usize) { + if self.quirks.clean_tables { + let slot = &self.ttbs()[idx] as *const SlotTTBS as *const u8; + pgtable::clean_poc(slot, size_of::()); + pgtable::poc_barrier(); + } + } + /// Returns the handoff region area fn handoff(&self) -> &Handoff { // SAFETY: pointer is non-null per the type invariant @@ -1636,9 +1700,12 @@ struct UatInner { /// An uncertain mapping/root is retained; future M4 allocation and publication fail. failed: AtomicBool, /// Clean page tables to the point of coherency for a walker that does not snoop the CPU - /// caches. Set only by [`Uat::new_adopted`]. - #[cfg(CONFIG_DRM_ASAHI_G16)] + /// caches: [`Uat::new_adopted`], and [`UatQuirks::clean_tables`] otherwise. clean_poc: bool, + /// This SoC's UAT policy. + quirks: UatQuirks, + /// No-map reserved regions holding firmware tables (`UatQuirks::reserved_tables`). + reserved: Option>, #[pin] shared: Mutex, #[pin] @@ -1703,7 +1770,7 @@ impl HandoffGuard<'_> { impl Drop for HandoffGuard<'_> { fn drop(&mut self) { if let Self::Legacy(shared) = self { - shared.handoff().unlock(); + shared.handoff_unlock(); } } } @@ -1737,7 +1804,7 @@ impl UatInner { let shared = self.lock(); self.check_m4()?; if !self.m4_adopted { - shared.handoff().lock(); + shared.handoff_lock(); return Ok(HandoffGuard::Legacy(shared)); } let transport = M4Transport { @@ -1812,9 +1879,9 @@ impl Handoff { } /// Initialize the handoff region - fn init(&self, firmware: UatFirmware) -> Result { + fn init(&self, firmware: UatFirmware, slot: u32) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(0, Ordering::Relaxed); + self.cur_slot.store(slot, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); fence(Ordering::SeqCst); @@ -1949,11 +2016,12 @@ impl Vm { geometry.ias, geometry.oas, uat_inner.firmware != UatFirmware::Handoff, + uat_inner.reserved.clone(), )? } else { UatPageTable::new(geometry.ias, geometry.oas)? + .with_native_gpu(uat_inner.quirks.native_gpu_permissions) }; - #[cfg(CONFIG_DRM_ASAHI_G16)] let page_table = { let mut page_table = page_table; if uat_inner.clean_poc { @@ -2749,6 +2817,7 @@ impl Drop for VmInner { uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); uat_inner.owned_user_roots[idx] = 0; + uat_inner.publish_slot(idx); } drop(transaction); @@ -2865,6 +2934,8 @@ impl Uat { failed: AtomicBool::new(false), // The G15 UAT walker does not snoop the CPU caches. clean_poc: true, + reserved: None, + quirks: UatQuirks::DEFAULT, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -2880,6 +2951,7 @@ impl Uat { map_kernel_to_user, handoff_rgn, ttbs_rgn, + quirks: UatQuirks::DEFAULT, }, "uat_shared" ), @@ -3084,6 +3156,7 @@ impl Uat { uat_inner.binding_published[idx] = false; uat_inner.owned_user_roots[idx] = ttb; binding.generation = binding_wire_generation(incarnation); + uat_inner.publish_slot(idx); drop(transaction); // Make sure all TLB entries from the previous owner of this ASID are gone @@ -3183,6 +3256,7 @@ impl Uat { } roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); + inner.publish_slot(id); } mem::sync(); mem::tlbi_asid(id as u8); @@ -3224,7 +3298,7 @@ impl Uat { // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { for (value, pte) in snapshot.iter_mut().zip(ptes) { *value = pte.load(Ordering::Relaxed); } @@ -3244,7 +3318,7 @@ impl Uat { pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. let matches = unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + pgtable::with_root_entries(self.inner.reserved.as_deref(), self.kernel_vm.ttb(), snapshot.len(), |ptes| { ptes.iter() .zip(snapshot) .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) @@ -3260,15 +3334,19 @@ impl Uat { /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are /// private to that instance. pub(crate) fn mirror_secondary_root(&self) -> Result { - let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; + let Some(secondary) = self.inner.lock().secondary_ttb1 else { + // A single-instance firmware has no secondary root to mirror. + return Ok(()); + }; let count = UAT_PGSZ / size_of::(); let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; + let reserved = self.inner.reserved.as_deref(); // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives // this `Uat`. unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(secondary, count, |secondary| { + pgtable::with_root_entries(reserved, self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(reserved, secondary, count, |secondary| { for (src, dst) in primary.iter().zip(secondary).skip(first) { let pte = src.load(Ordering::Relaxed); if pte != 0 && dst.load(Ordering::Relaxed) != pte { @@ -3285,24 +3363,33 @@ impl Uat { Ok(()) } + /// Discards the CPU's cached lines of the no-map table regions after the firmware built its + /// root there during boot, before the first host edit (`UatQuirks::reserved_tables`). + pub(crate) fn invalidate_reserved_tables(&self) { + if let Some(reserved) = self.inner.reserved.as_ref() { + reserved.invalidate(); + } + } + /// Activates the device-global render root in context 1 after the boot records retire. /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff().unlock(); + inner.handoff_unlock(); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.handoff().unlock(); + inner.publish_slot(KERNEL_ALIAS_CTX); + inner.handoff_unlock(); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -3312,7 +3399,12 @@ impl Uat { /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner(dev: driver::AsahiDevice, firmware: UatFirmware) -> Result> { + fn make_inner( + dev: driver::AsahiDevice, + firmware: UatFirmware, + quirks: UatQuirks, + reserved: Option>, + ) -> Result> { let handoff_rgn = Self::map_region(dev.as_ref().as_ref(), c"handoff", HANDOFF_SIZE, true)?; let ttbs_rgn = Self::map_region(dev.as_ref().as_ref(), c"ttbs", SLOTS_SIZE, true)?; @@ -3321,14 +3413,16 @@ impl Uat { let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); + let shared_quirks = quirks; Arc::pin_init( try_pin_init!(UatInner { firmware, m4_adopted: false, failed: AtomicBool::new(false), - #[cfg(CONFIG_DRM_ASAHI_G16)] - clean_poc: false, + clean_poc: quirks.clean_tables, + quirks, + reserved, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), @@ -3344,6 +3438,7 @@ impl Uat { map_kernel_to_user: false, handoff_rgn, ttbs_rgn, + quirks: shared_quirks, }, "uat_shared" ), @@ -3359,12 +3454,27 @@ impl Uat { geometry: UatGeometry, map_kernel_to_user: bool, firmware: UatFirmware, + quirks: UatQuirks, ) -> Result { dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - let inner = Self::make_inner(dev.clone(), firmware)?; - + mem::set_page_tlbi(quirks.page_tlbi); let of_node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let reserved = if quirks.reserved_tables { + let mut tables = pgtable::ReservedTables::new(); + for name in [c"pagetables", c"l2-pagetables"] { + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + // SAFETY: The boot loader reserved these regions (no-map) for the firmware's + // page tables; they stay reserved until reboot. + unsafe { tables.add(res)? }; + } + Some(Arc::new(tables, GFP_KERNEL)?) + } else { + None + }; + + let inner = Self::make_inner(dev.clone(), firmware, quirks, reserved)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; let ttb1 = res.start(); let ttb1size: usize = res.size().try_into()?; @@ -3372,11 +3482,11 @@ impl Uat { let (pagetables_size, secondary_ttb1) = match firmware { UatFirmware::Handoff => (PAGETABLES_SIZE, None), UatFirmware::PublishedRoots { - secondary_root_offset, - } => ( - secondary_root_offset as usize + UAT_PGSZ, - Some(ttb1 + secondary_root_offset), - ), + secondary_root_offset: Some(offset), + } => (offset as usize + UAT_PGSZ, Some(ttb1 + offset)), + UatFirmware::PublishedRoots { + secondary_root_offset: None, + } => (UAT_PGSZ, None), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); @@ -3427,11 +3537,15 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(firmware)?; + inner.handoff().init(firmware, quirks.handoff_slot)?; + if quirks.clean_tables { + pgtable::clean_poc(inner.handoff() as *const Handoff as *const u8, HANDOFF_SIZE); + pgtable::poc_barrier(); + } dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - inner.handoff().lock(); + inner.handoff_lock(); let ttbs = inner.ttbs(); @@ -3443,7 +3557,7 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - if secondary_ttb1.is_some() { + if matches!(firmware, UatFirmware::PublishedRoots { .. }) { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias .ttb0 @@ -3452,8 +3566,12 @@ impl Uat { .ttb1 .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } + if quirks.clean_tables { + pgtable::clean_poc(ttbs.as_ptr() as *const u8, SLOTS_SIZE); + pgtable::poc_barrier(); + } - inner.handoff().unlock(); + inner.handoff_unlock(); core::mem::drop(inner); diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 38c60052debce..05b0c4ad28715 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -18,7 +18,8 @@ use kernel::{ addr::PhysicalAddr, error::Result, page::Page, - prelude::*, // + prelude::*, + sync::Arc, // }; use crate::debug::*; @@ -96,7 +97,6 @@ pub(crate) fn clean_poc(ptr: *const u8, len: usize) { /// Cleans and invalidates the CPU data cache lines covering `len` bytes at `ptr` to the point of /// coherency, so that later CPU reads observe what a non-snooping agent wrote. Lines the CPU wrote /// are written back first. [`poc_barrier`] completes the maintenance. -#[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { if len == 0 { return; @@ -123,14 +123,108 @@ pub(crate) fn poc_barrier() { unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; } +/// Page tables in bootloader-reserved memory that the kernel does not map ("no-map"), reached +/// through a write-back mapping instead of `struct page` (the `pagetables` and `l2-pagetables` +/// regions, where the firmware builds its root and kernel-window tables). +pub(crate) struct ReservedTables { + regions: KVec, + starts: KVec, +} + +// SAFETY: The mappings are plain write-back memory owned for the lifetime of this object; table +// entries are only accessed as atomics. +unsafe impl Send for ReservedTables {} +// SAFETY: As above. +unsafe impl Sync for ReservedTables {} + +impl ReservedTables { + pub(crate) fn new() -> Self { + Self { + regions: KVec::new(), + starts: KVec::new(), + } + } + + /// Adds a reserved region, mapped write-back. + /// + /// # Safety + /// + /// `res` must be a bootloader-reserved region outside System RAM that stays reserved. + pub(crate) unsafe fn add(&mut self, res: kernel::io::resource::Resource) -> Result { + let start = res.start(); + // SAFETY: The caller guarantees the region; write-back matches how the firmware maps it. + let map = unsafe { + kernel::io::mem::Mem::try_new(res, kernel::io::mem::MemFlag::WB.into())? + }; + self.starts.push(start, GFP_KERNEL)?; + self.regions.push(map, GFP_KERNEL)?; + Ok(()) + } + + /// The CPU address of a page-table page in one of the regions. + fn lookup(&self, phys: PhysicalAddr) -> Option<*mut u8> { + for (start, map) in self.starts.iter().zip(self.regions.iter()) { + let Some(offset) = phys.checked_sub(*start) else { + continue; + }; + let offset = offset as usize; + if offset + UAT_PGSZ <= map.size() { + // SAFETY: The offset and one page lie inside the mapping. + return Some(unsafe { map.ptr().add(offset) }); + } + } + None + } + + /// Whether `phys` is a page of one of the regions. + pub(crate) fn contains(&self, phys: PhysicalAddr) -> bool { + self.lookup(phys).is_some() + } + + /// Drops the CPU's cached lines of every region, so that later reads see what the firmware + /// wrote while booting. Lines the host dirtied are written back first, never discarded. + pub(crate) fn invalidate(&self) { + for map in self.regions.iter() { + clean_invalidate_poc(map.ptr(), map.size()); + } + poc_barrier(); + } +} + +/// Runs `f` on a pointer to `len` bytes at `offset` in the page-table page at `phys`, through the +/// reserved mapping when the page lies in a reserved region, else through `struct page`. +/// +/// # Safety +/// +/// `phys` must be a page-table page that stays allocated while `f` runs. +unsafe fn with_table( + reserved: Option<&ReservedTables>, + phys: PhysicalAddr, + offset: usize, + len: usize, + f: impl FnOnce(*mut u8) -> Result, +) -> Result { + if let Some(base) = reserved.and_then(|r| r.lookup(phys)) { + if offset.checked_add(len).ok_or(EINVAL)? > UAT_PGSZ { + return Err(EINVAL); + } + // SAFETY: lookup() checked that the whole page is mapped. + return f(unsafe { base.add(offset) }); + } + // SAFETY: The caller guarantees an allocated page-table page in RAM. + let page = unsafe { Page::borrow_phys_unchecked(&phys) }; + page.with_pointer_into_page(offset, len, f) +} + /// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. /// /// # Safety /// -/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` -/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they -/// are only exposed as atomics. +/// `ttb` must be the physical address of a page table root, in RAM or in one of the `reserved` +/// regions, that stays allocated while `f` runs. Other agents (the GPU firmware) may access the +/// entries concurrently, which is why they are only exposed as atomics. pub(crate) unsafe fn with_root_entries( + reserved: Option<&ReservedTables>, ttb: PhysicalAddr, count: usize, f: impl FnOnce(&[AtomicU64]) -> R, @@ -138,15 +232,18 @@ pub(crate) unsafe fn with_root_entries( if count > UAT_NPTE { return Err(EINVAL); } - // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. - let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; - page.with_pointer_into_page(0, count * PTE_SIZE, |p| { - // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page - // table entries are naturally aligned. - Ok(f(unsafe { - core::slice::from_raw_parts(p as *const AtomicU64, count) - })) - }) + // SAFETY: A root outside the reserved regions must have a struct page. + if !reserved.is_some_and(|r| r.contains(ttb)) && unsafe { Page::borrow_phys(&ttb) }.is_none() { + return Err(EIO); + } + // SAFETY: The caller guarantees an allocated root page, in RAM or in a reserved region. + unsafe { + with_table(reserved, ttb, 0, count * PTE_SIZE, |p| { + // SAFETY: with_table() checked that `count` entries are mapped, and page table entries + // are naturally aligned. + Ok(f(core::slice::from_raw_parts(p as *const AtomicU64, count))) + }) + } } const PTE_TYPE_BITS: u64 = 3; @@ -281,6 +378,22 @@ impl Prot { (!need_read || readable) && (!need_write || writable) } + /// The native form of a GPU-only permission: AP 0 with the access in PXN (read) and UXN + /// (write), as [`Self::allows_gpu`] reads AP 0. The logical selectors RO, WO, RW, NA map to + /// the native RO, WO, RW, NA. Firmware permissions are returned unchanged. + const fn native_gpu(self) -> Self { + if self.ap != AP_GPU { + return self; + } + let selector = (self.high_bits >> 1) & 3; + Self { + ap: AP_FW_GPU, + high_bits: (self.high_bits & !(HIGH_BITS_PXN | HIGH_BITS_UXN)) + | (((selector + 1) & 3) << 1), + ..self + } + } + pub(crate) const fn is_cached_noncoherent(&self) -> bool { self.ap != AP_GPU && self.memattr == MEMATTR_CACHED } @@ -301,6 +414,10 @@ pub(crate) struct UatPageTable { ttb_owned: bool, /// Ownership could not be relinquished to firmware; physical tables remain until reboot. retained: bool, + /// Reserved regions holding tables this page table may reach. + reserved: Option>, + /// Store GPU-only permissions in the native AP 0 form ([`Prot::native_gpu`]). + native_gpu: bool, /// Tables below a root the driver did not allocate may belong to the bootloader or the /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the /// tables below the root, as with the handoff firmware interface. @@ -310,7 +427,6 @@ pub(crate) struct UatPageTable { oas_mask: u64, /// Clean new tables and changed entries to the point of coherency, for a UAT walker that does /// not snoop the CPU caches ([`Self::enable_clean_poc`]). - #[cfg(CONFIG_DRM_ASAHI_G16)] clean_poc: bool, } @@ -323,45 +439,48 @@ impl UatPageTable { ttb, ttb_owned: true, retained: false, + reserved: None, + native_gpu: false, driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_G16)] clean_poc: false, }) } /// Makes every later change of this page table visible to a UAT walker that does not snoop the - /// CPU caches (G16), and cleans the root, which was zeroed through the cacheable linear map. + /// CPU caches (G16, and `UatQuirks::clean_tables`), and cleans the root, which may have been + /// zeroed through the cacheable linear map. /// /// From now on, a new table page is cleaned before its parent descriptor names it, a changed /// table or leaf descriptor is cleaned after the store, and every mutating walk ends with a /// `dsb sy`. Call it before anything is mapped and before any context names the root. - #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn enable_clean_poc(&mut self) -> Result { self.clean_poc = true; - // SAFETY: The root is a RAM page owned or borrowed by this page table for its lifetime. - let root = unsafe { Page::borrow_phys(&self.ttb) }.ok_or(EIO)?; - root.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; + // SAFETY: The root is owned or borrowed by this page table for its lifetime, in RAM or in + // a reserved region. + unsafe { + with_table(self.reserved.as_deref(), self.ttb, 0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + }) + }?; poc_barrier(); Ok(()) } - /// Whether changes are cleaned to the point of coherency (always false without G16 support). + /// Whether changes are cleaned to the point of coherency. #[inline(always)] fn cleans_poc(&self) -> bool { - #[cfg(CONFIG_DRM_ASAHI_G16)] - { - self.clean_poc - } - #[cfg(not(CONFIG_DRM_ASAHI_G16))] - { - false - } + self.clean_poc + } + + /// Stores GPU-only permissions of this (client or kernel-lower) table in the native AP 0 form + /// (`UatQuirks::native_gpu_permissions`). Firmware roots never hold GPU-only mappings. + pub(crate) fn with_native_gpu(mut self, native_gpu: bool) -> Self { + self.native_gpu = native_gpu; + self } /// Borrow a root; track individual child ownership only when other agents @@ -372,6 +491,7 @@ impl UatPageTable { ias: u32, oas: u32, shared_tables: bool, + reserved: Option>, ) -> Result { mod_pr_debug!( "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", @@ -387,7 +507,9 @@ impl UatPageTable { return Err(EINVAL); } // SAFETY: The bootloader-managed root stays valid while this page table exists. - if unsafe { Page::borrow_phys(&ttb) }.is_none() { + let reachable = reserved.as_ref().is_some_and(|r| r.contains(ttb)) + || unsafe { Page::borrow_phys(&ttb) }.is_some(); + if !reachable { pr_err!( "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", ttb @@ -399,11 +521,12 @@ impl UatPageTable { ttb, ttb_owned: false, retained: false, + reserved, + native_gpu: false, driver_tables: shared_tables.then(KVec::new), va_range, ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, - #[cfg(CONFIG_DRM_ASAHI_G16)] clean_poc: false, }) } @@ -483,6 +606,7 @@ impl UatPageTable { // Handle the case where iova_range.end is just at the top boundary of the IAS let end = ((iova_range.end - 1) & self.ias_mask) + 1; + let reserved = self.reserved.clone(); let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); // For every child table, retain the parent page and entry that names it. // Children are detached before their parents are freed, so these borrowed @@ -526,55 +650,58 @@ impl UatPageTable { ); let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) as usize; - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); - pt_addr[level] = - upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { - let uptep = p as *const _ as *const Pte; - // SAFETY: with_pointer_into_page() ensures the pointer is valid, - // and our index is aligned so it is safe to deref as an AtomicU64. - let upte = unsafe { &*uptep }; - let mut upte_val = upte.load(Ordering::Relaxed); - // Allocate if requested - if upte_val == 0 && alloc { - let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); - if clean { - // A non-snooping walker must never follow a parent to a - // table whose zeroes are still in the CPU caches. - pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { - clean_poc(p, UAT_PGSZ); - Ok(()) - })?; - } - if let Some(tables) = self.driver_tables.as_mut() { - tables.push(pt_page.phys(), GFP_KERNEL)?; - } - let pt_paddr = Page::into_phys(pt_page); - upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; - // Complete child-table initialization before a - // concurrent device walk can follow its parent. - // A later mapping/TLBI barrier cannot order an - // already-visible parent against earlier clears. - crate::mem::sync(); - upte.store(upte_val, Ordering::Relaxed); - if clean { - clean_poc(uptep.cast(), PTE_SIZE); - } + let driver_tables = &mut self.driver_tables; + let oas_mask = self.oas_mask; + let entry = |p: *mut u8| -> Result> { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_table() ensures the pointer is valid, and our index is + // aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if clean { + // A non-snooping walker must never follow a parent to a + // table whose zeroes are still in the CPU caches. + pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; } - if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) - } else if upte_val == 0 || (!alloc && !free) { - mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); - Ok(None) - } else { - pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, - iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); - Ok(None) + if let Some(tables) = driver_tables.as_mut() { + tables.push(pt_page.phys(), GFP_KERNEL)?; } - })?; + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + // Complete child-table initialization before a + // concurrent device walk can follow its parent. + // A later mapping/TLBI barrier cannot order an + // already-visible parent against earlier clears. + crate::mem::sync(); + upte.store(upte_val, Ordering::Relaxed); + if clean { + clean_poc(uptep.cast(), PTE_SIZE); + } + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + pt_addr[level] = unsafe { + with_table(reserved.as_deref(), phys, upidx * PTE_SIZE, PTE_SIZE, entry) + }?; if pt_addr[level].is_some() { pt_parent[level] = Some((phys, upidx)); } @@ -610,13 +737,10 @@ impl UatPageTable { count, iova ); - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; - pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let leaves = |p: *mut u8| -> Result { let ptep = p as *const _ as *const Pte; // SAFETY: We know this is a valid pointer to PTEs and the range is valid and - // checked by with_pointer_into_page(). + // checked by with_table(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; let result = cb(iova, ptes); if clean && mutate { @@ -624,7 +748,12 @@ impl UatPageTable { clean_poc(p, count * PTE_SIZE); } result - })?; + }; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed tables in RAM or in a reserved region. + unsafe { + with_table(reserved.as_deref(), phys, idx * PTE_SIZE, count * PTE_SIZE, leaves) + }?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); iova = align(iova + 1, block); @@ -659,27 +788,34 @@ impl UatPageTable { return Ok(()); } if let Some((parent_phys, parent_index)) = parent { - // SAFETY: The bottom-up traversal keeps the parent allocated while - // its child descriptor is detached. A changed descriptor indicates - // an ownership violation; fail closed instead of freeing a table - // that may still be reachable. - let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; - page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { - // SAFETY: The live page-table page contains aligned u64 PTEs. - // with_pointer_into_page bounds this access to one PTE, and - // the parent remains allocated throughout the traversal. + let oas_mask = self.oas_mask; + let in_reserved = self.reserved.as_ref().is_some_and(|r| r.contains(parent_phys)); + // SAFETY: Only looks the page up; nothing is accessed through it. + if !in_reserved && unsafe { Page::borrow_phys(&parent_phys) }.is_none() { + return Err(EIO); + } + let reserved = self.reserved.as_deref(); + let cleans = self.cleans_poc(); + let detach = |p: *mut u8| -> Result { + // SAFETY: The live page-table page contains aligned u64 PTEs, and with_table() + // bounds this access to one PTE. let pte = unsafe { &*(p as *const AtomicU64) }; let current = pte.load(Ordering::Acquire); - let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); - if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { + let expected = phys & oas_mask & (!UAT_PGMSK as u64); + if current & oas_mask & (!UAT_PGMSK as u64) != expected { return Err(EIO); } pte.store(0, Ordering::Release); - if self.cleans_poc() { + if cleans { clean_poc(p, PTE_SIZE); } Ok(()) - })?; + }; + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + unsafe { with_table(reserved, parent_phys, parent_index * PTE_SIZE, PTE_SIZE, detach) }?; } if let Some(tables) = self.driver_tables.as_mut() { let index = tables.iter().position(|&table| table == phys).ok_or(EIO)?; @@ -780,6 +916,15 @@ impl UatPageTable { } } + /// The permission bits as this table stores them. + fn stored(&self, prot: Prot) -> Prot { + if self.native_gpu { + prot.native_gpu() + } else { + prot + } + } + pub(crate) fn map_pages( &mut self, iova_range: Range, @@ -799,6 +944,7 @@ impl UatPageTable { } let pte_bits = self.pte_bits(); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { @@ -825,6 +971,7 @@ impl UatPageTable { iova_range, prot ); + let prot = self.stored(prot); self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index e8faed1ec56e6..a85e5b585f319 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -60,6 +60,8 @@ #define APPLE_T8140_GFX_IOP_VBAR_BASE 0x482050000ULL #define APPLE_T8140_GFX1_WRAPPER_BASE 0x482e00000ULL #define APPLE_T8140_GFX1_IOP_VBAR_BASE 0x482850000ULL +#define APPLE_T6040_GFX_WRAPPER_BASE 0x28a600000ULL +#define APPLE_T6040_GFX_IOP_VBAR_BASE 0x28a050000ULL static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); @@ -116,9 +118,19 @@ static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_RECV0 == #define APPLE_G17_MBOX_RX_BUDGET 64 #define APPLE_G17_MBOX_ATOMIC_TX_US 1000 +/* One firmware role of an ASCWrap v6 provider, at its fixed CPU-physical windows. */ +struct apple_mbox_ascwrap_v6_role_desc { + const char *name; + enum apple_mbox_ascwrap_v6_role role; + resource_size_t wrapper_base; + resource_size_t iop_vbar_base; +}; + struct apple_mbox_hw { unsigned int reg_offset; bool is_ascwrap_v6; + const struct apple_mbox_ascwrap_v6_role_desc *ascwrap_v6_roles; + unsigned int num_ascwrap_v6_roles; bool ap_initializes_mailboxes; unsigned int control_full; @@ -675,6 +687,8 @@ static int apple_mbox_probe(struct platform_device *pdev) int ret; char *irqname; const char *firmware_role; + const struct apple_mbox_ascwrap_v6_role_desc *role; + unsigned int i; struct apple_mbox *mbox; struct device *dev = &pdev->dev; struct resource *res, *iop_vbar_res; @@ -691,7 +705,7 @@ static int apple_mbox_probe(struct platform_device *pdev) if (mbox->hw->is_ascwrap_v6) { /* * Validate the entire provider identity before mapping either - * resource. These are fixed T8140 CPU-physical windows and role + * resource. These are fixed per-SoC CPU-physical windows and role * assignments, not generic consumer-provided aliases. */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wrapper"); @@ -709,28 +723,26 @@ static int apple_mbox_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "missing ASCWrap v6 firmware role\n"); - if (!strcmp(firmware_role, "GFX")) { - mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; - if (res->start != APPLE_T8140_GFX_WRAPPER_BASE || - iop_vbar_res->start != APPLE_T8140_GFX_IOP_VBAR_BASE) - return dev_err_probe(dev, -EINVAL, - "GFX ASCWrap v6 resources do not match T8140\n"); - } else if (!strcmp(firmware_role, "GFX1")) { - mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; - if (res->start != APPLE_T8140_GFX1_WRAPPER_BASE || - iop_vbar_res->start != APPLE_T8140_GFX1_IOP_VBAR_BASE) - return dev_err_probe(dev, -EINVAL, - "GFX1 ASCWrap v6 resources do not match T8140\n"); - } else { + for (i = 0; i < mbox->hw->num_ascwrap_v6_roles; i++) { + role = &mbox->hw->ascwrap_v6_roles[i]; + if (!strcmp(firmware_role, role->name)) + break; + } + if (i == mbox->hw->num_ascwrap_v6_roles) return dev_err_probe(dev, -EINVAL, "invalid ASCWrap v6 firmware role\n"); - } + if (res->start != role->wrapper_base || + iop_vbar_res->start != role->iop_vbar_base) + return dev_err_probe(dev, -EINVAL, + "%s ASCWrap v6 resources do not match the SoC\n", + role->name); + mbox->ascwrap_v6.role = role->role; mbox->ascwrap_v6.wrapper_start = res->start; mbox->ascwrap_v6.wrapper_size = resource_size(res); mbox->ascwrap_v6.iop_vbar_start = iop_vbar_res->start; mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar_res); - /* T8140 implements both the CPU start and the CPU stop. */ + /* T8140 and T6040 implement both the CPU start and the CPU stop. */ mbox->ascwrap_v6.missing = 0; mbox->wrapper_regs = @@ -874,9 +886,44 @@ static const struct apple_mbox_hw apple_mbox_t8140_asc_hw = { .has_irq_controls = false, }; -static const struct apple_mbox_hw apple_mbox_ascwrap_v6_hw = { +static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t8140_ascwrap_v6_roles[] = { + { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_T8140_GFX_WRAPPER_BASE, APPLE_T8140_GFX_IOP_VBAR_BASE }, + { "GFX1", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, + APPLE_T8140_GFX1_WRAPPER_BASE, APPLE_T8140_GFX1_IOP_VBAR_BASE }, +}; + +/* The T6040 runs a single GPU firmware instance. */ +static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6040_ascwrap_v6_roles[] = { + { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_T6040_GFX_WRAPPER_BASE, APPLE_T6040_GFX_IOP_VBAR_BASE }, +}; + +static const struct apple_mbox_hw apple_mbox_t8140_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, + .ascwrap_v6_roles = apple_mbox_t8140_ascwrap_v6_roles, + .num_ascwrap_v6_roles = ARRAY_SIZE(apple_mbox_t8140_ascwrap_v6_roles), + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .has_irq_controls = false, +}; + +static const struct apple_mbox_hw apple_mbox_t6040_ascwrap_v6_hw = { .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, .is_ascwrap_v6 = true, + .ascwrap_v6_roles = apple_mbox_t6040_ascwrap_v6_roles, + .num_ascwrap_v6_roles = ARRAY_SIZE(apple_mbox_t6040_ascwrap_v6_roles), .control_full = APPLE_ASC_MBOX_CONTROL_FULL, .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, @@ -929,7 +976,8 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t6031-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8122-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, - { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, + { .compatible = "apple,t6040-ascwrap-v6", .data = &apple_mbox_t6040_ascwrap_v6_hw }, + { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_t8140_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, /*