From 3359c54f5d7a68ea16d1d42e727214e68d79c525 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 04:18:56 -0500 Subject: [PATCH 001/117] drm/asahi: uapi: add the G18 (M6) feature bits, flags and payloads The G18 backend (T8152, Mac mini M6) takes this driver's command layout. Two of its abilities have no equivalent among the shared features, so they take the next free feature bits: - DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH (bit 31): render commands may carry struct drm_asahi_g18_render_resources, flagged with DRM_ASAHI_RENDER_G18_RESOURCES (render flag bit 25). The kernel sizes the shared TA/3D private-memory pool from it. - DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES (bit 32): single-sample renders with 32x32 utiles also accept 40-, 56- and 72-byte samples. It is a #define after enum drm_asahi_feature: a 64-bit enumerator would widen that enum to unsigned long, which bindgen maps to usize. Compute commands get two G18-only flags: DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE (bit 18) selects the native texture compute mode, and DRM_ASAHI_COMPUTE_G18_RESOURCES (bit 19) adds struct drm_asahi_g18_compute_resources. The flagged payloads, struct drm_asahi_cmd_render_g18 (528 bytes) and struct drm_asahi_cmd_compute_g18 (264 bytes), are the complete render or compute command followed by its resources. Every field of the command keeps its meaning. The flag numbers leave render bits 21-24, compute bits 16-17 and feature bits 25, 27 and 30 to the M5 runtime's userspace, which uses them. No existing struct, field or value changes. G18 reports DRM_ASAHI_GPU_HAL_300; say so in its documentation. --- include/uapi/drm/asahi_drm.h | 174 ++++++++++++++++++++++++++++++++++- 1 file changed, 172 insertions(+), 2 deletions(-) diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 418522205ee6f..148045252287f 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -209,7 +209,7 @@ struct drm_asahi_params_global { * * G13-G15 on 14.x report DRM_ASAHI_GPU_HAL_LEGACY. * The exact J613 25G83 runtime and the G17P runtime report - * DRM_ASAHI_GPU_HAL_200. + * DRM_ASAHI_GPU_HAL_200. G18 (M6) reports DRM_ASAHI_GPU_HAL_300. */ __u32 gpu_hal_generation; @@ -241,7 +241,7 @@ enum drm_asahi_gpu_hal_generation { */ DRM_ASAHI_GPU_HAL_200 = 200, - /** @DRM_ASAHI_GPU_HAL_300: HAL300 command stream. Never reported by this kernel. */ + /** @DRM_ASAHI_GPU_HAL_300: HAL300 command stream, reported by G18 (M6). */ DRM_ASAHI_GPU_HAL_300 = 300, }; @@ -422,8 +422,33 @@ enum drm_asahi_feature { * include the earlier points of that timeline without adding GPU waits. */ DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, + + /* + * Bit 30 is used by the M5 runtime's userspace. Bits 31 and 32 belong + * to the G18 (M6, T8152) backend, which alone reports them. Bit 32 is + * DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES, defined after this enum so + * that the enum keeps its 32-bit type. + */ + + /** + * @DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH: G18: render commands accept + * DRM_ASAHI_RENDER_G18_RESOURCES. The kernel sizes the shared TA/3D + * private-memory pool from the struct drm_asahi_g18_render_resources + * the command carries and programs both launch pairs; userspace emits + * nonzero SPECTPR scratch frames only for such renders. + */ + DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH = (1ULL) << 31, }; +/* + * DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES: G18: single-sample, + * single-layer renders with 32x32 utiles also accept 40-, 56- and 72-byte + * samples (the hardware's native layouts), besides tiles of at most 32 KiB + * and 48..128-byte samples in steps of 16. A feature bit like those of enum + * drm_asahi_feature; a 64-bit enumerator would widen that enum's type. + */ +#define DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES ((1ULL) << 32) + #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 #define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 @@ -1251,6 +1276,18 @@ enum drm_asahi_render_flags { * may depend on it. Requires DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS. */ DRM_ASAHI_RENDER_FRAGMENT_BARRIERS = (1U << 20), + + /* + * Bits 21 to 24 are used by the M5 runtime's userspace. + */ + + /** + * @DRM_ASAHI_RENDER_G18_RESOURCES: G18 only; requires + * DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH. The payload is a complete + * struct drm_asahi_cmd_render_g18: the render command followed by a + * struct drm_asahi_g18_render_resources. Any bytes after it are zero. + */ + DRM_ASAHI_RENDER_G18_RESOURCES = (1U << 25), }; /** @@ -1737,6 +1774,139 @@ enum drm_asahi_compute_flags { * ordered helpers cannot time out while waiting for the build. */ DRM_ASAHI_COMPUTE_AS_BUILD = (1L << 2), + + /* + * Bits 16 and 17 are used by the M5 runtime's userspace. + */ + + /** + * @DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE: G18 only. The command's + * dispatch binds texture or image state (a texture table, a sampled + * texture, a storage image), so the kernel selects the compute mode + * the native driver uses for such commands. Other backends refuse it. + */ + DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE = (1L << 18), + + /** + * @DRM_ASAHI_COMPUTE_G18_RESOURCES: G18 only. The payload is a complete + * struct drm_asahi_cmd_compute_g18: the compute command, with + * @scs_layout zero, followed by a struct drm_asahi_g18_compute_resources. + * Any bytes after it are zero. + */ + DRM_ASAHI_COMPUTE_G18_RESOURCES = (1L << 19), +}; + +/** + * struct drm_asahi_g18_render_resources - G18 per-render program demand + * + * Carried by render commands with DRM_ASAHI_RENDER_G18_RESOURCES, after the + * complete struct drm_asahi_cmd_render (see struct drm_asahi_cmd_render_g18). + * + * Effective (linked) program demand in compiler units, maximized over every + * draw of the render: 16-bit registers, the static thread-private (TPS) + * frame in bytes as programmed in SPECTPR (a multiple of 16), and IPR/staging + * halfwords. Tile programs are background, end-of-tile, partial background + * and partial end-of-tile; a zero register count marks an absent program. + * Tile programs must not use scratch. At least one of the vertex and + * fragment frames is nonzero. Tile storage is taken from the render command. + */ +struct drm_asahi_g18_render_resources { + /** @version: Must be 1. */ + __u32 version; + /** @flags: Must be 3 (vertex and fragment programs present). */ + __u32 flags; + /** @vertex_registers_16: Vertex program registers, in 16-bit units. */ + __u32 vertex_registers_16; + /** @vertex_scratch_bytes: Vertex SPECTPR frame bytes. */ + __u32 vertex_scratch_bytes; + /** @vertex_staging_16: Vertex staging halfwords. */ + __u32 vertex_staging_16; + /** @fragment_registers_16: Fragment program registers, in 16-bit units. */ + __u32 fragment_registers_16; + /** @fragment_scratch_bytes: Fragment SPECTPR frame bytes. */ + __u32 fragment_scratch_bytes; + /** @fragment_staging_16: Fragment staging halfwords. */ + __u32 fragment_staging_16; + /** @tile_registers_16: Tile program registers, in 16-bit units. */ + __u32 tile_registers_16[4]; + /** @tile_staging_16: Tile program staging halfwords. */ + __u32 tile_staging_16[4]; + /** @reserved: MBZ. */ + __u32 reserved[4]; +}; + +/** + * struct drm_asahi_g18_compute_resources - G18 per-dispatch metadata + * + * Carried by compute commands with DRM_ASAHI_COMPUTE_G18_RESOURCES, after the + * complete struct drm_asahi_cmd_compute (see struct drm_asahi_cmd_compute_g18). + * + * Counts are compiler halfwords or bytes as named; the kernel converts them + * to native units. Version 1 scales @local_bytes + @variable_shared_bytes by + * four for shaders using byte offsets in element-index fields; version 2 uses + * the actual byte footprints. @flags: bit 0 main staging known, bit 1 + * constant program present, bit 2 constant staging known, bit 3 local + * geometry unknown (@local_size must then be zero). Other flag bits and + * @reserved must be zero. + */ +struct drm_asahi_g18_compute_resources { + /** @version: 1 or 2. */ + __u32 version; + /** @flags: See above. */ + __u32 flags; + /** @registers_16: Main program registers, in 16-bit units. */ + __u32 registers_16; + /** @scratch_bytes: Main program scratch bytes. */ + __u32 scratch_bytes; + /** @spectpr_scratch_bytes: SPECTPR frame bytes. */ + __u32 spectpr_scratch_bytes; + /** @staging_16: Main program staging halfwords. */ + __u32 staging_16; + /** @constant_registers_16: Constant program registers, in 16-bit units. */ + __u32 constant_registers_16; + /** @constant_scratch_bytes: Constant program scratch bytes. */ + __u32 constant_scratch_bytes; + /** @constant_staging_16: Constant program staging halfwords. */ + __u32 constant_staging_16; + /** @local_bytes: Threadgroup memory bytes. */ + __u32 local_bytes; + /** @imageblock_stride: Imageblock stride. */ + __u32 imageblock_stride; + /** @variable_shared_bytes: Variable shared memory bytes. */ + __u32 variable_shared_bytes; + /** @local_size: Workgroup size, or zero when unknown. */ + __u32 local_size[3]; + /** @reserved: MBZ. */ + __u32 reserved; +}; + +/** + * struct drm_asahi_cmd_render_g18 - G18 render payload with program demand + * + * The payload of a render command with DRM_ASAHI_RENDER_G18_RESOURCES. Every + * field of @legacy keeps its meaning; fields whose feature G18 does not + * report (timestamp copies, ray queries) are zero. + */ +struct drm_asahi_cmd_render_g18 { + /** @legacy: The render command. */ + struct drm_asahi_cmd_render legacy; + /** @resources: Program demand of the render. */ + struct drm_asahi_g18_render_resources resources; +}; + +/** + * struct drm_asahi_cmd_compute_g18 - G18 compute payload with dispatch metadata + * + * The payload of a compute command with DRM_ASAHI_COMPUTE_G18_RESOURCES. + * Every field of @base keeps its meaning; fields whose feature G18 does not + * report (timestamp copies, the auxiliary allocation, intersection tables) + * are zero, and so is @base.scs_layout. + */ +struct drm_asahi_cmd_compute_g18 { + /** @base: The compute command. */ + struct drm_asahi_cmd_compute base; + /** @resources: Metadata of the command's dispatches. */ + struct drm_asahi_g18_compute_resources resources; }; /** From 18d9c26fdbff8311c77a25efb5c7c2ff7e90eac5 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 04:18:56 -0500 Subject: [PATCH 002/117] rust: drm: mm: split off and rejoin a node's trailing reservation The G18 backend maps an alias into the unmapped guard pages that trail a kernel mapping, and must give the guard back if the alias is undone. Add NodeData::split_tail(), which turns the tail of a reservation into a node of its own, and NodeData::rejoin_tail(), which merges an adjacent tail back. Both are limited to allocators without colour callbacks, allocate before they mutate, and leave both reservations unchanged on failure. --- rust/kernel/drm/mm.rs | 95 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/rust/kernel/drm/mm.rs b/rust/kernel/drm/mm.rs index a01ed8b64c5a0..a182a5d608b85 100644 --- a/rust/kernel/drm/mm.rs +++ b/rust/kernel/drm/mm.rs @@ -7,7 +7,7 @@ use crate::{ alloc::flags::*, bindings, - error::{code::EINVAL, to_result, Result}, + error::{code::{EINVAL, EOVERFLOW}, to_result, Result}, sync::{new_mutex, Arc, Mutex, UniqueArc}, types::Opaque, }; @@ -48,9 +48,9 @@ pub struct NodeData, T> { inner: T, } -// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. +// SAFETY: Allocator ops take the mutex; node mutation requires exclusive pinned access. unsafe impl, T: Send> Send for NodeData {} -// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. +// SAFETY: Allocator ops take the mutex; node mutation requires exclusive pinned access. unsafe impl, T: Sync> Sync for NodeData {} /// Available MM node insertion modes @@ -139,6 +139,95 @@ impl, T> Deref for NodeData { } } +impl NodeData<(), T> { + /// Split a trailing reservation while retaining ownership of every address. + /// + /// This is restricted to allocators without callbacks. The caller decides + /// whether its payload can be split; no payload or page-table state is + /// copied. Allocation precedes mutation, and both new reservations become + /// visible under the same allocator mutex. + pub fn split_tail(self: Pin<&mut Self>, size: u64, inner: T) -> Result> { + if !self.valid || size == 0 || size >= self.node.size { + return Err(EINVAL); + } + let mut tail = KBox::new( + NodeData { + // SAFETY: The native node starts empty before reservation. + node: unsafe { core::mem::zeroed() }, + mm: self.mm.clone(), valid: false, _pin: PhantomPinned, inner, + }, + GFP_KERNEL, + )?; + // SAFETY: Neither pinned node nor its payload moves. Only allocator + // bookkeeping fields are changed while holding the allocator mutex. + let front = unsafe { self.get_unchecked_mut() }; + let guard = front.mm.lock(); + // SAFETY: The allocator is initialized and protected by guard. + if unsafe { (*guard.0.get()).color_adjust.is_some() } { return Err(EINVAL); } + let old_size = front.node.size; + tail.node.start = front.node.start + old_size - size; + tail.node.size = size; + tail.node.color = front.node.color; + // SAFETY: Both node allocations remain at fixed addresses. The + // smaller front and adjacent tail partition the original reservation. + unsafe { + bindings::drm_mm_remove_node(&mut front.node); + front.node.size = old_size - size; + let first = bindings::drm_mm_reserve_node(guard.0.get(), &mut front.node); + if first != 0 { + front.node.size = old_size; + assert_eq!(bindings::drm_mm_reserve_node(guard.0.get(), &mut front.node), 0); + return to_result(first).map(|_| unreachable!()); + } + let second = bindings::drm_mm_reserve_node(guard.0.get(), &mut tail.node); + if second != 0 { + bindings::drm_mm_remove_node(&mut front.node); + front.node.size = old_size; + assert_eq!(bindings::drm_mm_reserve_node(guard.0.get(), &mut front.node), 0); + return to_result(second).map(|_| unreachable!()); + } + } + tail.valid = true; + Ok(Pin::from(tail)) + } + + /// Rejoin an adjacent tail without allocation. On success the tail is + /// detached and may only be dropped; its payload is not destroyed here. + /// Errors leave both reservations unchanged. + pub fn rejoin_tail(self: Pin<&mut Self>, tail: Pin<&mut Self>) -> Result { + if !self.valid || !tail.valid || !core::ptr::eq(&*self.mm, &*tail.mm) + || self.node.start.checked_add(self.node.size) != Some(tail.node.start) + || self.node.color != tail.node.color { + return Err(EINVAL); + } + let combined = self.node.size.checked_add(tail.node.size).ok_or(EOVERFLOW)?; + // SAFETY: Exclusive pinned access permits field mutation, not moving + // the linked native nodes. The mutex excludes every other MM operation. + let front = unsafe { self.get_unchecked_mut() }; + let tail = unsafe { tail.get_unchecked_mut() }; + let guard = front.mm.lock(); + // SAFETY: The allocator remains initialized and locked. + if unsafe { (*guard.0.get()).color_adjust.is_some() } { return Err(EINVAL); } + let old_size = front.node.size; + // SAFETY: The two adjacent reservations cover exactly combined bytes. + // No other node can enter the temporary hole while the mutex is held. + unsafe { + bindings::drm_mm_remove_node(&mut tail.node); + bindings::drm_mm_remove_node(&mut front.node); + front.node.size = combined; + let result = bindings::drm_mm_reserve_node(guard.0.get(), &mut front.node); + if result != 0 { + front.node.size = old_size; + assert_eq!(bindings::drm_mm_reserve_node(guard.0.get(), &mut front.node), 0); + assert_eq!(bindings::drm_mm_reserve_node(guard.0.get(), &mut tail.node), 0); + return to_result(result); + } + } + tail.valid = false; + Ok(()) + } +} + impl, T> Drop for NodeData { fn drop(&mut self) { if self.valid { From 8ffc6f655a27b6af913ef9997d8a0a881fa868d7 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 04:18:57 -0500 Subject: [PATCH 003/117] soc: apple: mailbox: add the T8152 GFX ASCWrap v8 provider The M6 (T8152) GPU has two coprocessors, GFX and GFX1, behind ASCWrap v8 wrappers. Add an "apple,t8152-gfx-ascwrap-v8" provider: - a counted FIFO for its post-office window; - the lifecycle helpers the G18 backend uses to start and stop the coprocessors, gated on the PMGR PSD service of each role; - a receive poller. Starting or stopping a coprocessor needs apple_mailbox.ascwrap_v8_cpu_control=1. The receive interrupt is used only with apple_mailbox.ascwrap_v8_rx_irq=1; otherwise the receive side is polled. Both default to off and are read-only at runtime. Ported from our earlier out-of-tree M6 port onto this tree's mailbox core. --- drivers/soc/apple/mailbox.c | 612 ++++++++++++++++++++++++++++++++++-- 1 file changed, 579 insertions(+), 33 deletions(-) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 6f0d1f7957ec0..7823da9854c65 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) @@ -67,6 +68,81 @@ #define APPLE_T6040_GFX_IOP_VBAR_BASE 0x28a050000ULL #define APPLE_T6050_ASCWRAP_BASE_DELTA 0x380000000ULL +/* + * T8152 (J873) GPU ASCWrap v8 providers. The ADT gfx-asc/gfx1-asc nodes are + * iop,ascwrap-v8 with reg[0] = wrapper (0x74000), reg[1] = iop-vbar (8) and + * reg[2] = the post-office bank 0x316020000/0x10000, which is shared with + * PMP, DCP, DCPEXT0/1 and SCALER0. Only the GFX/GFX1 0x80-byte windows of + * that bank are claimed: offset (2 * msgbox-mailbox-num + !postoffice) * + * 0x80 with msgbox-using-alias=1 (m1n1 asc.c) gives 0x316020100 (mailbox 1) + * and 0x316020200 (mailbox 2). Each window is a counted FIFO: RX at +0, + * TX at +0x10 and a status word at +0x20 that holds the RX occupancy and the + * free TX slots. + * + * The wrapper is only touched through the lifecycle helpers, and every one + * of those accesses is preceded by a read of the block's PMGR2 PSD service + * (implicit services 81/82 at 0x303a15100/0x303a15200: state +0x80, ack + * +0xbc). The provider never writes PSD registers. + * + * Receive side. ASCWrap v8 has no mailbox-level interrupt control, so + * has_irq_controls is false and a level interrupt can only be masked at the + * AIC. By default the provider therefore never enables either GPU mailbox + * interrupt: apple_mbox_start() starts a one-jiffy receive poller, and a + * full TX FIFO is waited out by sleeping polls. Only ascwrap_v8_rx_irq=1 + * uses the recv-not-empty line; even then a line that keeps firing with + * nothing to read is disabled and receive falls back to the poller. Every + * post-office receive poll is preceded by the same PSD check as the wrapper + * accesses. + */ +#define APPLE_ASCWRAP_V8_WRAPPER_SIZE 0x74000 +#define APPLE_ASCWRAP_V8_IOP_VBAR_SIZE 0x8 +#define APPLE_ASCWRAP_V8_MBOX_SIZE 0x80 +#define APPLE_ASCWRAP_V8_PSD_SIZE 0x100 +/* m1n1 asc.c starts a v8 CPU with reg[0] + 0x44 bit 4. */ +#define APPLE_ASCWRAP_V8_CPU_CONTROL 0x44 +#define APPLE_ASCWRAP_V8_CPU_STATUS 0x48 +#define APPLE_ASCWRAP_V8_CPU_RUN BIT(4) +#define APPLE_PSD_STATE 0x80 +#define APPLE_PSD_ACK 0xbc +#define APPLE_PSD_STATE_MASK GENMASK(2, 0) +#define APPLE_T8152_GPU_ASC_PSD_ACTIVE 2 +/* apple_mbox_ascwrap_v6_lifecycle.missing bit: CPU control not enabled. */ +#define APPLE_ASCWRAP_V8_MISSING_CPU_CONTROL_APPROVAL BIT(0) +/* Receive poll period (poll-only mode, or after the storm guard tripped). */ +#define APPLE_ASCWRAP_V8_RX_POLL_JIFFIES 1 +/* Consecutive recv interrupts that drained nothing before the line is cut. */ +#define APPLE_ASCWRAP_V8_RX_STORM_LIMIT 1000 + +#define APPLE_T8152_GFX_WRAPPER_BASE 0x582600000ULL +#define APPLE_T8152_GFX_IOP_VBAR_BASE 0x582050000ULL +#define APPLE_T8152_GFX_MBOX_BASE 0x316020100ULL +#define APPLE_T8152_GFX_PSD_BASE 0x303a15100ULL +#define APPLE_T8152_GFX1_WRAPPER_BASE 0x582e00000ULL +#define APPLE_T8152_GFX1_IOP_VBAR_BASE 0x582850000ULL +#define APPLE_T8152_GFX1_MBOX_BASE 0x316020200ULL +#define APPLE_T8152_GFX1_PSD_BASE 0x303a15200ULL + +/* + * Starting or stopping the ASC CPU of a T8152 GPU wrapper changes the GPU's + * state and is not qualified as a default. Every v8 CPU start/stop request + * is refused (-EPERM) unless this is set; lifecycle reads are always allowed. + */ +static bool ascwrap_v8_cpu_control; +module_param(ascwrap_v8_cpu_control, bool, 0444); +MODULE_PARM_DESC(ascwrap_v8_cpu_control, + "Allow T8152 GPU ASCWrap v8 CPU start/stop writes (default: refuse)"); + +/* + * The GFX/GFX1 interrupt assignment has not been observed on hardware, and a + * wrongly wired level line would storm with interrupts that the handler + * cannot mask. The default services receive by polling and never enables a + * GPU mailbox interrupt. + */ +static bool ascwrap_v8_rx_irq; +module_param(ascwrap_v8_rx_irq, bool, 0444); +MODULE_PARM_DESC(ascwrap_v8_rx_irq, + "Service T8152 GPU ASCWrap v8 receive with the recv-not-empty interrupt (default: poll, no GPU mailbox interrupt is enabled)"); + static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_CONTROL == @@ -140,6 +216,10 @@ struct apple_mbox_hw { bool is_ascwrap_v6; const struct apple_mbox_ascwrap_v6_role_desc *ascwrap_v6_roles; unsigned int num_ascwrap_v6_roles; + /* T8152 GPU ASCWrap v8: wrapper + iop-vbar + post-office window + PSD. */ + bool is_ascwrap_v8; + /* Status word counts RX occupancy and free TX slots; paired transfers. */ + bool counted_fifo; bool ap_initializes_mailboxes; unsigned int control_full; @@ -160,6 +240,68 @@ struct apple_mbox_hw { unsigned int irq_bit_send_empty; }; +/* + * Provider-private state kept outside the consumer-visible struct apple_mbox. + * Every provider is allocated as one of these; apple_mbox_get() hands out a + * pointer to the embedded mailbox. + */ +struct apple_mbox_priv { + struct apple_mbox mbox; + /* ASCWrap v8 only: the provider's PMGR2 PSD service block (read-only). */ + void __iomem *psd_regs; + /* ASCWrap v8 only: receive by polling; fixed at probe. */ + bool rx_poll_only; + /* rx_lock: recv interrupt enabled and not yet disabled again. */ + bool rx_irq_live; + /* rx_lock: consecutive recv interrupts that drained nothing. */ + unsigned int rx_irq_empty_run; + /* rx_lock: an unpowered-PSD receive skip was logged this activation. */ + bool rx_psd_skip_logged; + struct delayed_work rx_poll_work; +}; + +static struct apple_mbox_priv *apple_mbox_to_priv(struct apple_mbox *mbox) +{ + return container_of(mbox, struct apple_mbox_priv, mbox); +} + +/* Only providers allocated by apple_mbox_probe() have a receive poller. */ +static bool apple_mbox_has_rx_poller(const struct apple_mbox *mbox) +{ + return mbox->hw->is_ascwrap_v8; +} + +/* Both ASCWrap generations share the consumer lifecycle API below. */ +static bool apple_mbox_has_cpu_lifecycle(const struct apple_mbox *mbox) +{ + return mbox->hw->is_ascwrap_v6 || mbox->hw->is_ascwrap_v8; +} + +/* + * Read-only check that a v8 ASC block is powered before any wrapper MMIO. + * The T8152 GFX/GFX1 services are active in state 2. Reading STATE clears + * the transition ACK, so ACK is sampled first and not required to remain + * set for steady-state access. The cold state 6 is rejected. Never writes + * the PSD request register. + */ +static int apple_mbox_ascwrap_v8_psd_powered(struct apple_mbox *mbox, + u32 *state, u32 *ack) +{ + void __iomem *psd = apple_mbox_to_priv(mbox)->psd_regs; + + *state = 0; + *ack = 0; + if (!mbox->hw->is_ascwrap_v8 || !psd) + return -ENODEV; + + *ack = readl(psd + APPLE_PSD_ACK); + *state = readl(psd + APPLE_PSD_STATE); + if (FIELD_GET(APPLE_PSD_STATE_MASK, *state) != APPLE_T8152_GPU_ASC_PSD_ACTIVE) + return -EHOSTDOWN; + + return 0; +} + int apple_mbox_ascwrap_v6_get_lifecycle( struct apple_mbox *mbox, struct apple_mbox_ascwrap_v6_lifecycle *lifecycle) @@ -168,7 +310,7 @@ int apple_mbox_ascwrap_v6_get_lifecycle( return -EINVAL; if (READ_ONCE(mbox->removing)) return -ENODEV; - if (!mbox->hw->is_ascwrap_v6) + if (!apple_mbox_has_cpu_lifecycle(mbox)) return -EOPNOTSUPP; *lifecycle = mbox->ascwrap_v6; @@ -178,7 +320,7 @@ EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_get_lifecycle); int apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(struct apple_mbox *mbox) { - if (!mbox || !mbox->hw->is_ascwrap_v6) + if (!mbox || !apple_mbox_has_cpu_lifecycle(mbox)) return -EOPNOTSUPP; if (READ_ONCE(mbox->removing)) return -ENODEV; @@ -199,14 +341,35 @@ static void apple_mbox_ascwrap_v6_stop_cpu_locked(struct apple_mbox *mbox) if (!mbox->ascwrap_v6_cpu_running) return; - cpu_control = (u8 __iomem *)mbox->wrapper_regs + - APPLE_ASCWRAP_V6_CPU_CONTROL; + if (mbox->hw->is_ascwrap_v8) { + u32 state, ack; - /* Stop: clear RUN, reread, clear bit 5. */ - val = readl(cpu_control); - writel(val & ~APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); - val = readl(cpu_control); - writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); + /* + * Reached only after an allowed v8 start. As m1n1 + * asc_cpu_stop(): clear RUN at +0x44. An unpowered block cannot + * be running, and touching its wrapper would fault, so skip the + * write then. + */ + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_CONTROL; + if (apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack)) { + dev_warn(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; CPU stop write skipped\n", + state, ack); + } else { + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V8_CPU_RUN, cpu_control); + } + } else { + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; + + /* Stop: clear RUN, reread, clear bit 5. */ + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + val = readl(cpu_control); + writel(val & ~APPLE_ASCWRAP_V6_CPU_STOP_SECOND_CLEAR, cpu_control); + } mbox->ascwrap_v6_cpu_running = false; pm_runtime_mark_last_busy(mbox->dev); @@ -231,8 +394,11 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) u32 val; int ret = 0; - if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + if (!mbox || !apple_mbox_has_cpu_lifecycle(mbox) || !mbox->wrapper_regs) return -EOPNOTSUPP; + /* See the ascwrap_v8_cpu_control parameter. */ + if (mbox->hw->is_ascwrap_v8 && !ascwrap_v8_cpu_control) + return -EPERM; mutex_lock(&mbox->lifecycle_lock); if (mbox->removing) { @@ -247,6 +413,17 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) ret = -EBUSY; goto out_unlock; } + if (mbox->hw->is_ascwrap_v8) { + u32 state, ack; + + ret = apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack); + if (ret) { + dev_err(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; CPU start refused\n", + state, ack); + goto out_unlock; + } + } ret = pm_runtime_resume_and_get(mbox->dev); if (ret) { @@ -255,12 +432,20 @@ int apple_mbox_ascwrap_v6_start_cpu(struct apple_mbox *mbox) goto out_unlock; } - cpu_control = (u8 __iomem *)mbox->wrapper_regs + - APPLE_ASCWRAP_V6_CPU_CONTROL; + if (mbox->hw->is_ascwrap_v8) { + /* As m1n1 asc_cpu_start(): set RUN at reg[0] + 0x44. */ + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_CONTROL; + val = readl(cpu_control); + writel(val | APPLE_ASCWRAP_V8_CPU_RUN, cpu_control); + } else { + cpu_control = (u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V6_CPU_CONTROL; - /* Start: read, set RUN, write. */ - val = readl(cpu_control); - writel(val | APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + /* Start: read, set RUN, write. */ + val = readl(cpu_control); + writel(val | APPLE_ASCWRAP_V6_CPU_RUN, cpu_control); + } mbox->ascwrap_v6_cpu_running = true; out_unlock: @@ -273,8 +458,11 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) { int ret = 0; - if (!mbox || !mbox->hw->is_ascwrap_v6 || !mbox->wrapper_regs) + if (!mbox || !apple_mbox_has_cpu_lifecycle(mbox) || !mbox->wrapper_regs) return -EOPNOTSUPP; + /* See the ascwrap_v8_cpu_control parameter. */ + if (mbox->hw->is_ascwrap_v8 && !ascwrap_v8_cpu_control) + return -EPERM; mutex_lock(&mbox->lifecycle_lock); if (mbox->removing) { @@ -282,6 +470,7 @@ int apple_mbox_ascwrap_v6_stop_cpu(struct apple_mbox *mbox) goto out_unlock; } + /* A no-op unless this provider started the CPU itself. */ apple_mbox_ascwrap_v6_stop_cpu_locked(mbox); out_unlock: @@ -300,12 +489,32 @@ int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, { int ret = 0; - if (!mbox || !control || !status || !mbox->hw->is_ascwrap_v6 || + if (!mbox || !control || !status || !apple_mbox_has_cpu_lifecycle(mbox) || !mbox->wrapper_regs) return -EOPNOTSUPP; mutex_lock(&mbox->lifecycle_lock); if (mbox->removing) { ret = -ENODEV; + } else if (mbox->hw->is_ascwrap_v8) { + u32 state, ack; + + /* PSD check first; an unpowered wrapper read would SError. */ + *control = 0; + *status = 0; + ret = apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack); + if (ret) { + dev_info_ratelimited(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; CPU_CONTROL/STATUS reads skipped\n", + state, ack); + } else { + *control = readl((u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_CONTROL); + *status = readl((u8 __iomem *)mbox->wrapper_regs + + APPLE_ASCWRAP_V8_CPU_STATUS); + dev_dbg(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x CPU_CONTROL=%#x CPU_STATUS=%#x\n", + state, ack, *control, *status); + } } else { *control = readl(mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_CONTROL); *status = readl(mbox->wrapper_regs + APPLE_ASCWRAP_V6_CPU_STATUS); @@ -315,6 +524,72 @@ int apple_mbox_ascwrap_v6_get_cpu_state(struct apple_mbox *mbox, } EXPORT_SYMBOL_GPL(apple_mbox_ascwrap_v6_get_cpu_state); +static bool apple_mbox_tx_full(struct apple_mbox *mbox, u32 status) +{ + if (mbox->hw->counted_fifo) + return !(status & mbox->hw->control_full); + return status & mbox->hw->control_full; +} + +static bool apple_mbox_rx_empty(struct apple_mbox *mbox, u32 status) +{ + if (mbox->hw->counted_fifo) + return !(status & mbox->hw->control_empty); + return status & mbox->hw->control_empty; +} + +/* + * A counted FIFO transfers each message with one paired access. Its status + * word holds the RX occupancy and the free TX slots, not full/empty flags. + * Keep the two words of a message together. + */ +static void apple_mbox_write_msg(struct apple_mbox *mbox, + struct apple_mbox_msg msg) +{ + u64 high = FIELD_PREP(APPLE_MBOX_MSG1_MSG, msg.msg1); + +#ifdef CONFIG_ARM64 + if (mbox->hw->counted_fifo) { + u64 low = (__force u64)cpu_to_le64(msg.msg0); + + high = (__force u64)cpu_to_le64(high); + wmb(); + asm volatile("stp %0, %1, [%2]" + : : "r" (low), "r" (high), + "r" (mbox->regs + mbox->hw->a2i_send0) + : "memory"); + } else +#endif + { + writeq_relaxed(msg.msg0, mbox->regs + mbox->hw->a2i_send0); + writeq_relaxed(high, mbox->regs + mbox->hw->a2i_send1); + } +} + +static struct apple_mbox_msg apple_mbox_read_msg(struct apple_mbox *mbox) +{ + struct apple_mbox_msg msg; + u64 low, high; + +#ifdef CONFIG_ARM64 + if (mbox->hw->counted_fifo) { + asm volatile("ldp %0, %1, [%2]" + : "=&r" (low), "=&r" (high) + : "r" (mbox->regs + mbox->hw->i2a_recv0) + : "memory"); + rmb(); + low = le64_to_cpu((__force __le64)low); + high = le64_to_cpu((__force __le64)high); + } else +#endif + { + low = readq_relaxed(mbox->regs + mbox->hw->i2a_recv0); + high = readq_relaxed(mbox->regs + mbox->hw->i2a_recv1); + } + msg.msg0 = low; + msg.msg1 = FIELD_GET(APPLE_MBOX_MSG1_MSG, high); + return msg; +} int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, bool atomic) @@ -332,7 +607,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, } mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); - while (mbox_ctrl & mbox->hw->control_full) { + while (apple_mbox_tx_full(mbox, mbox_ctrl)) { now = jiffies; if (time_after_eq(now, deadline)) { u32 i2a = readl_relaxed(mbox->regs + mbox->hw->i2a_control); @@ -348,6 +623,23 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, return -ETIMEDOUT; } remaining = deadline - now; + if (!atomic && apple_mbox_has_rx_poller(mbox) && + apple_mbox_to_priv(mbox)->rx_poll_only) { + /* + * Poll-only v8: never enable the send-empty line. + * Sleep-poll the TX status, touching MMIO only under + * tx_lock so that removal cannot unmap it underneath. + */ + spin_unlock_irqrestore(&mbox->tx_lock, flags); + usleep_range(100, 200); + spin_lock_irqsave(&mbox->tx_lock, flags); + if (unlikely(READ_ONCE(mbox->removing))) { + spin_unlock_irqrestore(&mbox->tx_lock, flags); + return -ENODEV; + } + mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); + continue; + } if (apple_mbox_is_g17(mbox) && !atomic && mbox->irq_send_empty < 0) { /* The GPU has no send-empty IRQ. Do not busy wait with IRQs * off while dead firmware keeps its FIFO full. Recheck both @@ -366,7 +658,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, if (atomic || mbox->irq_send_empty < 0) { ret = readl_poll_timeout_atomic( mbox->regs + mbox->hw->a2i_control, mbox_ctrl, - !(mbox_ctrl & mbox->hw->control_full), 100, + !apple_mbox_tx_full(mbox, mbox_ctrl), 100, apple_mbox_is_g17(mbox) ? APPLE_G17_MBOX_ATOMIC_TX_US : APPLE_MBOX_TX_TIMEOUT * 1000); @@ -419,9 +711,7 @@ int apple_mbox_send(struct apple_mbox *mbox, const struct apple_mbox_msg msg, mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->a2i_control); } - writeq_relaxed(msg.msg0, mbox->regs + mbox->hw->a2i_send0); - writeq_relaxed(FIELD_PREP(APPLE_MBOX_MSG1_MSG, msg.msg1), - mbox->regs + mbox->hw->a2i_send1); + apple_mbox_write_msg(mbox, msg); spin_unlock_irqrestore(&mbox->tx_lock, flags); @@ -467,13 +757,25 @@ static int apple_mbox_poll_locked(struct apple_mbox *mbox) /* A stopped receiver may still have firmware traffic in its FIFO. */ if (!mbox->rx) return 0; + if (mbox->hw->is_ascwrap_v8) { + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + u32 state, ack; + + /* PSD check before the post-office status read. */ + ret = apple_mbox_ascwrap_v8_psd_powered(mbox, &state, &ack); + if (ret) { + if (!priv->rx_psd_skip_logged) + dev_info(mbox->dev, + "ASCWrap v8 PSD state=%#x ack=%#x is not powered; receive poll skipped\n", + state, ack); + priv->rx_psd_skip_logged = true; + return ret; + } + } mbox_ctrl = readl_relaxed(mbox->regs + mbox->hw->i2a_control); - while (!(mbox_ctrl & mbox->hw->control_empty) && budget--) { - msg.msg0 = readq_relaxed(mbox->regs + mbox->hw->i2a_recv0); - msg.msg1 = FIELD_GET( - APPLE_MBOX_MSG1_MSG, - readq_relaxed(mbox->regs + mbox->hw->i2a_recv1)); + while (!apple_mbox_rx_empty(mbox, mbox_ctrl) && budget--) { + msg = apple_mbox_read_msg(mbox); mbox->rx(mbox, msg, mbox->cookie); ret++; @@ -529,6 +831,32 @@ int apple_mbox_set_rx_util_min(struct apple_mbox *mbox, int util_min) } EXPORT_SYMBOL_GPL(apple_mbox_set_rx_util_min); +/* + * ASCWrap v8 storm guard, rx_lock held. The handler cannot mask the line at + * the mailbox (no irq controls), so a line that keeps firing with nothing to + * read (for example a send-empty line wired as recv-not-empty) would lock up + * the CPU while always returning IRQ_HANDLED. Cut it at the AIC and poll. + */ +static void apple_mbox_v8_rx_irq_guard(struct apple_mbox *mbox, int drained) +{ + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + + if (drained > 0) { + priv->rx_irq_empty_run = 0; + return; + } + if (!priv->rx_irq_live || + ++priv->rx_irq_empty_run < APPLE_ASCWRAP_V8_RX_STORM_LIMIT) + return; + + priv->rx_irq_live = false; + disable_irq_nosync(mbox->irq_recv_not_empty); + dev_err(mbox->dev, + "ASCWrap v8 recv-not-empty IRQ %d fired %u times without a message; disabled, receive falls back to polling\n", + mbox->irq_recv_not_empty, priv->rx_irq_empty_run); + queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, 0); +} + static irqreturn_t apple_mbox_recv_irq(int irq, void *data) { struct apple_mbox *mbox = data; @@ -542,6 +870,8 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) spin_lock_irqsave(&mbox->rx_lock, flags); if (likely(!READ_ONCE(mbox->removing))) consumed = apple_mbox_poll_locked(mbox); + if (mbox->hw->is_ascwrap_v8) + apple_mbox_v8_rx_irq_guard(mbox, consumed); spin_unlock_irqrestore(&mbox->rx_lock, flags); /* G17 uses a oneshot threaded IRQ. Leave it masked during this sleep @@ -570,6 +900,61 @@ int apple_mbox_poll(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_poll); +/* ASCWrap v8 receive poller: poll-only mode, or after the storm guard. */ +static void apple_mbox_v8_rx_poll_work(struct work_struct *work) +{ + struct apple_mbox_priv *priv = container_of(to_delayed_work(work), + struct apple_mbox_priv, + rx_poll_work); + struct apple_mbox *mbox = &priv->mbox; + + if (!READ_ONCE(mbox->active) || READ_ONCE(mbox->removing)) + return; + apple_mbox_poll(mbox); + queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, + APPLE_ASCWRAP_V8_RX_POLL_JIFFIES); +} + +/* lifecycle_lock held, mbox->active already set. */ +static void apple_mbox_v8_rx_start(struct apple_mbox *mbox) +{ + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + unsigned long flags; + + spin_lock_irqsave(&mbox->rx_lock, flags); + priv->rx_irq_empty_run = 0; + priv->rx_psd_skip_logged = false; + priv->rx_irq_live = !priv->rx_poll_only; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + if (priv->rx_poll_only) + queue_delayed_work(system_highpri_wq, &priv->rx_poll_work, 0); + else + enable_irq(mbox->irq_recv_not_empty); +} + +/* + * lifecycle_lock held (or removal), mbox->active already cleared. The storm + * guard and this path each claim rx_irq_live under rx_lock, so exactly one + * of them balances the enable_irq(). The poller re-queues itself, which + * cancel_delayed_work_sync() handles; it never takes lifecycle_lock. + */ +static void apple_mbox_v8_rx_stop(struct apple_mbox *mbox) +{ + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + unsigned long flags; + bool live; + + spin_lock_irqsave(&mbox->rx_lock, flags); + live = priv->rx_irq_live; + priv->rx_irq_live = false; + spin_unlock_irqrestore(&mbox->rx_lock, flags); + + if (live) + disable_irq(mbox->irq_recv_not_empty); + cancel_delayed_work_sync(&priv->rx_poll_work); +} + int apple_mbox_start(struct apple_mbox *mbox) { u32 control; @@ -610,6 +995,12 @@ int apple_mbox_start(struct apple_mbox *mbox) mbox->regs + mbox->hw->irq_enable); } + if (apple_mbox_has_rx_poller(mbox)) { + WRITE_ONCE(mbox->active, true); + apple_mbox_v8_rx_start(mbox); + goto out_unlock; + } + enable_irq(mbox->irq_recv_not_empty); mbox->active = true; @@ -624,8 +1015,13 @@ static void apple_mbox_stop_locked(struct apple_mbox *mbox) if (!mbox->active) return; - mbox->active = false; - disable_irq(mbox->irq_recv_not_empty); + if (apple_mbox_has_rx_poller(mbox)) { + WRITE_ONCE(mbox->active, false); + apple_mbox_v8_rx_stop(mbox); + } else { + mbox->active = false; + disable_irq(mbox->irq_recv_not_empty); + } pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } @@ -682,13 +1078,19 @@ static void apple_mbox_quiesce(void *data) * inversion. The removal fence prevents any matching restart. */ was_active = mbox->active; - mbox->active = false; + WRITE_ONCE(mbox->active, false); mutex_unlock(&mbox->lifecycle_lock); if (was_active) { - disable_irq(mbox->irq_recv_not_empty); + if (apple_mbox_has_rx_poller(mbox)) + apple_mbox_v8_rx_stop(mbox); + else + disable_irq(mbox->irq_recv_not_empty); pm_runtime_mark_last_busy(mbox->dev); pm_runtime_put_autosuspend(mbox->dev); } + /* The v8 poller also stops on the removal fence; make sure it is gone. */ + if (apple_mbox_has_rx_poller(mbox)) + cancel_delayed_work_sync(&apple_mbox_to_priv(mbox)->rx_poll_work); /* Serialize once-only masking with the IRQ and send-error paths. */ spin_lock_irqsave(&mbox->tx_lock, flags); @@ -764,6 +1166,121 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name) } EXPORT_SYMBOL(apple_mbox_get_byname); +static void apple_mbox_iounmap(void *addr) +{ + iounmap((__force void __iomem *)addr); +} + +/* + * Validate and map one T8152 GPU ASCWrap v8 provider. Nothing here reads or + * writes device registers: the four windows are only mapped. The wrapper, + * iop-vbar and post-office window are requested exclusively; the PSD block + * is mapped without a request so that a GPU power owner may also map the + * same service block (the provider itself only ever reads it). + */ +static int apple_mbox_probe_ascwrap_v8(struct platform_device *pdev, + struct apple_mbox *mbox) +{ + struct device *dev = &pdev->dev; + struct apple_mbox_priv *priv = apple_mbox_to_priv(mbox); + struct resource *wrapper, *iop_vbar, *window, *psd; + u64 wrapper_base, iop_vbar_base, window_base, psd_base; + const char *firmware_role; + int ret; + + wrapper = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wrapper"); + iop_vbar = platform_get_resource_byname(pdev, IORESOURCE_MEM, "iop-vbar"); + window = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mailbox"); + psd = platform_get_resource_byname(pdev, IORESOURCE_MEM, "psd"); + if (!wrapper || !iop_vbar || !window || !psd || + resource_size(wrapper) != APPLE_ASCWRAP_V8_WRAPPER_SIZE || + resource_size(iop_vbar) != APPLE_ASCWRAP_V8_IOP_VBAR_SIZE || + resource_size(window) != APPLE_ASCWRAP_V8_MBOX_SIZE || + resource_size(psd) != APPLE_ASCWRAP_V8_PSD_SIZE) + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v8 provider resources\n"); + + ret = of_property_read_string(dev->of_node, "apple,firmware-role", + &firmware_role); + if (ret) + return dev_err_probe(dev, ret, + "missing ASCWrap v8 firmware role\n"); + + if (!strcmp(firmware_role, "GFX")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; + wrapper_base = APPLE_T8152_GFX_WRAPPER_BASE; + iop_vbar_base = APPLE_T8152_GFX_IOP_VBAR_BASE; + window_base = APPLE_T8152_GFX_MBOX_BASE; + psd_base = APPLE_T8152_GFX_PSD_BASE; + } else if (!strcmp(firmware_role, "GFX1")) { + mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; + wrapper_base = APPLE_T8152_GFX1_WRAPPER_BASE; + iop_vbar_base = APPLE_T8152_GFX1_IOP_VBAR_BASE; + window_base = APPLE_T8152_GFX1_MBOX_BASE; + psd_base = APPLE_T8152_GFX1_PSD_BASE; + } else { + return dev_err_probe(dev, -EINVAL, + "invalid ASCWrap v8 firmware role\n"); + } + + /* + * Fixed per-SoC CPU-physical windows. In particular the post-office + * window must be exactly this role's 0x80 bytes: the rest of the bank + * belongs to PMP, DCP, DCPEXT and SCALER0. + */ + if (wrapper->start != wrapper_base || iop_vbar->start != iop_vbar_base || + window->start != window_base || psd->start != psd_base) + return dev_err_probe(dev, -EINVAL, + "%s ASCWrap v8 resources do not match the T8152 profile\n", + firmware_role); + + mbox->ascwrap_v6.wrapper_start = wrapper->start; + mbox->ascwrap_v6.wrapper_size = resource_size(wrapper); + mbox->ascwrap_v6.iop_vbar_start = iop_vbar->start; + mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar); + mbox->ascwrap_v6.missing = ascwrap_v8_cpu_control ? + 0 : APPLE_ASCWRAP_V8_MISSING_CPU_CONTROL_APPROVAL; + priv->rx_poll_only = !ascwrap_v8_rx_irq; + INIT_DELAYED_WORK(&priv->rx_poll_work, apple_mbox_v8_rx_poll_work); + + mbox->wrapper_regs = devm_platform_ioremap_resource_byname(pdev, "wrapper"); + if (IS_ERR(mbox->wrapper_regs)) + return PTR_ERR(mbox->wrapper_regs); + + mbox->iop_vbar_regs = + devm_platform_ioremap_resource_byname(pdev, "iop-vbar"); + if (IS_ERR(mbox->iop_vbar_regs)) + return PTR_ERR(mbox->iop_vbar_regs); + + mbox->regs = devm_platform_ioremap_resource_byname(pdev, "mailbox"); + if (IS_ERR(mbox->regs)) + return PTR_ERR(mbox->regs); + + /* + * Same posted/non-posted choice devm_ioremap_resource() makes, but + * without claiming the region. Released after apple_mbox_quiesce(), + * which is registered later and may still read the PSD state. + */ + if (psd->flags & IORESOURCE_MEM_NONPOSTED) + priv->psd_regs = ioremap_np(psd->start, resource_size(psd)); + else + priv->psd_regs = ioremap(psd->start, resource_size(psd)); + if (!priv->psd_regs) + return -ENOMEM; + ret = devm_add_action_or_reset(dev, apple_mbox_iounmap, + (__force void *)priv->psd_regs); + if (ret) + return ret; + + dev_info(dev, + "%s ASCWrap v8 provider: wrapper %pR, post-office window %pR, PSD %pR; CPU control %s; receive %s\n", + firmware_role, wrapper, window, psd, + ascwrap_v8_cpu_control ? "allowed" : "refused (default)", + ascwrap_v8_rx_irq ? "recv-not-empty IRQ" : + "polled, no IRQ enabled (default)"); + return 0; +} + static int apple_mbox_probe(struct platform_device *pdev) { int ret; @@ -771,19 +1288,23 @@ static int apple_mbox_probe(struct platform_device *pdev) const char *firmware_role; const struct apple_mbox_ascwrap_v6_role_desc *role; unsigned int i; + struct apple_mbox_priv *priv; struct apple_mbox *mbox; struct device *dev = &pdev->dev; struct resource *res, *iop_vbar_res; - mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); - if (!mbox) + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) return -ENOMEM; + mbox = &priv->mbox; mbox->dev = &pdev->dev; mbox->hold_rx_in_sleep = of_property_read_bool(dev->of_node, "apple,hold-rx-in-sleep"); mbox->hw = of_device_get_match_data(dev); if (!mbox->hw) return -EINVAL; + if (mbox->hw->counted_fifo && !IS_ENABLED(CONFIG_ARM64)) + return -EOPNOTSUPP; if (mbox->hw->is_ascwrap_v6) { /* @@ -840,6 +1361,10 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->regs = (u8 __iomem *)mbox->wrapper_regs + mbox->hw->reg_offset; + } else if (mbox->hw->is_ascwrap_v8) { + ret = apple_mbox_probe_ascwrap_v8(pdev, mbox); + if (ret) + return ret; } else { res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res || resource_size(res) < mbox->hw->reg_offset + @@ -1058,6 +1583,26 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { .irq_bit_send_empty = APPLE_M3_MBOX_IRQ_A2I_EMPTY, }; +/* + * T8152/J873 GPU GFX/GFX1 ASCWrap v8: a counted FIFO at the role's + * post-office window (reg "mailbox"), plus wrapper lifecycle reads gated on + * the role's PSD service. + */ +static const struct apple_mbox_hw apple_mbox_t8152_gfx_ascwrap_v8_hw = { + .is_ascwrap_v8 = true, + .counted_fifo = true, + .control_full = GENMASK(11, 8), /* free TX slots */ + .control_empty = GENMASK(3, 0), /* RX occupancy */ + .a2i_control = 0x20, + .a2i_send0 = 0x10, + .a2i_send1 = 0x18, + .i2a_control = 0x20, + .i2a_recv0 = 0x00, + .i2a_recv1 = 0x08, + .has_irq_controls = false, +}; + +static_assert(0x20 + sizeof(u32) <= APPLE_ASCWRAP_V8_MBOX_SIZE); /* T6050's native ASCWrap v6 uses the same RUN/stop sequence. Aura grants * these complete wrapper/VBAR windows in the retained GPU aperture. */ @@ -1115,6 +1660,7 @@ static const struct apple_mbox_hw apple_mbox_t6050_apif_ascwrap_v6_hw = { static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t6031-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8122-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, + { .compatible = "apple,t8152-gfx-ascwrap-v8", .data = &apple_mbox_t8152_gfx_ascwrap_v8_hw }, { .compatible = "apple,t6040-ascwrap-v6", .data = &apple_mbox_t6040_ascwrap_v6_hw }, { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_t8140_ascwrap_v6_hw }, { .compatible = "apple,t6050-apif-ascwrap-v6", .data = &apple_mbox_t6050_apif_ascwrap_v6_hw }, From b7069ac48f534cba09316f2d84bf1a0219f24324 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 04:18:57 -0500 Subject: [PATCH 004/117] soc: apple: rtkit: let a client own the firmware crash buffer The G18 GPU firmware places its crash buffer in memory the GPU UAT already maps, so the RTKit core must neither allocate nor map it. Add the APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG ops flag, in the style of the existing CoreAnalytics flag, and RtKit::new_g17p() to create such a client from Rust. Based on earlier work by DjDeveloperr. --- drivers/soc/apple/rtkit.c | 45 +++++++++++++++++++++++++++++---- include/linux/soc/apple/rtkit.h | 9 +++++++ rust/bindings/bindings_helper.h | 1 + rust/kernel/soc/apple/rtkit.rs | 19 ++++++++++++++ 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index d352083aaa26c..188a67d28cade 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -153,6 +153,18 @@ static bool apple_rtkit_needs_coreanalytics(struct apple_rtkit *rtk) return rtk->ops->flags & APPLE_RTKIT_NEEDS_COREANALYTICS; } +static bool apple_rtkit_firmware_owns_crashlog(struct apple_rtkit *rtk) +{ + return rtk->ops->flags & APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG; +} + +/* The crashlog endpoint is started while the endpoint map is handled. */ +static bool apple_rtkit_early_crashlog(struct apple_rtkit *rtk) +{ + return apple_rtkit_coproc_places_buffers(rtk) || + apple_rtkit_firmware_owns_crashlog(rtk); +} + static int apple_rtkit_management_send(struct apple_rtkit *rtk, u8 type, u64 msg) { @@ -232,10 +244,11 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) } /* - * For co-processors that place their own buffers, the crashlog - * endpoint is started before the map is acknowledged. + * For co-processors that place their own buffers, or only their + * crashlog buffer, the crashlog endpoint is started before the map is + * acknowledged: G17P can request its crash DVA right after. */ - if (apple_rtkit_coproc_places_buffers(rtk) && base == 0 && + if (apple_rtkit_early_crashlog(rtk) && base == 0 && test_bit(APPLE_RTKIT_EP_CRASHLOG, &bitmap)) { ret = apple_rtkit_start_ep(rtk, APPLE_RTKIT_EP_CRASHLOG); if (ret) { @@ -276,7 +289,7 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_EP_TRACEKIT: /* already started while handling the map */ if (ep == APPLE_RTKIT_EP_CRASHLOG && - apple_rtkit_coproc_places_buffers(rtk)) + apple_rtkit_early_crashlog(rtk)) break; dev_dbg(rtk->dev, "RTKit: Starting system endpoint 0x%02x\n", ep); @@ -410,6 +423,14 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n", request.size, &request.iova); + /* This DVA belongs to the co-processor's own UAT: record it, unmapped. */ + if (apple_rtkit_firmware_owns_crashlog(rtk) && + ep == APPLE_RTKIT_EP_CRASHLOG && request.iova) { + *buffer = request; + buffer->is_mapped = true; + goto reply; + } + if (request.iova && !rtk->ops->shmem_setup && !rtk->ops->shmem_setup_ep) { err = -EINVAL; goto error; @@ -440,7 +461,9 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, buffer->iova); } apple_rtkit_send_message(rtk, ep, reply, NULL, false); - } else if (apple_rtkit_coproc_places_buffers(rtk)) { + } else if (apple_rtkit_coproc_places_buffers(rtk) || + (apple_rtkit_firmware_owns_crashlog(rtk) && + ep == APPLE_RTKIT_EP_CRASHLOG)) { /* acknowledge the co-processor's own buffer with its request */ apple_rtkit_send_message(rtk, ep, msg, NULL, false); } @@ -734,6 +757,18 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) dev_err(rtk->dev, "RTKit: co-processor has crashed\n"); + /* A firmware-owned crash buffer has no CPU mapping to copy from. */ + if (apple_rtkit_firmware_owns_crashlog(rtk) && + !rtk->crashlog_buffer.buffer && !rtk->crashlog_buffer.iomem) { + dev_err(rtk->dev, + "RTKit: firmware-owned crash log at DVA %pad (0x%zx bytes)\n", + &rtk->crashlog_buffer.iova, rtk->crashlog_buffer.size); + rtk->crashed = true; + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); + return; + } + /* * create a shadow copy here to make sure the co-processor isn't able * to change the log while we're dumping it. this also ensures diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 65bd28a6f0240..3a898e6f3d35c 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -57,6 +57,15 @@ struct apple_rtkit_inherited_buffer { /* This client requires endpoint 7 setup before IOP power can be acknowledged. */ #define APPLE_RTKIT_NEEDS_COREANALYTICS BIT(1) +/* + * The co-processor places its crashlog buffer in its own address space (the + * G17P/G18G GPU UAT), not in this device's DMA domain. That buffer request is + * acknowledged by sending it back unchanged, without mapping it, and a crash + * is reported with no host copy of the log. The crashlog endpoint is started + * while the endpoint map is handled. Other buffers are allocated as usual. + */ +#define APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG BIT(2) + /* * Struct to represent implementation-specific RTKit operations. * diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 56f9f84689804..2731a39a0255c 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -158,6 +158,7 @@ const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; const u32 RUST_CONST_HELPER_APPLE_RTKIT_NEEDS_COREANALYTICS = APPLE_RTKIT_NEEDS_COREANALYTICS; +const u32 RUST_CONST_HELPER_APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG = APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG; const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 73896fea53e00..5e0763ceb0529 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -245,6 +245,11 @@ impl RtKit { ..Self::VTABLE }; + const G17P_VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + flags: Self::VTABLE.flags | bindings::APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG, + ..Self::VTABLE + }; + /// Creates a new RTKit client for a given device and optional mailbox name or index. pub fn new( dev: &device::Device, @@ -268,6 +273,20 @@ impl RtKit { Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::COREANALYTICS_VTABLE) } + /// Creates a G17P client whose crash DVA belongs to the GPU UAT. + /// + /// The firmware's crashlog buffer request is acknowledged without mapping it, and a + /// crash reaches [`Operations::crashed`] with no host copy of the log. Other buffer + /// requests are handled as for [`RtKit::new`]. + pub fn new_g17p( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ) -> Result { + Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::G17P_VTABLE) + } + fn new_with_ops( dev: &device::Device, mbox_name: Option<&'static CStr>, From b2e300ad24ade07a0d11c0e16ab9d22258bab903 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 04:18:57 -0500 Subject: [PATCH 005/117] drm/asahi: g18: import the G18G (T8152) backend Import the G18 backend of the Mac mini M6 (J873, T8152, G18G B1) from our earlier out-of-tree M6 port, which ran it on an older copy of this driver, together with its UAT support, mmu/t8152.rs. That port passed every test of the OpenGL 4.6 CTS once on the M6. The files are unchanged apart from some comments. Nothing builds them yet: the following commits move the backend to this driver's command layout and core, and then wire it up behind its own Kconfig option. Credits: - The G17P init-data model in g18/fw/initdata.rs (InstanceRole, ChannelEntry, the UAT level descriptors, the region runs, INITDATA_VERSION_WORD and the serializers around them, about 227 lines) is DjDeveloperr's work, from his August 2026 rebuild of the G17P (T8140) init-data model. Eryk Wieliczko carried it into the port, where it was adapted for G18. It was not published before. - The rest of the backend is Acelogic's. --- drivers/gpu/drm/asahi/g18/address_map.rs | 214 + .../gpu/drm/asahi/g18/boot/compute_session.rs | 190 + drivers/gpu/drm/asahi/g18/boot/mod.rs | 2130 ++++ drivers/gpu/drm/asahi/g18/boot/recovery.rs | 432 + .../gpu/drm/asahi/g18/boot/render_stage.rs | 41 + drivers/gpu/drm/asahi/g18/boot/session.rs | 309 + drivers/gpu/drm/asahi/g18/buffer/live.rs | 32 + drivers/gpu/drm/asahi/g18/buffer/mod.rs | 10 + drivers/gpu/drm/asahi/g18/buffer/object.rs | 201 + drivers/gpu/drm/asahi/g18/buffer/pool.rs | 708 ++ .../drm/asahi/g18/buffer/pool_application.rs | 38 + .../drm/asahi/g18/buffer/pool_owner_probe.rs | 254 + .../asahi/g18/buffer/pool_shared_compute.rs | 229 + drivers/gpu/drm/asahi/g18/compat/dep_stats.rs | 104 + .../gpu/drm/asahi/g18/compat/dvfs_parity.rs | 125 + drivers/gpu/drm/asahi/g18/compat/e2e.rs | 372 + drivers/gpu/drm/asahi/g18/compat/g17p.rs | 574 ++ drivers/gpu/drm/asahi/g18/compat/gem_stats.rs | 91 + drivers/gpu/drm/asahi/g18/compat/host_path.rs | 780 ++ drivers/gpu/drm/asahi/g18/compat/mem_path.rs | 236 + drivers/gpu/drm/asahi/g18/compat/mod.rs | 28 + .../gpu/drm/asahi/g18/compat/queue_slots.rs | 188 + .../asahi/g18/compat/recovery_isolation.rs | 644 ++ .../gpu/drm/asahi/g18/compat/recovery_path.rs | 399 + .../gpu/drm/asahi/g18/compat/stackwatch.rs | 150 + .../gpu/drm/asahi/g18/compat/timing_buffer.rs | 929 ++ drivers/gpu/drm/asahi/g18/debug.rs | 56 + drivers/gpu/drm/asahi/g18/drm.rs | 709 ++ .../drm/asahi/g18/fixture/compute_payload.rs | 179 + .../drm/asahi/g18/fixture/compute_triangle.rs | 103 + drivers/gpu/drm/asahi/g18/fixture/mod.rs | 10 + .../drm/asahi/g18/fixture/render_pixels.rs | 42 + drivers/gpu/drm/asahi/g18/fw/compute.rs | 446 + .../gpu/drm/asahi/g18/fw/compute_memory.rs | 66 + .../gpu/drm/asahi/g18/fw/compute_params.rs | 1619 +++ drivers/gpu/drm/asahi/g18/fw/compute_state.rs | 64 + drivers/gpu/drm/asahi/g18/fw/config.rs | 137 + drivers/gpu/drm/asahi/g18/fw/events.rs | 84 + .../gpu/drm/asahi/g18/fw/graphics_payload.rs | 188 + drivers/gpu/drm/asahi/g18/fw/initdata.rs | 6549 ++++++++++++ drivers/gpu/drm/asahi/g18/fw/m5_render.rs | 512 + .../gpu/drm/asahi/g18/fw/m5_render_target.rs | 121 + drivers/gpu/drm/asahi/g18/fw/mod.rs | 50 + .../gpu/drm/asahi/g18/fw/parameter_buffer.rs | 227 + drivers/gpu/drm/asahi/g18/fw/render.rs | 811 ++ .../gpu/drm/asahi/g18/fw/render_assembly.rs | 167 + .../gpu/drm/asahi/g18/fw/render_command.rs | 925 ++ drivers/gpu/drm/asahi/g18/fw/render_params.rs | 2877 ++++++ .../drm/asahi/g18/fw/render_passthrough.rs | 200 + drivers/gpu/drm/asahi/g18/fw/rtkit.rs | 73 + drivers/gpu/drm/asahi/g18/fw/scheduler.rs | 69 + drivers/gpu/drm/asahi/g18/fw/sksm.rs | 392 + drivers/gpu/drm/asahi/g18/fw/store.rs | 82 + drivers/gpu/drm/asahi/g18/fw/submission.rs | 9012 +++++++++++++++++ drivers/gpu/drm/asahi/g18/fw/uma.rs | 297 + drivers/gpu/drm/asahi/g18/fw/vm_layout.rs | 84 + drivers/gpu/drm/asahi/g18/hw.rs | 167 + drivers/gpu/drm/asahi/g18/identity.rs | 173 + .../gpu/drm/asahi/g18/initdata/application.rs | 82 + .../asahi/g18/initdata/application_compute.rs | 534 + drivers/gpu/drm/asahi/g18/initdata/mod.rs | 948 ++ drivers/gpu/drm/asahi/g18/mod.rs | 33 + drivers/gpu/drm/asahi/g18/probe.rs | 339 + drivers/gpu/drm/asahi/g18/queue/graphics.rs | 1077 ++ .../asahi/g18/queue/graphics_application.rs | 145 + drivers/gpu/drm/asahi/g18/queue/mod.rs | 10 + drivers/gpu/drm/asahi/g18/queue/probe.rs | 438 + drivers/gpu/drm/asahi/g18/queue/scheduler.rs | 190 + drivers/gpu/drm/asahi/g18/regs.rs | 451 + drivers/gpu/drm/asahi/g18/runtime/compute.rs | 289 + .../drm/asahi/g18/runtime/compute_payload.rs | 193 + .../gpu/drm/asahi/g18/runtime/graphics_job.rs | 375 + .../g18/runtime/graphics_job_application.rs | 202 + .../drm/asahi/g18/runtime/graphics_memory.rs | 239 + drivers/gpu/drm/asahi/g18/runtime/mod.rs | 12 + drivers/gpu/drm/asahi/g18/uapi/application.rs | 599 ++ .../drm/asahi/g18/uapi/application_runtime.rs | 68 + drivers/gpu/drm/asahi/g18/uapi/command.rs | 2226 ++++ .../gpu/drm/asahi/g18/uapi/compute_inputs.rs | 224 + .../drm/asahi/g18/uapi/compute_metadata.rs | 51 + .../gpu/drm/asahi/g18/uapi/compute_plan.rs | 167 + .../drm/asahi/g18/uapi/graphics_scratch.rs | 259 + drivers/gpu/drm/asahi/g18/uapi/mod.rs | 20 + .../gpu/drm/asahi/g18/uapi/render_policy.rs | 237 + drivers/gpu/drm/asahi/mmu/t8152.rs | 651 ++ 85 files changed, 45958 insertions(+) create mode 100644 drivers/gpu/drm/asahi/g18/address_map.rs create mode 100644 drivers/gpu/drm/asahi/g18/boot/compute_session.rs create mode 100644 drivers/gpu/drm/asahi/g18/boot/mod.rs create mode 100644 drivers/gpu/drm/asahi/g18/boot/recovery.rs create mode 100644 drivers/gpu/drm/asahi/g18/boot/render_stage.rs create mode 100644 drivers/gpu/drm/asahi/g18/boot/session.rs create mode 100644 drivers/gpu/drm/asahi/g18/buffer/live.rs create mode 100644 drivers/gpu/drm/asahi/g18/buffer/mod.rs create mode 100644 drivers/gpu/drm/asahi/g18/buffer/object.rs create mode 100644 drivers/gpu/drm/asahi/g18/buffer/pool.rs create mode 100644 drivers/gpu/drm/asahi/g18/buffer/pool_application.rs create mode 100644 drivers/gpu/drm/asahi/g18/buffer/pool_owner_probe.rs create mode 100644 drivers/gpu/drm/asahi/g18/buffer/pool_shared_compute.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/dep_stats.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/dvfs_parity.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/e2e.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/g17p.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/gem_stats.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/host_path.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/mem_path.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/mod.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/queue_slots.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/recovery_isolation.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/recovery_path.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/stackwatch.rs create mode 100644 drivers/gpu/drm/asahi/g18/compat/timing_buffer.rs create mode 100644 drivers/gpu/drm/asahi/g18/debug.rs create mode 100644 drivers/gpu/drm/asahi/g18/drm.rs create mode 100644 drivers/gpu/drm/asahi/g18/fixture/compute_payload.rs create mode 100644 drivers/gpu/drm/asahi/g18/fixture/compute_triangle.rs create mode 100644 drivers/gpu/drm/asahi/g18/fixture/mod.rs create mode 100644 drivers/gpu/drm/asahi/g18/fixture/render_pixels.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/compute.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/compute_memory.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/compute_params.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/compute_state.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/config.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/events.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/graphics_payload.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/initdata.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/m5_render.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/m5_render_target.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/mod.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/parameter_buffer.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/render.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/render_assembly.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/render_command.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/render_params.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/render_passthrough.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/rtkit.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/scheduler.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/sksm.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/store.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/submission.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/uma.rs create mode 100644 drivers/gpu/drm/asahi/g18/fw/vm_layout.rs create mode 100644 drivers/gpu/drm/asahi/g18/hw.rs create mode 100644 drivers/gpu/drm/asahi/g18/identity.rs create mode 100644 drivers/gpu/drm/asahi/g18/initdata/application.rs create mode 100644 drivers/gpu/drm/asahi/g18/initdata/application_compute.rs create mode 100644 drivers/gpu/drm/asahi/g18/initdata/mod.rs create mode 100644 drivers/gpu/drm/asahi/g18/mod.rs create mode 100644 drivers/gpu/drm/asahi/g18/probe.rs create mode 100644 drivers/gpu/drm/asahi/g18/queue/graphics.rs create mode 100644 drivers/gpu/drm/asahi/g18/queue/graphics_application.rs create mode 100644 drivers/gpu/drm/asahi/g18/queue/mod.rs create mode 100644 drivers/gpu/drm/asahi/g18/queue/probe.rs create mode 100644 drivers/gpu/drm/asahi/g18/queue/scheduler.rs create mode 100644 drivers/gpu/drm/asahi/g18/regs.rs create mode 100644 drivers/gpu/drm/asahi/g18/runtime/compute.rs create mode 100644 drivers/gpu/drm/asahi/g18/runtime/compute_payload.rs create mode 100644 drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs create mode 100644 drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs create mode 100644 drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs create mode 100644 drivers/gpu/drm/asahi/g18/runtime/mod.rs create mode 100644 drivers/gpu/drm/asahi/g18/uapi/application.rs create mode 100644 drivers/gpu/drm/asahi/g18/uapi/application_runtime.rs create mode 100644 drivers/gpu/drm/asahi/g18/uapi/command.rs create mode 100644 drivers/gpu/drm/asahi/g18/uapi/compute_inputs.rs create mode 100644 drivers/gpu/drm/asahi/g18/uapi/compute_metadata.rs create mode 100644 drivers/gpu/drm/asahi/g18/uapi/compute_plan.rs create mode 100644 drivers/gpu/drm/asahi/g18/uapi/graphics_scratch.rs create mode 100644 drivers/gpu/drm/asahi/g18/uapi/mod.rs create mode 100644 drivers/gpu/drm/asahi/g18/uapi/render_policy.rs create mode 100644 drivers/gpu/drm/asahi/mmu/t8152.rs diff --git a/drivers/gpu/drm/asahi/g18/address_map.rs b/drivers/gpu/drm/asahi/g18/address_map.rs new file mode 100644 index 0000000000000..1a73e894dd4a1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/address_map.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! J873 fixed VA reservations. Addresses are policy, never captured pointers. +//! The allocator owns suballocations inside the named legacy arenas. Compact +//! aliases intentionally share physical backing but have disjoint VA extents. +#![allow(dead_code)] + +pub(crate) const PAGE: u64 = 0x4000; +pub(crate) const KERNEL_BASE: u64 = 0xffff_fc20_0000_0000; +pub(crate) const COMPACT_BIAS: u64 = 0x10_0000_0000; +// Optional H1 control: the first Gallium VDM address from retained run123. +pub(crate) const GRAPHICS_HIGH_VDM: u64 = 0x5f_ffe2_8000; +// Native PPP page 0x58000 aliases the first Gallium PPP page. Preserve offsets +// between all four native PPP pages; these aliases share original backing. +pub(crate) const GRAPHICS_HIGH_PPP_BASE: u64 = 0x5f_ffea_c000; +pub(crate) const GRAPHICS_HIGH_PPP_PAGES: [u64; 4] = [0, 0x48000, 0x58000, 0x68000]; +pub(crate) const PB_TABLE: u64 = KERNEL_BASE + 0x015e_0000; +pub(crate) const UMA_TABLE: u64 = KERNEL_BASE + 0x015e_8000; +pub(crate) const COMPUTE_GRAPH: u64 = KERNEL_BASE + 0xc000_0000; +pub(crate) const COMPUTE_ENTRY_HIGH: u64 = KERNEL_BASE + 0x0039_0000; +pub(crate) const COMPUTE_ENTRY_LOW: u64 = 0x70_005f_0000; +pub(crate) const DYNAMIC_HIGH_START: u64 = KERNEL_BASE + 0x0168_0000; +pub(crate) const DYNAMIC_HIGH_END: u64 = KERNEL_BASE + 0x1000_0000; +pub(crate) const POOL_DIRECTORY: u64 = 0x70_0200_0000; +pub(crate) const POOL_BLOCK_LIST: u64 = 0x70_0400_0000; +pub(crate) const POOL_BLOCKS: u64 = 0x70_0500_0000; +pub(crate) const POOL_BLOCK_STRIDE: u64 = 0x20_0000; +// Job-VM-only 0x1a510 object for g18_app_preempt_mode=1: the first pages of +// the VM_CREATE auxiliary driver arena (m5_vm::auxiliary_range), which +// gem_bind refuses to userspace. Native uses a 48 KiB process object. +pub(crate) const APPLICATION_COMPUTE_PREEMPT_DRIVER: u64 = COMPACT_BIAS + (1 << 32); +pub(crate) const APPLICATION_COMPUTE_PREEMPT_DRIVER_BYTES: u64 = 0xc000; + +// Graphics reservations are beyond both the compute graph and its low arena. +// Queue/PB owner integration must consume these, or update this table as one +// unit; the legacy M5 graph/entries collide with COMPUTE_GRAPH/ENTRY_*. +pub(crate) const GRAPHICS_QUEUE_HIGH: u64 = KERNEL_BASE + 0xc200_0000; +pub(crate) const GRAPHICS_QUEUE_BYTES: usize = 0x40000; +pub(crate) const GRAPHICS_ENTRY_HIGH: u64 = KERNEL_BASE + 0x0050_0000; +pub(crate) const GRAPHICS_ENTRY_LOW: u64 = 0x70_1100_0000; +pub(crate) const GRAPHICS_ENTRY_BYTES: usize = 0x40000; +pub(crate) const GRAPHICS_JOB_LIST: u64 = GRAPHICS_QUEUE_HIGH + 0x400; +pub(crate) const GRAPHICS_QUEUE_CONTROL: u64 = GRAPHICS_QUEUE_HIGH + 0x500; +pub(crate) const GRAPHICS_JOB_HIGH: u64 = KERNEL_BASE + 0xc210_0000; +pub(crate) const GRAPHICS_JOB_LOW: u64 = 0x70_1200_0000; +pub(crate) const GRAPHICS_JOB_BYTES: usize = 0x40000; +pub(crate) const GRAPHICS_SIGNALS: u64 = KERNEL_BASE + 0xc218_0000; +pub(crate) const GRAPHICS_SIGNALS_BYTES: usize = 0x10000; +pub(crate) const GRAPHICS_SCHEDULER_START: u64 = KERNEL_BASE + 0xc220_0000; +pub(crate) const GRAPHICS_SCHEDULER_END: u64 = GRAPHICS_SCHEDULER_START + 0x8000; +pub(crate) const GRAPHICS_PB_HIGH: u64 = KERNEL_BASE + 0xc230_0000; +pub(crate) const GRAPHICS_PB_BYTES: usize = 0x100000; +pub(crate) const GRAPHICS_UMA_HIGH: u64 = KERNEL_BASE + 0xc300_0000; +pub(crate) const GRAPHICS_UMA_LIST: u64 = GRAPHICS_UMA_HIGH + 0x34000; +pub(crate) const GRAPHICS_UMA_STATE: u64 = GRAPHICS_UMA_HIGH + 0x38000; +pub(crate) const GRAPHICS_UMA_COUNTER: u64 = GRAPHICS_UMA_HIGH + 0x3c000; +pub(crate) const GRAPHICS_UMA_BACKING: u64 = GRAPHICS_UMA_HIGH + 0x100000; +pub(crate) const GRAPHICS_UMA_BYTES: u64 = 0x3000000; +pub(crate) const GRAPHICS_UMA_GROWTH: u64 = GRAPHICS_UMA_HIGH + 0x2a00000; +pub(crate) const GRAPHICS_IDEAL_BLOCKS: u64 = 0x70_2000_0000; +pub(crate) const GRAPHICS_TARGET_OFFSET: u64 = 0x0100_0000; +pub(crate) const GRAPHICS_TARGET_BYTES: usize = 0x0100_0000; +pub(crate) const GRAPHICS_PB_OFFSET: u64 = 0x0200_0000; +pub(crate) const GRAPHICS_PB_COMPACT_BYTES: usize = 0x0100_0000; +pub(crate) const PB_GRAPH: u64 = GRAPHICS_PB_HIGH + 0x4000; +pub(crate) const PB_PAGE_LIST_HIGH: u64 = GRAPHICS_PB_HIGH + 0x10000; +pub(crate) const PB_PAGE_LIST_LOW: u64 = GRAPHICS_PB_OFFSET; +pub(crate) const PB_PAGE_LIST_BYTES: usize = 0x1c000; +pub(crate) const PB_METRICS_HIGH: u64 = GRAPHICS_PB_HIGH + 0x30000; +pub(crate) const PB_METRICS_LOW: u64 = GRAPHICS_PB_OFFSET + 0x20000; +pub(crate) const PB_SCENE_SCRATCH: u64 = GRAPHICS_PB_OFFSET + 0x28000; +pub(crate) const PB_DISCARD: u64 = GRAPHICS_PB_OFFSET + 0x30000; +pub(crate) const PB_BLOCKS: u64 = GRAPHICS_PB_OFFSET + 0x100000; +pub(crate) const QUEUE_IDS: [u8; 2] = [9, 10]; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Region { pub name: &'static str, pub base: u64, pub bytes: u64 } +const fn r(name: &'static str, base: u64, bytes: u64) -> Region { Region { name, base, bytes } } + +pub(crate) const REGIONS: &[Region] = &[ + r("capture-compact", 0, 0x78000), + r("capture-compact-biased", COMPACT_BIAS, 0x78000), + r("capture-high-vdm-control", GRAPHICS_HIGH_VDM, PAGE), + r("capture-high-ppp-0", GRAPHICS_HIGH_PPP_BASE - 0x58000, PAGE), + r("capture-high-ppp-48000", GRAPHICS_HIGH_PPP_BASE - 0x10000, PAGE), + r("capture-high-ppp-58000", GRAPHICS_HIGH_PPP_BASE, PAGE), + r("capture-high-ppp-68000", GRAPHICS_HIGH_PPP_BASE + 0x10000, PAGE), + r("capture-data", 0x100_0000_0000, 0x3d8000), + r("init-private-roots-fwctl", KERNEL_BASE + 0x20000, 0x1a4000), + r("pb-table", PB_TABLE, 0x4000), r("uma-table", UMA_TABLE, 0x8000), + r("dynamic-high-pool-scheduler-compute", DYNAMIC_HIGH_START, + DYNAMIC_HIGH_END - DYNAMIC_HIGH_START), + r("compute-queue", COMPUTE_GRAPH, PAGE), + r("compute-entry-high", COMPUTE_ENTRY_HIGH, 0x20000), + r("compute-entry-low", COMPUTE_ENTRY_LOW, 0x20000), + r("init-shared", KERNEL_BASE + 0xc0788000, 0x28000), + r("init-sentinel", KERNEL_BASE + 0xc07e0000, PAGE), + r("completion-high", KERNEL_BASE + 0xc07e8000, 4 * 0x28000), + r("pb-table-low", 0x70_0183_8000, 0x4000), + r("uma-table-low", 0x70_0184_0000, 0x8000), + r("completion-low", 0x70_0184_8000, 4 * 0x28000), + r("pool-directory-reservation", POOL_DIRECTORY, 0x2000000), + r("pool-block-list", POOL_BLOCK_LIST, PAGE), + r("application-compute-directory", POOL_DIRECTORY + 0x40000000, 0x2000000), + r("application-compute-block-list", POOL_BLOCK_LIST + 0x40000000, PAGE), + r("application-compute-block0", POOL_BLOCKS + 0x40000000, 0x204000), + r("application-compute-block1", POOL_BLOCKS + 0x40300000, 0x204000), + r("application-compute-block2", POOL_BLOCKS + 0x40600000, 0x204000), + r("application-compute-block3", POOL_BLOCKS + 0x40900000, 0x204000), + r("application-compute-preempt", 0x70_5000_0000, PAGE), + r("application-compute-preempt-driver", APPLICATION_COMPUTE_PREEMPT_DRIVER, + APPLICATION_COMPUTE_PREEMPT_DRIVER_BYTES + PAGE), + // Replacement views have independent sparse directories and block lists. + // Each scratch aperture includes every supported 2-MiB block plus guards + // at a 3-MiB stride; four-GiB bank spacing also covers the full 2-GiB plan. + r("compute-replacement-a-directory", POOL_DIRECTORY + 0x2_4000_0000, 0x2000000), + r("compute-replacement-a-list", POOL_BLOCK_LIST + 0x2_4000_0000, PAGE), + r("compute-replacement-a-blocks", POOL_BLOCKS + 0x2_4000_0000, 0xc0000000), + r("compute-replacement-b-directory", POOL_DIRECTORY + 0x3_4000_0000, 0x2000000), + r("compute-replacement-b-list", POOL_BLOCK_LIST + 0x3_4000_0000, PAGE), + r("compute-replacement-b-blocks", POOL_BLOCKS + 0x3_4000_0000, 0xc0000000), + r("pool-block0-guard", POOL_BLOCKS, 0x104000), + r("pool-block1-guard", POOL_BLOCKS + POOL_BLOCK_STRIDE, 0x104000), + r("pool-block2-guard", POOL_BLOCKS + 2 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block3-guard", POOL_BLOCKS + 3 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block4-guard", POOL_BLOCKS + 4 * POOL_BLOCK_STRIDE, 0x104000), + // Compute currently allocates five blocks. Reserve twelve for the pending + // graphics pool so its additional backing cannot enter another arena. + r("pool-block5-guard", POOL_BLOCKS + 5 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block6-guard", POOL_BLOCKS + 6 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block7-guard", POOL_BLOCKS + 7 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block8-guard", POOL_BLOCKS + 8 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block9-guard", POOL_BLOCKS + 9 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block10-guard", POOL_BLOCKS + 10 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block11-guard", POOL_BLOCKS + 11 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block12-guard", POOL_BLOCKS + 12 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block13-guard", POOL_BLOCKS + 13 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block14-guard", POOL_BLOCKS + 14 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block15-guard", POOL_BLOCKS + 15 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block16-guard", POOL_BLOCKS + 16 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block17-guard", POOL_BLOCKS + 17 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block18-guard", POOL_BLOCKS + 18 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block19-guard", POOL_BLOCKS + 19 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block20-guard", POOL_BLOCKS + 20 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block21-guard", POOL_BLOCKS + 21 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block22-guard", POOL_BLOCKS + 22 * POOL_BLOCK_STRIDE, 0x104000), + r("pool-block23-guard", POOL_BLOCKS + 23 * POOL_BLOCK_STRIDE, 0x104000), + r("compute-low-arena", 0x70_0800_0000, 0x8000000), + r("graphics-queues", GRAPHICS_QUEUE_HIGH, GRAPHICS_QUEUE_BYTES as u64), + r("graphics-entries-high", GRAPHICS_ENTRY_HIGH, GRAPHICS_ENTRY_BYTES as u64), + r("graphics-entries-low", GRAPHICS_ENTRY_LOW, GRAPHICS_ENTRY_BYTES as u64), + r("graphics-job-high", GRAPHICS_JOB_HIGH, GRAPHICS_JOB_BYTES as u64), + r("graphics-job-low", GRAPHICS_JOB_LOW, GRAPHICS_JOB_BYTES as u64), + r("graphics-signals", GRAPHICS_SIGNALS, GRAPHICS_SIGNALS_BYTES as u64), + r("graphics-scheduler", GRAPHICS_SCHEDULER_START, + GRAPHICS_SCHEDULER_END - GRAPHICS_SCHEDULER_START), + r("graphics-pb-high", GRAPHICS_PB_HIGH, GRAPHICS_PB_BYTES as u64), + r("graphics-uma-high", GRAPHICS_UMA_HIGH, GRAPHICS_UMA_BYTES), + r("graphics-ideal-blocks", GRAPHICS_IDEAL_BLOCKS, 40 * POOL_BLOCK_STRIDE), + r("graphics-target", GRAPHICS_TARGET_OFFSET, GRAPHICS_TARGET_BYTES as u64), + r("graphics-target-biased", COMPACT_BIAS + GRAPHICS_TARGET_OFFSET, GRAPHICS_TARGET_BYTES as u64), + r("graphics-pb", GRAPHICS_PB_OFFSET, GRAPHICS_PB_COMPACT_BYTES as u64), + r("graphics-pb-biased", COMPACT_BIAS + GRAPHICS_PB_OFFSET, GRAPHICS_PB_COMPACT_BYTES as u64), +]; + +pub(crate) const fn nonoverlapping(regions: &[Region]) -> bool { + let mut i = 0; + while i < regions.len() { + let a = regions[i]; + if a.bytes == 0 || a.base & (PAGE - 1) != 0 || a.bytes & (PAGE - 1) != 0 + || a.base > u64::MAX - a.bytes { return false; } + let mut j = 0; + while j < i { + let b = regions[j]; + if a.base < b.base + b.bytes && b.base < a.base + a.bytes { return false; } + j += 1; + } + i += 1; + } + true +} +const _: () = assert!(nonoverlapping(REGIONS)); + +pub(crate) const GRAPHICS_UMA_REGIONS: &[Region] = &[ + r("uma-control", GRAPHICS_UMA_HIGH, 0x40000), + r("uma-ideal-backing", GRAPHICS_UMA_BACKING, 40 * 0x100000), + r("uma-ideal-growth", GRAPHICS_UMA_GROWTH, 12 * 21 * PAGE), +]; +const _: () = assert!(nonoverlapping(GRAPHICS_UMA_REGIONS)); +const _: () = assert!(GRAPHICS_UMA_GROWTH + 12 * 21 * PAGE <= GRAPHICS_UMA_HIGH + GRAPHICS_UMA_BYTES); + +pub(crate) const PB_REGIONS: &[Region] = &[ + r("pb-state", GRAPHICS_PB_HIGH, PAGE), r("pb-graph", PB_GRAPH, 0x8000), + r("pb-page-list-high", PB_PAGE_LIST_HIGH, PB_PAGE_LIST_BYTES as u64), + r("pb-metrics-high", PB_METRICS_HIGH, PAGE), + r("pb-page-list-low", PB_PAGE_LIST_LOW, PB_PAGE_LIST_BYTES as u64), + r("pb-metrics-low", PB_METRICS_LOW, PAGE), r("pb-scene-scratch", PB_SCENE_SCRATCH, PAGE), + r("pb-discard", PB_DISCARD, 0x8000), r("pb-blocks", PB_BLOCKS, 8 * 0x80000), +]; +const _: () = assert!(nonoverlapping(PB_REGIONS)); + +const fn pb_regions_contained() -> bool { + let mut i = 0; + while i < PB_REGIONS.len() { + let child = PB_REGIONS[i]; + let high = child.base >= GRAPHICS_PB_HIGH + && child.base + child.bytes <= GRAPHICS_PB_HIGH + GRAPHICS_PB_BYTES as u64; + let low = child.base >= GRAPHICS_PB_OFFSET + && child.base + child.bytes <= GRAPHICS_PB_OFFSET + GRAPHICS_PB_COMPACT_BYTES as u64; + if !high && !low { return false; } + i += 1; + } + true +} +const _: () = assert!(pb_regions_contained()); diff --git a/drivers/gpu/drm/asahi/g18/boot/compute_session.rs b/drivers/gpu/drm/asahi/g18/boot/compute_session.rs new file mode 100644 index 0000000000000..8af55d934298f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/boot/compute_session.rs @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Application compute submission on the retained firmware session. +use super::*; + +impl G18Session { + #[inline(never)] + pub(super) fn submit_application_compute(&mut self, vm: &mmu::Vm, + translated: &crate::g18::uapi::command::TranslatedComputeCommand) + -> core::result::Result<(), Failure> { + self.poll_idle_faults("application compute prepare")?; + let (command, pool, generation, knobs) = self.memory.as_mut().ok_or(EIO)? + .prepare_application_compute(&self.dev, vm, translated)?; + if generation == 1 { + self.queue_probe.as_mut().ok_or(EIO)?.prepare_initial_context(command.context_id)?; + for phase in 1..=3 { + self.check_faults("application compute binding")?; + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + match phase { + 1 => { + if probe.counters()? != [0, 0, 1] || probe.slot() != [0; 5] { + return Err(EBUSY.into()); + } + }, + 2 => probe.prepare_scheduler_bind()?, + _ => probe.prepare_pool_bind(pool)?, + } + let after = self.memory.as_mut().ok_or(EIO)? + .publish_application_compute_outer(phase)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + let mut passed = false; + for _ in 0..CONTROL_POLLS { + self.check_faults("application compute binding")?; + let outer = self.memory.as_mut().ok_or(EIO)?.compute_outer_counters()?; + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + if outer == [after; 3] && probe.counters()? == [phase; 3] + && G18QueueProbe::installed(probe.slot()) + && (phase < 2 || G18QueueProbe::scheduler_bound(probe.scheduler_status()?)) + && (phase < 3 || G18QueueProbe::pool_bound_for_context( + probe.pool_status(), pool, command.context_id)) { + passed = true; break; + } + fsleep(Delta::from_millis(1)); + } + if !passed { return Err(ETIMEDOUT.into()); } + dev_info!(self.device(), "G18_APP_COMPUTE_BIND_PASS phase={}\n", phase); + } + } + let (previous_pool, bound_context) = self.memory.as_ref().ok_or(EIO)? + .application_compute_previous_binding()?; + let previous_context = if generation == 1 { command.context_id } else { bound_context }; + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + if let Some(what) = probe.compute_unready(previous_pool, previous_context, generation)? { + let (counters, slot, scheduler, pool_words) = probe.compute_state()?; + dev_err!(self.device(), + "G18_APP_COMPUTE_UNREADY compute={} unready={} counters={:?} slot={:x?} scheduler={:x?} pool={:x?}\n", + generation, what, counters, slot, scheduler, pool_words); + return Err(EINVAL.into()); + } + self.memory.as_mut().ok_or(EIO)?.install_application_compute_pool(&self.dev)?; + self.queue_probe.as_mut().ok_or(EIO)? + .prepare_application_compute_rebind(command, pool, previous_pool, previous_context, generation)?; + self.poll_idle_faults("application compute pre-kick")?; + let memory = self.memory.as_mut().ok_or(EIO)?; + let completed = memory.compute().completion_snapshot()?; + if completed.stamps != [0; 2] || completed.timestamps != [0; 2] + || memory.completion_masks() != [0; 2] { return Err(EBUSY.into()); } + // The exact knob snapshot this command was built from (g18_app_*). + crate::g18_kick_info!(self.device(), "G18_APP_COMPUTE_KNOBS compute={} {}\n", generation, knobs); + let memory = self.memory.as_mut().ok_or(EIO)?; + let roots = memory.compute().publish_context()?; + if memory.scheduler().publish_kick(0)? != generation { return Err(EIO.into()); } + self.queue_probe.as_mut().ok_or(EIO)?.publish_compute_job()?; + let inner_after = (3 + generation * 3) % 256; + let outer_after = self.memory.as_mut().ok_or(EIO)? + .publish_application_compute_outer(inner_after)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + crate::g18_kick_info!(self.device(), + "G18_APP_KICK compute={} context={} roots={:x?} stamp={:#x}\n", + generation, command.context_id, roots, command.expected_stamp); + let expected_stamp = command.expected_stamp; + // Phases: outer, inner, scheduler submitted, completed, linked, stamp, + // completion mask, pool binding visibility. + let kicked = Instant::::now(); + let mut first = [0i64; 8]; + let mut gpu_ticks = 0u64; + let mut pool_pending_reported = false; + let end = self.wait_job("application compute completion", |session, ms, recovered, report| { + let inner = session.queue_probe.as_mut().ok_or(EIO)?.counters()?; + let pool_words = session.queue_probe.as_ref().ok_or(EIO)?.pool_status(); + let pool_bound = G18QueueProbe::pool_bound_for_context( + pool_words, pool, command.context_id); + let memory = session.memory.as_mut().ok_or(EIO)?; + let outer = memory.compute_outer_counters()?; + let completed = memory.compute().completion_snapshot()?; + let scheduler = memory.scheduler().snapshot(0)?; + let masks = memory.completion_masks(); + // What the firmware owns: the queue consumed and the scheduler + // retired. After a recovery the dispatch never completes, so its + // stamp, timestamps and completion event are not required. + let consumed = outer == [outer_after; 3] && inner == [inner_after; 3] + && scheduler.submitted == generation && scheduler.completed == generation + && scheduler.linked == 0; + let finished = completed.stamps[0] == expected_stamp + && completed.timestamps[0] != 0 + && completed.timestamps[1] >= completed.timestamps[0] + && masks[0] & (1 << QUEUE_ID) != 0; + recovery::note_phases(&mut first, &kicked, &[outer == [outer_after; 3], inner == [inner_after; 3], + scheduler.submitted == generation, scheduler.completed == generation, scheduler.linked == 0, + completed.stamps[0] == expected_stamp, masks[0] & (1 << QUEUE_ID) != 0, pool_bound]); + if finished { + gpu_ticks = completed.timestamps[1] - completed.timestamps[0]; + } + // Queue/stamp completion can precede an observable pool-table + // update. Keep both owners until the full binding is visible, + // within the same bounded wait as command retirement. + if consumed && finished && !pool_bound && !pool_pending_reported { + dev_info!(session.device(), + "G18_APP_COMPUTE_POOL_PENDING generation={} binding={:?} observed={:x?}; owners retained\n", + generation, pool, pool_words); + pool_pending_reported = true; + } + if report { + dev_info!(session.device(), + "G18_APP_COMPUTE_WAIT generation={} ms={} recovered={} outer={:?} inner={:?} stamps={:x?} timestamps={:x?} masks={:x?} scheduler={:?} expected_outer={} expected_inner={} expected_stamp={:#x} consumed={} finished={} pool_bound={} pool={:x?}\n", + generation, ms, recovered, outer, inner, completed.stamps, completed.timestamps, + masks, scheduler, outer_after, inner_after, expected_stamp, consumed, finished, pool_bound, pool_words); + } + Ok(consumed && ((finished && pool_bound) || recovered)) + })?; + let recovery = match end { + JobEnd::Retired => match self.poll_faults("application compute retired")? { + None => { + let observed = self.queue_probe.as_ref().ok_or(EIO)?.pool_status(); + if !G18QueueProbe::pool_bound_for_context(observed, pool, command.context_id) { + dev_err!(self.device(), + "G18_COMPUTE_POOL_VERIFY_FAILED compute={} binding={:?} observed={:x?}; both owners retained\n", + generation, pool, observed); + return Err(EIO.into()); + } + if pool.generation != previous_pool.generation { + crate::g18_kick_info!(self.device(), + "G18_COMPUTE_POOL_VERIFY compute={} generation={} state={:#x} context={} slot={} observed={:x?} verified=1\n", + generation, pool.generation, pool.state, command.context_id, pool.slot, observed); + } + if recovery::phases_due(generation) { + dev_info!(self.device(), + "G18_JOB_PHASES compute={} us={:?} gpu_ticks={} (outer inner submitted completed linked stamp mask pool)\n", + generation, first, gpu_ticks); + } + self.queue_probe.as_mut().ok_or(EIO)?.retire_application_compute(generation)?; + self.memory.as_mut().ok_or(EIO)? + .retire_application_compute(generation, command.context_id)?; + crate::g18_kick_info!(self.device(), + "G18_APP_FRAME_PASS compute={} stamp={:#x} scheduler={} outer={} inner={}\n", + generation, expected_stamp, generation, outer_after, inner_after); + return Ok(()); + } + // Retired, but a recovery followed at once: do not vouch for + // the dispatch. Its queue is retired either way. + Some(recovery) => recovery, + }, + JobEnd::Recovered(recovery) => recovery, + JobEnd::Stranded(recovery) => { + self.memory.as_mut().ok_or(EIO)?.quarantine_application_compute(); + dev_err!(self.device(), + "G18_APP_STRANDED compute={} epoch={} host={}: queue never retired; compute quarantined, owners retained\n", + generation, recovery.epoch, recovery.host); + return Err(Failure::Job(JobFailure::Stranded(recovery, JobPath::Compute))); + } + }; + self.queue_probe.as_mut().ok_or(EIO)?.retire_application_compute(generation)?; + self.memory.as_mut().ok_or(EIO)?.retire_recovered_application_compute(generation)?; + dev_err!(self.device(), "G18_APP_RECOVERED compute={} epoch={} host={}: queue retired\n", + generation, recovery.epoch, recovery.host); + // The next job needs the queue's firmware bindings as they were; a + // recovery may reset them. Refuse the path rather than kick into it. + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + let unready = probe.compute_unready(pool, command.context_id, generation + 1)?; + let (counters, slot, scheduler, pool_words) = probe.compute_state()?; + dev_err!(self.device(), + "G18_APP_COMPUTE_POSTRECOVERY compute={} unready={:?} counters={:?} slot={:x?} scheduler={:x?} pool={:x?}\n", + generation, unready, counters, slot, scheduler, pool_words); + match unready { + None => Err(Failure::Job(JobFailure::Recovered(recovery))), + Some(what) => Err(Failure::Job(JobFailure::Unbound(recovery, JobPath::Compute, what))), + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/boot/mod.rs b/drivers/gpu/drm/asahi/g18/boot/mod.rs new file mode 100644 index 0000000000000..232b43acc6eb7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/boot/mod.rs @@ -0,0 +1,2130 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! J873 / T8152 / G18G (26A428, RTKit 3514.0.15) staged live boot. +//! +//! Plan milestone M1 step 1.8 plus the M0 probes. Derived from +//! `m5_live_boot.rs`; the M5 (T8142) path is not touched by anything here. +//! +//! The stage comes from `asahi.g18_initdata_test`. The driver's T8152 probe +//! gate ([`crate::driver`]) has already sampled the three GPU PSD services, +//! mapped SGX and, only if the sgx PSD proved the block powered, read the ID +//! registers. It hands everything over in a [`T8152Probe`]. +//! +//! | stage | effect | approval | +//! |-------|--------|----------| +//! | 0 | (driver) refuse with ENODEV before any MMIO | none | +//! | 1 | read-only: PSD state, SGX ID, both wrappers' CPU control/status, GFX/GFX1 post-office status words. Sends nothing. ENODEV | none | +//! | 2 | GPU must ALREADY be powered (iBoot or stage 90). Start GFX1 then GFX ASC CPU, RTKit HELLO/EPMAP with syslog and crashlog attached, 2 s syslog window, stop. ENODEV | REQUIRES USER APPROVAL | +//! | 3 | stage 2 + init-data graph, publish GFX then GFX1 root on EP 0x20 (0x81 doorbell), wait for each 0x0009 ACK, stop. ENODEV | REQUIRES USER APPROVAL | +//! | 4 | stage 3 + 0x89 power-transition-end to both roles, EP 0x21 idle-policy wake, passive 60 s event/trace drain, stop. Returns the retained owner | REQUIRES USER APPROVAL | +//! | 5 | stage 4 plus one owned empty queue registration before the passive observation. No GPU job. Returns the retained owner | REQUIRES USER APPROVAL | +//! | 6 | stage 5 plus a second tag15 binding the owned scheduler/control object. No GPU job | REQUIRES USER APPROVAL | +//! | 7 | stage 6 plus the owned initial UMA pool binding. No GPU job or grow request | REQUIRES USER APPROVAL | +//! | 90 | PMGR2 PSD request 0 for sgx, service 3, ONLY. Poll ACK before reading STATE. No automatic restore. ENODEV | REQUIRES USER APPROVAL | +//! | 91 | request 2 for gfx-asc, service 81 (sgx must already read state 0) | REQUIRES USER APPROVAL | +//! | 92 | request 2 for gfx1-asc, service 82 (sgx must already read state 0) | REQUIRES USER APPROVAL | +//! | 93 | restore: idle request (6) on every GPU service not already idle, in reverse native order 81, 82, 3. ENODEV | REQUIRES USER APPROVAL | +//! +//! Stages 2..7 start the ASC CPUs, which the mailbox provider additionally +//! refuses unless `apple_mailbox.ascwrap_v8_cpu_control=1` (also REQUIRES +//! USER APPROVAL). Stages 90..93 are the only code in the driver that writes +//! a PSD request register; each writes one service (93: only the services +//! it restores), and none is ever implied by another stage. Native order is +//! 90, 92, 91. J873 hardware confirms STATE reads consume ACK: transition +//! waits poll ACK before reading STATE once. Later access gates require +//! each service's active state without requiring an already-consumed ACK. +//! +//! The GFX/GFX1 mailbox providers exist only when the loader enabled their +//! nodes, which it does together with the firmware receipt. A missing +//! provider is reported and turned into ENODEV, never EPROBE_DEFER, so the +//! driver core cannot re-run a stage in a deferral loop. The providers +//! receive by polling unless `apple_mailbox.ascwrap_v8_rx_irq=1` (REQUIRES +//! USER APPROVAL), so no GPU mailbox interrupt is enabled by default. +//! +//! Safety rules this module follows: +//! - Every SGX or ASC wrapper MMIO read follows a PSD read of that block that +//! proves it powered. Wrapper reads go through the provider's C helpers, +//! which read the role's PSD block first; SGX reads use +//! [`regs::t8152_prove_sgx_powered`]. The post-office status word is read +//! only after this module read the role's PSD block as powered. +//! - Only the provider's own 0x80-byte post-office window is touched (the +//! status word at +0x20); never the rest of the shared bank. +//! - Once an ASC CPU was asked to start, no firmware-visible owner (UAT, +//! init-data graph, RTKit transport) is ever freed: the session is stopped +//! and then leaked, and the module is pinned, until reboot. There is one +//! firmware init per boot; a second attempt is refused. +//! - On any firmware fault the crashlog is dumped: the RTKit crash callback +//! dumps a host buffer when there is one, and otherwise (and after any +//! failed stage) scans the role's `__DATA` reservation for the `CLHE` +//! crashlog header and dumps what it finds. + +use crate::{ + driver::AsahiDevice, + g18::probe::{G18Runtime, T8152Probe}, + g18::initdata::G18FirmwareMemory, + g18::runtime::compute::G18PreparedComputeCommand, + g18::buffer::pool::G18UmaPoolBinding, + g18::queue::probe::{G18QueueProbe, QUEUE_ID}, + g18::fw::rtkit as g17_rtkit, mmu, g18::regs as regs, +}; +use core::{ + ffi::c_void, + mem::MaybeUninit, + ptr::NonNull, + sync::atomic::{AtomicBool, AtomicU32, Ordering}, +}; +use kernel::{ + bindings, c_str, + device::Core, + devres::Devres, + error::{from_err_ptr, to_result}, + io::{mem::IoMem, Io, Mmio, MmioRaw}, + iosys_map::IoSysMapRef, + macros::vtable, + new_mutex, platform, + prelude::*, + soc::apple::rtkit, + sync::{aref::ARef, Arc, Mutex}, + time::{delay::fsleep, Delta, Instant, Monotonic}, +}; + +// --------------------------------------------------------------------------- +// Stages +// --------------------------------------------------------------------------- + +/// Read-only probe. +pub(crate) const STAGE_READ_ONLY: u32 = 1; +/// REQUIRES USER APPROVAL: ASC CPU start + RTKit HELLO/EPMAP. +pub(crate) const STAGE_RTKIT: u32 = 2; +/// REQUIRES USER APPROVAL: stage 2 + init-data roots and 0x0009 ACKs. +pub(crate) const STAGE_INITDATA: u32 = 3; +/// REQUIRES USER APPROVAL: stage 3 + openings, idle wake, 60 s heartbeat. +pub(crate) const STAGE_HEARTBEAT: u32 = 4; +/// Owned empty queue registration, then the same 60 s stability observation. +pub(crate) const STAGE_QUEUE_REGISTER: u32 = 5; +/// Register an owned queue and bind its scheduler/control, without a GPU job. +pub(crate) const STAGE_SCHEDULER_BIND: u32 = 6; +/// Bind the owned empty compute UMA pool, without submitting a GPU job. +pub(crate) const STAGE_POOL_BIND: u32 = 7; +/// One captured addition with retained resources and independent completion/output gates. +pub(crate) const STAGE_COMPUTE_ADD: u32 = 8; +pub(crate) const STAGE_RENDER: u32 = crate::g18::boot::render_stage::STAGE_RENDER; +pub(crate) const STAGE_DRM: u32 = 10; +pub(crate) mod render_stage; + +#[path = "session.rs"] +mod application; +pub(crate) use application::start_drm; +/// REQUIRES USER APPROVAL: PMGR2 PSD power-on request for sgx (service 3) +/// only. Standalone; no other stage requests PSD power. +pub(crate) const STAGE_PSD_POWER_ON: u32 = 90; +/// REQUIRES USER APPROVAL: as [`STAGE_PSD_POWER_ON`] for gfx-asc (81). +pub(crate) const STAGE_PSD_POWER_ON_GFX: u32 = 91; +/// REQUIRES USER APPROVAL: as [`STAGE_PSD_POWER_ON`] for gfx1-asc (82). +pub(crate) const STAGE_PSD_POWER_ON_GFX1: u32 = 92; +/// REQUIRES USER APPROVAL: idle request on every non-idle GPU service. +pub(crate) const STAGE_PSD_RESTORE_IDLE: u32 = 93; + +/// Stage values this module implements. The driver refuses anything else +/// with EINVAL before any MMIO. +pub(crate) const fn stage_is_known(stage: u32) -> bool { + matches!( + stage, + STAGE_READ_ONLY + | STAGE_RTKIT + | STAGE_INITDATA + | STAGE_HEARTBEAT + | STAGE_QUEUE_REGISTER + | STAGE_SCHEDULER_BIND + | STAGE_POOL_BIND + | STAGE_COMPUTE_ADD + | STAGE_RENDER + | STAGE_DRM + | STAGE_PSD_POWER_ON + | STAGE_PSD_POWER_ON_GFX + | STAGE_PSD_POWER_ON_GFX1 + | STAGE_PSD_RESTORE_IDLE + ) +} + +/// The single PSD service a power-on stage requests, if `stage` is one. +const fn psd_power_on_service(stage: u32) -> Option { + match stage { + STAGE_PSD_POWER_ON => Some(regs::T8152PsdService::Sgx), + STAGE_PSD_POWER_ON_GFX => Some(regs::T8152PsdService::GfxAsc), + STAGE_PSD_POWER_ON_GFX1 => Some(regs::T8152PsdService::Gfx1Asc), + _ => None, + } +} + +/// Stages that write a PSD request register. +const fn stage_writes_psd(stage: u32) -> bool { + psd_power_on_service(stage).is_some() || stage == STAGE_PSD_RESTORE_IDLE +} + +/// Stages that run the loaded firmware and therefore need the loader's +/// G18G firmware receipt and the 42-bit DMA mask. +pub(crate) const fn stage_runs_firmware(stage: u32) -> bool { + matches!(stage, STAGE_RTKIT | STAGE_INITDATA | STAGE_HEARTBEAT + | STAGE_QUEUE_REGISTER | STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD | STAGE_RENDER | STAGE_DRM) +} + +/// Set once any firmware stage reached its first ASC CPU start request. +/// The firmware initializes once per boot; the module is pinned from then on, +/// so this survives unbind/rebind and refuses a second attempt. +static G18_FIRMWARE_ATTEMPTED: AtomicBool = AtomicBool::new(false); +/// Bit per PSD request kind written by this module instance: bit 0 sgx, +/// bit 1 gfx-asc, bit 2 gfx1-asc (power-on), bit 3 restore. Each kind is +/// written at most once per module load; a re-bind does not repeat it. +static G18_PSD_REQUESTS: AtomicU32 = AtomicU32::new(0); +const PSD_REQUEST_RESTORE_BIT: u32 = 1 << 3; + +const fn psd_request_bit(service: regs::T8152PsdService) -> u32 { + match service { + regs::T8152PsdService::Sgx => 1 << 0, + regs::T8152PsdService::GfxAsc => 1 << 1, + regs::T8152PsdService::Gfx1Asc => 1 << 2, + } +} + +// --------------------------------------------------------------------------- +// Fixed 26A428 protocol facts (plan step 1.7, FW_PROVEN unless noted) +// --------------------------------------------------------------------------- + +const EP_GFX_MESSAGES: u8 = g17_rtkit::ENDPOINT_GFX_MESSAGES; // 0x20 +const EP_GFX_INTERRUPTS: u8 = g17_rtkit::ENDPOINT_GFX_INTERRUPTS; // 0x21 +/// Init-data ACK on EP 0x20 (b000 0x24c78), identical to M5. +const INITDATA_ACK: u64 = 0x0009_0000_0000_0000; +/// EP 0x21 device-control doorbell for the idle-policy wake (M5 value, +/// unchanged in the 26A428 EP 0x21 handler 0x254cc-0x25610). +const IDLE_POLICY_WAKE: u64 = 0x0084_0000_0000_0011; + +const ACK_POLLS: u32 = 200; // x 10 ms +const CONTROL_POLLS: u32 = 1000; // x 1 ms +const SYSLOG_WINDOW_MS: u32 = 2000; +const HEARTBEAT_MS: u32 = 60_000; +const HEARTBEAT_STEP_MS: u32 = 100; +const MESSAGE_LOG_BUDGET: u32 = 32; + +/// PSD service block layout (the GPU power tool's contract). +const PSD_STATE: usize = 0x80; +const PSD_REQUEST: usize = 0x90; +const PSD_ACK: usize = 0xbc; +const PSD_BLOCK_SIZE: u64 = 0x100; +/// Cold-boot idle state of all three GPU services on J873. +const PSD_STATE_IDLE: u32 = 6; +const PSD_POLLS: u32 = 1000; // x 1 ms, as gpu_power.py (1 s) + +/// ASCWrap v8 CPU control RUN bit (m1n1 asc.c; INFERRED for the GFX +/// wrappers, see the mailbox provider). +const ASCWRAP_V8_CPU_RUN: u32 = 1 << 4; +/// Counted-FIFO status word inside the role's 0x80-byte post-office window. +const POSTOFFICE_STATUS: usize = 0x20; +const POSTOFFICE_STATUS_WINDOW: usize = POSTOFFICE_STATUS + core::mem::size_of::(); + +/// RTKit crashlog container (`CLHE` header, sections, `CLHE` trailer). +const fn fourcc(a: u8, b: u8, c: u8, d: u8) -> u32 { + ((a as u32) << 24) | ((b as u32) << 16) | ((c as u32) << 8) | d as u32 +} +const CRASHLOG_HEADER: u32 = fourcc(b'C', b'L', b'H', b'E'); +const CRASHLOG_STRING: u32 = fourcc(b'C', b's', b't', b'r'); +const CRASHLOG_REGISTERS: u32 = fourcc(b'C', b'r', b'g', b'8'); +/// Largest crashlog this module copies and hex-dumps (the ADT +/// coredump-copy-buffer is 0x8000). +const CRASHLOG_MAX_BYTES: usize = 0x8000; +/// Granule of the `__DATA` crashlog scan. +const CRASHLOG_SCAN_STEP: usize = 0x1000; + +// --------------------------------------------------------------------------- +// Roles +// --------------------------------------------------------------------------- + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum G18Role { + /// Mailbox index 0, ASC `gfx-asc`, post-office mailbox 1. + Gfx, + /// Mailbox index 1, ASC `gfx1-asc`, post-office mailbox 2. + Gfx1, +} + +impl G18Role { + const ALL: [Self; 2] = [Self::Gfx, Self::Gfx1]; + /// M5 order, carried to G18G (INFERRED; the 26A428 bootFirmware publish + /// order is not re-derived): GFX1 starts and publishes first. + const START_ORDER: [Self; 2] = [Self::Gfx1, Self::Gfx]; + /// Reverse of the start order. + const STOP_ORDER: [Self; 2] = [Self::Gfx, Self::Gfx1]; + + const fn index(self) -> usize { + match self { + Self::Gfx => 0, + Self::Gfx1 => 1, + } + } + + const fn name(self) -> &'static str { + match self { + Self::Gfx => "GFX", + Self::Gfx1 => "GFX1", + } + } + + /// `enum apple_mbox_ascwrap_v6_role` value the provider reports. + const fn lifecycle_role(self) -> u32 { + match self { + Self::Gfx => 1, + Self::Gfx1 => 2, + } + } + + /// ADT gfx-asc/gfx1-asc reg[0] (iop,ascwrap-v8). + const fn wrapper(self) -> (u64, u64) { + match self { + Self::Gfx => (0x5_8260_0000, 0x7_4000), + Self::Gfx1 => (0x5_82e0_0000, 0x7_4000), + } + } + + /// ADT gfx-asc/gfx1-asc reg[1]. + const fn iop_vbar(self) -> (u64, u64) { + match self { + Self::Gfx => (0x5_8205_0000, 0x8), + Self::Gfx1 => (0x5_8285_0000, 0x8), + } + } + + /// The role's 0x80-byte post-office window (informational). + const fn postoffice_window(self) -> u64 { + match self { + Self::Gfx => 0x3_1602_0100, + Self::Gfx1 => 0x3_1602_0200, + } + } + + const fn psd(self) -> regs::T8152PsdService { + match self { + Self::Gfx => regs::T8152PsdService::GfxAsc, + Self::Gfx1 => regs::T8152PsdService::Gfx1Asc, + } + } + + const fn data_region_name(self) -> &'static CStr { + match self { + Self::Gfx => c_str!("gfx-data"), + Self::Gfx1 => c_str!("gfx1-data"), + } + } + + /// The `__DATA_SHARED_RO` reservation right after `__DATA`. On J873 the + /// RTKit crash callback reports the firmware-owned crash buffer here + /// (e.g. GFX1: 0x10003900000, 0x8000 bytes). + const fn shared_ro_region_name(self) -> &'static CStr { + match self { + Self::Gfx => c_str!("gfx-data-shared-ro"), + Self::Gfx1 => c_str!("gfx1-data-shared-ro"), + } + } + + /// 26A428 `__DATA` sizes (b000 / b010), as `Uat::new_t8152` checks. + const fn data_size(self) -> u64 { + match self { + Self::Gfx => 0xb_c000, + Self::Gfx1 => 0xb_4000, + } + } +} + +// Compile-time guards on the stage gates and the fixed J873 facts. +const _: () = { + assert!(!stage_is_known(0) && !stage_is_known(11) && !stage_is_known(89) && !stage_is_known(94)); + assert!(stage_is_known(STAGE_RENDER) && stage_runs_firmware(STAGE_RENDER)); + assert!(!stage_writes_psd(STAGE_RENDER)); + assert!(stage_is_known(STAGE_READ_ONLY) && stage_is_known(STAGE_PSD_POWER_ON)); + assert!(stage_is_known(STAGE_PSD_POWER_ON_GFX) && stage_is_known(STAGE_PSD_POWER_ON_GFX1)); + assert!(stage_is_known(STAGE_PSD_RESTORE_IDLE)); + // Neither the read-only probe nor a power stage runs firmware, and no + // firmware stage writes PSD. + assert!(!stage_runs_firmware(STAGE_READ_ONLY) && !stage_writes_psd(STAGE_READ_ONLY)); + assert!(!stage_runs_firmware(STAGE_PSD_POWER_ON) && !stage_runs_firmware(STAGE_PSD_POWER_ON_GFX)); + assert!(!stage_runs_firmware(STAGE_PSD_POWER_ON_GFX1) && !stage_runs_firmware(STAGE_PSD_RESTORE_IDLE)); + assert!(stage_runs_firmware(STAGE_RTKIT) && stage_runs_firmware(STAGE_INITDATA)); + assert!(stage_runs_firmware(STAGE_HEARTBEAT)); + assert!(stage_is_known(STAGE_POOL_BIND) && stage_runs_firmware(STAGE_POOL_BIND)); + assert!(stage_is_known(STAGE_COMPUTE_ADD) && stage_runs_firmware(STAGE_COMPUTE_ADD)); + assert!(!stage_writes_psd(STAGE_RTKIT) && !stage_writes_psd(STAGE_INITDATA)); + assert!(!stage_writes_psd(STAGE_HEARTBEAT)); + assert!(!stage_writes_psd(STAGE_POOL_BIND)); + assert!(!stage_writes_psd(STAGE_COMPUTE_ADD)); + // Exactly one service per power-on stage, sgx first. + assert!(matches!(psd_power_on_service(STAGE_PSD_POWER_ON), Some(regs::T8152PsdService::Sgx))); + assert!(matches!(psd_power_on_service(STAGE_PSD_POWER_ON_GFX), Some(regs::T8152PsdService::GfxAsc))); + assert!(matches!(psd_power_on_service(STAGE_PSD_POWER_ON_GFX1), Some(regs::T8152PsdService::Gfx1Asc))); + assert!(psd_power_on_service(STAGE_PSD_RESTORE_IDLE).is_none() && stage_writes_psd(STAGE_PSD_RESTORE_IDLE)); + assert!(psd_request_bit(regs::T8152PsdService::Sgx) | psd_request_bit(regs::T8152PsdService::GfxAsc) + | psd_request_bit(regs::T8152PsdService::Gfx1Asc) | PSD_REQUEST_RESTORE_BIT == 0xf); + // GFX1 starts and publishes first; stop is the reverse. + assert!(G18Role::START_ORDER[0].index() == 1 && G18Role::START_ORDER[1].index() == 0); + assert!(G18Role::STOP_ORDER[0].index() == 0 && G18Role::STOP_ORDER[1].index() == 1); + // PSD services 81/82 and the post-office windows of mailboxes 1/2 + // ((2 * mb + !po) * 0x80 into 0x316020000). + assert!(G18Role::Gfx.psd().physical_base() == 0x3_03a1_5100); + assert!(G18Role::Gfx1.psd().physical_base() == 0x3_03a1_5200); + assert!(G18Role::Gfx.postoffice_window() == 0x3_1602_0000 + 2 * 0x80); + assert!(G18Role::Gfx1.postoffice_window() == 0x3_1602_0000 + 4 * 0x80); + assert!(POSTOFFICE_STATUS_WINDOW <= 0x80); + // __DATA sizes are 16 KiB multiples and hold at least one crash view. + assert!(G18Role::Gfx.data_size() % 0x4000 == 0 && G18Role::Gfx1.data_size() % 0x4000 == 0); + assert!(CRASHLOG_MAX_BYTES as u64 <= G18Role::Gfx1.data_size()); + assert!(CRASHLOG_HEADER == 0x434c_4845); +}; + +// --------------------------------------------------------------------------- +// PSD service blocks +// --------------------------------------------------------------------------- + +const fn psd_resource_name(service: regs::T8152PsdService) -> &'static CStr { + match service { + regs::T8152PsdService::Sgx => c_str!("psd-sgx"), + regs::T8152PsdService::GfxAsc => c_str!("psd-gfx-asc"), + regs::T8152PsdService::Gfx1Asc => c_str!("psd-gfx1-asc"), + } +} + +/// One GPU PSD service block, mapped (not claimed) from the GPU node for the +/// duration of a stage. Only [`PsdBlock::request`] writes, and only stages +/// 90..93 call it. +struct PsdBlock { + service: regs::T8152PsdService, + io: Pin>>>, +} + +impl PsdBlock { + fn map(pdev: &platform::Device, service: regs::T8152PsdService) -> Result { + let name = psd_resource_name(service); + let resource = pdev.resource_by_name(name).ok_or(ENODEV)?; + if resource.start() != service.physical_base() || resource.size() != PSD_BLOCK_SIZE { + dev_err!( + pdev.as_ref(), + "G18_PSD service {} resource {:#x}+{:#x} is not {:#x}+{:#x}\n", + service.service_id(), + resource.start(), + resource.size(), + service.physical_base(), + PSD_BLOCK_SIZE + ); + return Err(EINVAL); + } + let request = pdev.io_request_by_name(name).ok_or(ENODEV)?; + let io = KBox::pin_init(request.iomap(), GFP_KERNEL)?; + Ok(Self { service, io }) + } + + fn sample(&self) -> Result { + let io = self.io.try_access().ok_or(ENODEV)?; + // Reading STATE consumes ACK; capture ACK first for telemetry. + let ack = io.try_read32(PSD_ACK)?; + let state = io.try_read32(PSD_STATE)?; + Ok(regs::T8152PsdSample { + service: self.service, + state, + ack, + }) + } + + /// Poll ACK without touching STATE until the transition completes. + fn acknowledged_sample(&self) -> Result> { + let io = self.io.try_access().ok_or(ENODEV)?; + let ack = io.try_read32(PSD_ACK)?; + if ack & 1 == 0 { + return Ok(None); + } + let state = io.try_read32(PSD_STATE)?; + Ok(Some(regs::T8152PsdSample { service: self.service, state, ack })) + } + + /// REQUIRES USER APPROVAL. Accept only this service's native active + /// state (SGX 0, ASCs 2) or idle (6). Called by stages 90..93 only. + fn request(&self, value: u32) -> Result { + if value != self.service.active_state() && value != PSD_STATE_IDLE { + return Err(EINVAL); + } + let io = self.io.try_access().ok_or(ENODEV)?; + io.try_write32(value, PSD_REQUEST) + } +} + +fn psd_line(sample: ®s::T8152PsdSample) -> (u32, u32, u32, u32, &'static str) { + ( + sample.service.service_id(), + sample.state, + sample.power_state(), + sample.ack, + sample.verdict(), + ) +} + +fn log_psd(pdev: &platform::Device, label: &str, sample: ®s::T8152PsdSample) { + let (id, state, power, ack, verdict) = psd_line(sample); + dev_info!( + pdev.as_ref(), + "G18_PSD {} service={} state={:#x} power-state={} ack={:#x} -> {}\n", + label, + id, + state, + power, + ack, + verdict + ); +} + +/// Fresh read-only samples of all three GPU services. +fn read_all_psd(pdev: &platform::Device) -> Result<[regs::T8152PsdSample; 3]> { + let mut samples = [regs::T8152PsdSample { + service: regs::T8152PsdService::Sgx, + state: 0, + ack: 0, + }; 3]; + for (slot, service) in samples.iter_mut().zip(regs::T8152PsdService::ALL) { + *slot = regs::t8152_read_psd(pdev, service)?; + } + Ok(samples) +} + +// --------------------------------------------------------------------------- +// Mailbox providers (apple,t8152-gfx-ascwrap-v8) +// --------------------------------------------------------------------------- + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +struct G18Lifecycle { + role: u32, + wrapper: (u64, u64), + iop_vbar: (u64, u64), + missing: u32, +} + +/// Borrowed view of one ASCWrap v8 provider. The C provider owns every +/// mapping and every lifecycle transition, and reads the role's PSD block +/// before each wrapper access. This view never touches the wrapper itself; +/// the one direct read it makes is the post-office status word, gated here +/// on the role's PSD block. +struct G18Provider { + mbox: *mut bindings::apple_mbox, + role: G18Role, +} + +// SAFETY: the provider is kept alive by the device link apple_mbox_get() +// establishes; every mutation is serialized inside the C provider. +unsafe impl Send for G18Provider {} +// SAFETY: as above; the only Rust-side MMIO is one relaxed status read. +unsafe impl Sync for G18Provider {} + +impl G18Provider { + /// Look up the role's provider. A provider that is not bound makes + /// apple_mbox_get() return EPROBE_DEFER; that is reported and turned into + /// ENODEV here so that no stage can put the GPU device into a deferral + /// loop (every retry would repeat the PSD and SGX reads). The loader + /// enables both mailbox nodes only together with the firmware receipt. + fn get(dev: &kernel::device::Device, role: G18Role) -> Result { + // SAFETY: `dev` is live; apple_mbox_get() links the provider to it. + let mbox = unsafe { + from_err_ptr(bindings::apple_mbox_get(dev.as_raw(), role.index().try_into()?)) + }; + match mbox { + Ok(mbox) => Ok(Self { mbox, role }), + Err(error) => Err(no_probe_defer(dev, role.name(), "mailbox provider", error)), + } + } + + fn lifecycle(&self) -> Result { + let mut raw = MaybeUninit::::uninit(); + // SAFETY: `self.mbox` is a live provider and `raw` is writable. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_lifecycle(self.mbox, raw.as_mut_ptr()) + })?; + // SAFETY: the successful call initialized every field. + let raw = unsafe { raw.assume_init() }; + Ok(G18Lifecycle { + role: raw.role as u32, + wrapper: (raw.wrapper_start, raw.wrapper_size), + iop_vbar: (raw.iop_vbar_start, raw.iop_vbar_size), + missing: raw.missing, + }) + } + + /// Exact role and resources of the J873 v8 provider. The 0x74000 + /// wrapper size identifies the v8 provider, whose `regs` mapping is the + /// role's 0x80-byte post-office window. + fn validate(&self, dev: &kernel::device::Device) -> Result { + let lifecycle = self.lifecycle()?; + if lifecycle.role != self.role.lifecycle_role() + || lifecycle.wrapper != self.role.wrapper() + || lifecycle.iop_vbar != self.role.iop_vbar() + { + dev_err!( + dev, + "G18 {} mailbox provider mismatch: role={} wrapper={:x?} iop-vbar={:x?}; expected role={} wrapper={:x?} iop-vbar={:x?}\n", + self.role.name(), + lifecycle.role, + lifecycle.wrapper, + lifecycle.iop_vbar, + self.role.lifecycle_role(), + self.role.wrapper(), + self.role.iop_vbar() + ); + return Err(EINVAL); + } + Ok(lifecycle) + } + + fn require_safe_cpu_lifecycle(&self) -> Result { + // SAFETY: capability check on the live provider. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(self.mbox) }) + } + + /// REQUIRES USER APPROVAL (the provider refuses with EPERM unless + /// `apple_mailbox.ascwrap_v8_cpu_control=1`; it also refuses an unpowered + /// block). + fn start_cpu(&self) -> Result { + // SAFETY: the provider serializes the lifecycle and owns the MMIO. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_start_cpu(self.mbox) }) + } + + /// As [`Self::start_cpu`]; a no-op unless this provider started the CPU. + fn stop_cpu(&self) -> Result { + // SAFETY: as above. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }) + } + + /// CPU_CONTROL (+0x44) and CPU_STATUS (+0x48). The provider reads the + /// role's PSD first and returns EHOSTDOWN, without touching the wrapper, + /// when the block is not powered. + fn cpu_state(&self) -> Result<(u32, u32)> { + let mut control = 0; + let mut status = 0; + // SAFETY: live provider; both outputs are writable stack storage. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_cpu_state(self.mbox, &mut control, &mut status) + })?; + Ok((control, status)) + } + + /// Read the counted-FIFO status word (+0x20) of this role's post-office + /// window, after reading the role's PSD block. Returns `None` for the + /// word when the block is not powered. Call only after [`Self::validate`]. + fn postoffice_status( + &self, + pdev: &platform::Device, + ) -> Result<(regs::T8152PsdSample, Option)> { + let psd = regs::t8152_read_psd(pdev, self.role.psd())?; + if !psd.powered() { + return Ok((psd, None)); + } + // SAFETY: the provider (device-link protected) owns this mapping. + let (window, removing) = unsafe { + ( + (*self.mbox).regs as usize, + core::ptr::read_volatile(core::ptr::addr_of!((*self.mbox).removing)), + ) + }; + if window == 0 || removing { + return Err(ENODEV); + } + let raw = MmioRaw::::new(window, POSTOFFICE_STATUS_WINDOW)?; + // SAFETY: `raw` lies inside the provider's 0x80-byte window and + // covers exactly the single status read below. + let status = unsafe { Mmio::from_raw(&raw) }.relaxed().read32(POSTOFFICE_STATUS); + Ok((psd, Some(status))) + } +} + +/// Map EPROBE_DEFER to ENODEV with an explanation; pass anything else on. +fn no_probe_defer(dev: &kernel::device::Device, role: &str, what: &str, error: Error) -> Error { + if error != EPROBE_DEFER { + return error; + } + dev_err!( + dev, + "G18 {} {} not bound: the GFX/GFX1 mailbox nodes are disabled (the loader enables them only together with the firmware receipt) or the provider failed to probe; ENODEV, not deferring\n", + role, + what + ); + ENODEV +} + +fn log_cpu_state(dev: &kernel::device::Device, role: G18Role, label: &str, provider: &G18Provider) { + match provider.cpu_state() { + Ok((control, status)) => dev_info!( + dev, + "G18_ASC {} {} CPU_CONTROL={:#x} (RUN={}) CPU_STATUS={:#x}\n", + role.name(), + label, + control, + (control & ASCWRAP_V8_CPU_RUN != 0) as u32, + status + ), + Err(error) if error == regs::t8152_not_powered() => dev_info!( + dev, + "G18_ASC {} {} wrapper not read: role PSD not powered\n", + role.name(), + label + ), + Err(error) => dev_err!( + dev, + "G18_ASC {} {} CPU state read failed ({:?})\n", + role.name(), + label, + error + ), + } +} + +// --------------------------------------------------------------------------- +// Crashlog dump +// --------------------------------------------------------------------------- + +fn le_u32(raw: &[u8], offset: usize) -> Option { + Some(u32::from_le_bytes(raw.get(offset..offset.checked_add(4)?)?.try_into().ok()?)) +} + +fn le_u64(raw: &[u8], offset: usize) -> Option { + Some(u64::from_le_bytes(raw.get(offset..offset.checked_add(8)?)?.try_into().ok()?)) +} + +fn fourcc_text(kind: u32) -> [u8; 4] { + let mut text = kind.to_be_bytes(); + for byte in text.iter_mut() { + if !byte.is_ascii_graphic() { + *byte = b'.'; + } + } + text +} + +/// Log a crashlog: parsed sections (strings, the exception register frame +/// with the G17P `Crg8` layout, INFERRED for G18G) and a hex dump of every +/// non-zero 32-byte line, bounded to [`CRASHLOG_MAX_BYTES`]. +fn dump_crashlog(dev: &kernel::device::Device, role: &str, source: &str, raw: &[u8]) { + let header = le_u32(raw, 0).unwrap_or(0); + let declared = le_u32(raw, 8).unwrap_or(0) as usize; + let valid = header == CRASHLOG_HEADER && declared >= 0x20 && declared <= raw.len(); + let total = if valid { declared } else { raw.len() }.min(CRASHLOG_MAX_BYTES); + dev_err!( + dev, + "G18_CRASHLOG {} source={} bytes={} header={} declared={:#x} dumped={:#x}\n", + role, + source, + raw.len(), + core::str::from_utf8(&fourcc_text(header)).unwrap_or("????"), + declared, + total + ); + + if valid { + let mut offset = 0x20usize; + for _ in 0..64 { + let (Some(kind), Some(size)) = (le_u32(raw, offset), le_u32(raw, offset + 12)) else { + break; + }; + let size = size as usize; + if kind == CRASHLOG_HEADER { + break; + } + let Some(end) = offset.checked_add(size) else { + break; + }; + if size < 16 || end > declared { + dev_err!(dev, "G18_CRASHLOG {} section at {:#x} has bad size {:#x}\n", role, offset, size); + break; + } + dev_err!( + dev, + "G18_CRASHLOG {} section {} at {:#x} size {:#x}\n", + role, + core::str::from_utf8(&fourcc_text(kind)).unwrap_or("????"), + offset, + size + ); + let body = offset + 16; + if kind == CRASHLOG_STRING && size >= 20 { + let id = le_u32(raw, body).unwrap_or(0); + let bytes = &raw[body + 4..end]; + let length = bytes.iter().position(|b| *b == 0).unwrap_or(bytes.len()).min(256); + match core::str::from_utf8(&bytes[..length]) { + Ok(message) => dev_err!(dev, "G18_CRASHLOG {} message {}: {}\n", role, id, message), + Err(_) => dev_err!(dev, "G18_CRASHLOG {} message {}: (not UTF-8)\n", role, id), + } + } else if kind == CRASHLOG_REGISTERS && size >= 16 + 0x350 { + // Crg8 begins with an eight-byte frame descriptor, then X0. + // Verified against the 0x401 frame in the stage6 CLHE dump. + let reg = |index: usize| le_u64(raw, body + 8 + index * 8).unwrap_or(0); + for first in (0..31).step_by(4) { + let last = (first + 4).min(31); + let mut words = [0u64; 4]; + for (slot, index) in words.iter_mut().zip(first..last) { + *slot = reg(index); + } + dev_err!(dev, "G18_CRASHLOG {} x{}..x{}: {:x?}\n", role, first, last - 1, + &words[..last - first]); + } + dev_err!( + dev, + "G18_CRASHLOG {} exception pc={:#x} far={:#x} esr={:#x} sp={:#x} psr={:#x}\n", + role, + le_u64(raw, body + 0x108).unwrap_or(0), + le_u64(raw, body + 0x330).unwrap_or(0), + le_u32(raw, body + 0x340).unwrap_or(0), + le_u64(raw, body + 0x100).unwrap_or(0), + le_u64(raw, body + 0x110).unwrap_or(0) + ); + } + offset = end; + } + } + + let mut zero_lines = 0u32; + let mut offset = 0usize; + while offset + 32 <= total { + let mut words = [0u32; 8]; + for (index, word) in words.iter_mut().enumerate() { + *word = le_u32(raw, offset + index * 4).unwrap_or(0); + } + if words.iter().all(|word| *word == 0) { + zero_lines += 1; + } else { + dev_err!( + dev, + "G18_CRASHLOG {} +{:05x}: {:08x} {:08x} {:08x} {:08x} {:08x} {:08x} {:08x} {:08x}\n", + role, offset, words[0], words[1], words[2], words[3], words[4], words[5], + words[6], words[7] + ); + } + offset += 32; + } + dev_err!(dev, "G18_CRASHLOG {} end; {} all-zero lines omitted\n", role, zero_lines); +} + +/// Invalidate the AP's view of a range before reading memory the GPU +/// firmware wrote (clean+invalidate: nothing here is ever written by the AP). +fn invalidate_dcache_range(pointer: *const u8, length: usize) { + let ctr: u64; + // SAFETY: CTR_EL0 is readable at EL1. + unsafe { + core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)); + } + let line = 4usize << ((ctr >> 16) & 0xf); + let mut address = pointer as usize & !(line - 1); + let end = pointer as usize + length; + while address < end { + // SAFETY: `address` lies in a live cacheable mapping owned by the + // caller; clean+invalidate by VA has no architectural side effect on + // the data. + unsafe { + core::arch::asm!("dc civac, {address}", address = in(reg) address, + options(nostack, preserves_flags)); + } + address += line; + } + // SAFETY: barrier only. + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; +} + +/// Read-only write-back mapping of a firmware carveout range (not Linux +/// RAM), used only to find and copy a crashlog. +struct PhysMap { + pointer: NonNull, + size: usize, +} + +impl PhysMap { + fn new(physical: u64, size: usize) -> Result { + if size == 0 || physical & (crate::pgtable::UAT_PGMSK as u64) != 0 { + return Err(EINVAL); + } + // SAFETY: page-aligned firmware reservation validated by the caller + // against the GPU node; released by Drop and only read. + let pointer = unsafe { bindings::memremap(physical, size, bindings::MEMREMAP_WB as _) }; + Ok(Self { pointer: NonNull::new(pointer).ok_or(ENOMEM)?, size }) + } + + fn copy(&self, offset: usize, out: &mut [u8]) -> Result { + let end = offset.checked_add(out.len()).ok_or(EOVERFLOW)?; + if (offset | out.len()) & 3 != 0 || end > self.size { + return Err(EINVAL); + } + // SAFETY: bounds checked above against the mapping size. + let base = unsafe { (self.pointer.as_ptr() as *const u8).add(offset) }; + invalidate_dcache_range(base, out.len()); + for (index, chunk) in out.chunks_exact_mut(4).enumerate() { + // SAFETY: 4-byte aligned, inside the mapping. + let word = unsafe { core::ptr::read_volatile(base.add(index * 4) as *const u32) }; + chunk.copy_from_slice(&word.to_le_bytes()); + } + Ok(()) + } +} + +impl Drop for PhysMap { + fn drop(&mut self) { + // SAFETY: returned by memremap and not yet unmapped. + unsafe { bindings::memunmap(self.pointer.as_ptr()) }; + } +} + +// --------------------------------------------------------------------------- +// RTKit +// --------------------------------------------------------------------------- + +/// Per-role callback state. Callbacks only publish atomics and log. +struct G18RoleState { + dev: ARef, + role: G18Role, + /// The role's `__DATA` reservation (physical base, size). + data: (u64, u64), + /// The role's `__DATA_SHARED_RO` reservation, (0, 0) when absent. + shared_ro: (u64, u64), + acked: AtomicBool, + crashed: AtomicBool, + crash_dumped: AtomicBool, + messages: AtomicU32, +} + +impl G18RoleState { + fn crashed(&self) -> bool { + self.crashed.load(Ordering::Acquire) + } + + /// Scan the role's `__DATA` for a `CLHE` crashlog and dump it. Firmware + /// with a firmware-owned crash buffer (the G17P RTKit mode) hands the + /// host no bytes; on M5 that buffer is `__DATA + 0xa0000`. The G18G + /// offset is unknown, so scan every 4 KiB. Runs at most once per role. + fn dump_data_crashlog(&self, reason: &str) { + if self.crash_dumped.swap(true, Ordering::AcqRel) { + return; + } + let name = self.role.name(); + let mut found = self.scan_crashlog("__DATA", self.data, reason); + if self.shared_ro.1 != 0 { + found += self.scan_crashlog("__DATA_SHARED_RO", self.shared_ro, reason); + } + if found == 0 { + dev_err!(self.dev.as_ref(), "G18_CRASHLOG {} no CLHE header in __DATA or __DATA_SHARED_RO\n", name); + } + } + + /// Scan one reservation every CRASHLOG_SCAN_STEP for a `CLHE` crashlog + /// and dump what it finds. Read-only mapping. Returns the count found. + fn scan_crashlog(&self, label: &str, region: (u64, u64), reason: &str) -> u32 { + let dev: &kernel::device::Device = self.dev.as_ref(); + let name = self.role.name(); + let (base, size) = region; + let Ok(size) = usize::try_from(size) else { + return 0; + }; + dev_err!(dev, "G18_CRASHLOG {} scanning {} {:#x}+{:#x} ({})\n", name, label, base, size, reason); + let map = match PhysMap::new(base, size) { + Ok(map) => map, + Err(error) => { + dev_err!(dev, "G18_CRASHLOG {} {} map failed ({:?})\n", name, label, error); + return 0; + } + }; + let mut found = 0u32; + let mut offset = 0usize; + while offset + 16 <= size { + let mut head = [0u8; 16]; + if map.copy(offset, &mut head).is_ok() && le_u32(&head, 0) == Some(CRASHLOG_HEADER) { + let declared = le_u32(&head, 8).unwrap_or(0) as usize; + if declared < 0x20 || declared > size - offset { + dev_err!(dev, "G18_CRASHLOG {} CLHE at {}+{:#x} declares {:#x} bytes; skipped\n", + name, label, offset, declared); + } else { + let length = (declared.min(CRASHLOG_MAX_BYTES) + 3) & !3; + let length = length.min(size - offset); + match KVVec::from_elem(0u8, length, GFP_KERNEL) { + Ok(mut bytes) => { + if map.copy(offset, &mut bytes).is_ok() { + dev_err!(dev, "G18_CRASHLOG {} found at {}+{:#x} (PA {:#x})\n", + name, label, offset, base + offset as u64); + dump_crashlog(dev, name, label, &bytes); + found += 1; + } + } + Err(_) => dev_err!(dev, "G18_CRASHLOG {} no memory for {:#x} bytes\n", name, length), + } + } + } + offset += CRASHLOG_SCAN_STEP; + } + found + } +} + +/// No host-backed RTKit buffers: exactly the M5 transport configuration +/// (`G17PNoRtkitBuffer`). Host-allocated requests use the RTKit core's +/// coherent-DMA fallback; the firmware-owned crash buffer is echoed. +struct G18NoRtkitBuffer; + +impl rtkit::Buffer for G18NoRtkitBuffer { + fn iova(&self) -> Result { + Err(ENOTSUPP) + } + + fn buf(&mut self) -> Result> { + Err(ENOTSUPP) + } +} + +struct G18RtkitOps; + +#[vtable] +impl rtkit::Operations for G18RtkitOps { + type Data = Arc; + type Buffer = G18NoRtkitBuffer; + + fn recv_message_early( + data: ::Borrowed<'_>, + endpoint: u8, + message: u64, + ) -> bool { + if endpoint == EP_GFX_MESSAGES && message == INITDATA_ACK { + data.acked.store(true, Ordering::Release); + true + } else { + false + } + } + + fn recv_message( + data: ::Borrowed<'_>, + endpoint: u8, + message: u64, + ) { + let ordinal = data.messages.fetch_add(1, Ordering::Relaxed); + if ordinal < MESSAGE_LOG_BUDGET || ordinal.is_power_of_two() { + dev_info!( + data.dev.as_ref(), + "G18_MSG role={} ordinal={} ep={:#x} type={:#06x} payload={:#018x}\n", + data.role.name(), + ordinal, + endpoint, + message >> 48, + message + ); + } + } + + fn crashed( + data: ::Borrowed<'_>, + crashlog: Option<&[u8]>, + ) { + data.crashed.store(true, Ordering::Release); + dev_err!( + data.dev.as_ref(), + "G18 firmware crashed role={} host-crashlog={}\n", + data.role.name(), + crashlog.map_or(0, |bytes| bytes.len()) + ); + match crashlog { + Some(bytes) if !bytes.is_empty() => { + data.crash_dumped.store(true, Ordering::Release); + dump_crashlog(data.dev.as_ref(), data.role.name(), "rtkit", bytes); + } + // Firmware-owned buffer: the RTKit core logged its DVA. + _ => data.dump_data_crashlog("rtkit crash callback, firmware-owned buffer"), + } + } +} + +// --------------------------------------------------------------------------- +// Session +// --------------------------------------------------------------------------- + +struct G18RoleSlot { + role: G18Role, + provider: G18Provider, + rtkit: rtkit::RtKit, + started: bool, +} + +struct G18Session { + roles: [G18RoleSlot; 2], + states: [Arc; 2], + memory: Option>, + queue_probe: Option, + render_job: Option>, + application_job: Option>, + uat: Arc, + _registers: regs::Resources, + dev: ARef, + stage: u32, +} + +fn cntvct() -> u64 { + let value: u64; + // SAFETY: read-only architectural counter. + unsafe { + core::arch::asm!("mrs {value}, cntvct_el0", value = out(reg) value, + options(nomem, nostack, preserves_flags)); + } + value +} + +impl G18Session { + fn device(&self) -> &kernel::device::Device { + self.dev.as_ref() + } + + fn any_crashed(&self) -> bool { + self.states.iter().any(|state| state.crashed()) + } + + /// Drain the report rings once the graph exists. Returns whether a + /// recovery (type 4) event was seen. + fn drain(&mut self) -> Result { + match self.memory.as_mut() { + Some(memory) => memory.drain_reports(&self.dev), + None => Ok(false), + } + } + + /// Fault check used by every wait loop. + fn check_faults(&mut self, label: &str) -> Result { + // No host SGX MMIO here: firmware power-gates SGX at will, and a + // gated read is a synchronous external abort (stage34 oops at + // SGX+0xd8c0 during root publication). A GMMU fault reaches us as a + // firmware recovery event. Read retained DRAM for diagnosis; a + // userspace read of power-gated SGX registers is unsafe as well. + if self.drain()? { + dev_err!(self.device(), "G18 {}: firmware posted a recovery event\n", label); + return Err(EIO); + } + if self.any_crashed() { + dev_err!(self.device(), "G18 {}: firmware crashed\n", label); + return Err(EIO); + } + Ok(()) + } + + fn stop_cpus(&mut self) -> Result { + let mut failure = None; + for role in G18Role::STOP_ORDER { + let slot = &mut self.roles[role.index()]; + if slot.started { + match slot.provider.stop_cpu() { + Ok(()) => slot.started = false, + Err(error) => failure = Some(error), + } + } + log_cpu_state(self.dev.as_ref(), role, "after-stop", &self.roles[role.index()].provider); + } + failure.map_or(Ok(()), Err) + } + + /// Dump whatever crash evidence exists after a failed stage. + fn dump_fault_evidence(&mut self, reason: &str) { + for _ in 0..4 { + if self.drain().is_err() { + break; + } + } + if let Some(memory) = self.memory.as_mut() { + match memory.control_counters() { + Ok(counters) => dev_err!(self.dev.as_ref(), "G18 fault: control counters {:?}\n", counters), + Err(error) => dev_err!(self.dev.as_ref(), "G18 fault: counters unreadable ({:?})\n", error), + } + memory.log_summary(&self.dev); + dev_err!(self.dev.as_ref(), "G18 fault: report counts {:?}\n", memory.report_counts()); + } + for state in self.states.iter() { + state.dump_data_crashlog(reason); + } + } + + fn run(&mut self) -> Result { + // Defense in depth for any future caller that bypasses driver probe. + if self.stage == STAGE_RENDER { + crate::g18::boot::render_stage::admit(self.device(), + *crate::module_parameters::g18_render_variant.value(), + *crate::module_parameters::g18_render_profile.value())?; + } + self.boot_rtkit()?; + if self.stage == STAGE_RTKIT { + self.syslog_window()?; + return Ok(()); + } + self.build_initdata()?; + self.publish_roots()?; + if self.stage == STAGE_INITDATA { + return Ok(()); + } + self.retire_openings()?; + self.wake_idle_policy()?; + if matches!(self.stage, STAGE_QUEUE_REGISTER | STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD) { + self.register_queue()?; + } + if matches!(self.stage, STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD) { + self.bind_scheduler()?; + } + if self.stage == STAGE_POOL_BIND { + let binding = self.memory.as_mut().ok_or(EIO)?.prepare_uma_pool_binding()?; + self.bind_pool(binding)?; + } + if self.stage == STAGE_COMPUTE_ADD { + let (command, pool) = self.memory.as_mut().ok_or(EIO)?.prepare_addition(&self.dev)?; + self.bind_pool(pool)?; + self.submit_addition(command, pool)?; + } + if self.stage == STAGE_RENDER { self.submit_render()?; } + if self.stage == STAGE_DRM {return Ok(());} + self.heartbeat() + } + + /// Stage 2: start each ASC CPU (GFX1, then GFX) and run RTKit + /// HELLO/EPMAP. The RTKit core starts the crashlog endpoint before + /// acknowledging the EPMAP that advertises it (firmware-owned crash + /// buffer, as G17P/M5), then syslog and the other system endpoints. + fn boot_rtkit(&mut self) -> Result { + for role in G18Role::START_ORDER { + let dev = self.dev.clone(); + let slot = &mut self.roles[role.index()]; + log_cpu_state(dev.as_ref(), role, "before-start", &slot.provider); + // The provider re-reads the role's PSD block and refuses an + // unpowered block; it refuses without the CPU-control approval. + let (control, _) = slot.provider.cpu_state()?; + if control & ASCWRAP_V8_CPU_RUN == 0 { + slot.provider.start_cpu()?; + } else { + dev_info!(dev.as_ref(), "G18_ASC {} adopted: RUN already set, start skipped\n", role.name()); + } + slot.started = true; + log_cpu_state(dev.as_ref(), role, "after-start", &slot.provider); + let started = Instant::::now(); + let booted = Pin::new(&mut slot.rtkit).boot(); + let elapsed = started.elapsed().as_micros_ceil(); + let mut endpoints = 0u64; + for endpoint in 0..64u8 { + if Pin::new(&mut slot.rtkit).has_endpoint(endpoint) { + endpoints |= 1 << endpoint; + } + } + dev_info!( + dev.as_ref(), + "G18_RTKIT role={} boot={:?} after {} us; EPMAP[0..64]={:#018x} (crashlog={} syslog={} ep20={} ep21={} ep23={})\n", + role.name(), + booted, + elapsed, + endpoints, + (endpoints >> 1) & 1, + (endpoints >> 2) & 1, + (endpoints >> EP_GFX_MESSAGES) & 1, + (endpoints >> EP_GFX_INTERRUPTS) & 1, + (endpoints >> 0x23) & 1 + ); + booted?; + if self.states[role.index()].crashed() { + return Err(EIO); + } + } + Ok(()) + } + + /// Stage 2 only: give the syslog endpoint time to print its banner + /// (expected 'G18 Firmware initializing', 'FW Build version: b0'; the + /// RTKit core logs each entry as "RTKit: syslog message"). + fn syslog_window(&mut self) -> Result { + dev_info!(self.device(), "G18_STAGE2 syslog window open for {} ms\n", SYSLOG_WINDOW_MS); + for _ in 0..SYSLOG_WINDOW_MS / 10 { + if self.any_crashed() { + return Err(EIO); + } + fsleep(Delta::from_millis(10)); + } + for role in G18Role::ALL { + log_cpu_state(self.dev.as_ref(), role, "after-syslog-window", &self.roles[role.index()].provider); + dev_info!( + self.device(), + "G18_STAGE2 role={} non-system messages={}\n", + role.name(), + self.states[role.index()].messages.load(Ordering::Relaxed) + ); + } + dev_info!(self.device(), "G18_STAGE2 syslog window closed\n"); + Ok(()) + } + + /// Stage 3: map the init-data graph under the firmware-published roots + /// (same order as M5; no SGX pre-init pokes are issued, none are derived + /// for G18G yet). + fn build_initdata(&mut self) -> Result { + let roots = self.uat.prepare_t8140_firmware_high_roots()?; + let memory = G18FirmwareMemory::new_with_uat(&self.dev, self.uat.clone(), cntvct(), matches!(self.stage, STAGE_RENDER | STAGE_DRM))?; + self.memory = Some(memory); + if matches!(self.stage, STAGE_RENDER | STAGE_DRM) { + self.render_job = Some(crate::g18::boot::render_stage::prepare(&self.dev, &self.uat)?); + self.memory.as_mut().ok_or(EIO)?.prepare_render(&self.dev, + self.render_job.as_mut().ok_or(EIO)?)?; + } + if matches!(self.stage, STAGE_QUEUE_REGISTER | STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD) + || (self.stage == STAGE_DRM && *crate::module_parameters::g18_admit_compute.value() == 1) { + self.queue_probe = Some(G18QueueProbe::new(&self.dev, &self.uat, + matches!(self.stage, STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD | STAGE_DRM))?); + } + self.uat.confirm_t8140_primary_high_root(roots)?; + self.uat.mirror_t8140_secondary_high_root()?; + let memory = self.memory.as_mut().ok_or(EIO)?; + if matches!(self.stage, STAGE_COMPUTE_ADD | STAGE_RENDER | STAGE_DRM) { + memory.use_firmware_queue_programming(&self.dev)?; + } + memory.log_summary(&self.dev); + dev_info!(self.dev.as_ref(), "G18_INIT status links before publish {:x?}; counters {:?}\n", + memory.status_links()?, memory.control_counters()?); + Ok(()) + } + + /// Stage 3: publish GFX's root, then GFX1's, each on EP 0x20 with the + /// 0x81 doorbell, and wait for that role's 0x0009 ACK. Native 26A428 + /// captures 051/053/073 all publish in this order. The ASC CPU/RTKit + /// startup order is separate and remains GFX1 first. + fn publish_roots(&mut self) -> Result { + for role in [G18Role::Gfx, G18Role::Gfx1] { + let index = role.index(); + let address = self.memory.as_ref().ok_or(EIO)?.root(role == G18Role::Gfx1); + let message = g17_rtkit::encode_initdata_doorbell(address).map_err(|_| EINVAL)?; + { + let slot = &mut self.roles[index]; + for endpoint in [EP_GFX_MESSAGES, EP_GFX_INTERRUPTS] { + if !Pin::new(&mut slot.rtkit).has_endpoint(endpoint) { + dev_err!(self.dev.as_ref(), "G18_INIT role={} did not advertise EP {:#x}\n", + role.name(), endpoint); + return Err(ENODEV); + } + } + Pin::new(&mut slot.rtkit).start_endpoint(EP_GFX_MESSAGES)?; + Pin::new(&mut slot.rtkit).start_endpoint(EP_GFX_INTERRUPTS)?; + Pin::new(&mut slot.rtkit).send_message(EP_GFX_MESSAGES, message)?; + } + dev_info!(self.device(), "G18_INIT role={} published root={:#x} doorbell={:#018x}\n", + role.name(), address, message); + let started = Instant::::now(); + let mut acked = false; + for _ in 0..ACK_POLLS { + self.check_faults("initdata ACK wait")?; + if self.states[index].acked.load(Ordering::Acquire) { + acked = true; + break; + } + fsleep(Delta::from_millis(10)); + } + if !acked { + dev_err!(self.device(), "G18_INIT role={} no 0x0009 ACK within {} ms\n", + role.name(), ACK_POLLS * 10); + return Err(ETIMEDOUT); + } + dev_info!(self.device(), "G18_INIT role={} initdata-ack after {} us\n", + role.name(), started.elapsed().as_micros_ceil()); + } + let memory = self.memory.as_mut().ok_or(EIO)?; + dev_info!(self.dev.as_ref(), "G18_INIT both roles ACKed; status links {:x?}; counters {:?}; completion cursors {:?}\n", + memory.status_links()?, memory.control_counters()?, memory.completion_cursors()?); + self.check_faults("after both ACKs") + } + + /// Stage 4: 0x89 power-transition-end to GFX, then GFX1, and wait for + /// both opening records (0x15 / 0x2a) to retire. + fn retire_openings(&mut self) -> Result { + let before = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + dev_info!(self.device(), "G18_OPEN before power-transition-end {:?}\n", before); + for role in G18Role::STOP_ORDER { + Pin::new(&mut self.roles[role.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, g17_rtkit::encode_power_transition_end())?; + } + let mut counters = before; + for _ in 0..CONTROL_POLLS { + self.check_faults("opening retire")?; + counters = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + if counters == [[1; 3]; 2] { + dev_info!(self.device(), "G18_OPEN both control openings retired {:?}\n", counters); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.device(), "G18_OPEN openings did not retire {:?}\n", counters); + Err(ETIMEDOUT) + } + + /// Stage 4: publish IdlePowerOff(0) (opcode 0x0a) and ring EP 0x21 with + /// the idle-policy wake, then wait for GFX's control ring to retire it. + fn wake_idle_policy(&mut self) -> Result { + self.memory.as_mut().ok_or(EIO)?.publish_idle_wake()?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, IDLE_POLICY_WAKE)?; + let mut counters = [[0; 3]; 2]; + for _ in 0..CONTROL_POLLS { + self.check_faults("idle-policy wake")?; + counters = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + if counters == [[2; 3], [1; 3]] { + dev_info!(self.device(), "G18_POWER idle policy retired {:?}\n", counters); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.device(), "G18_POWER wake did not retire {:?}\n", counters); + Err(ETIMEDOUT) + } + + fn queue_probe_status(&mut self) -> Result<([u32; 3], [u32; 3], [u64; 5])> { + let outer = self.memory.as_mut().ok_or(EIO)?.compute_outer_counters()?; + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + Ok((outer, probe.counters()?, probe.slot())) + } + + /// Register one empty queue. No scheduler binding, event or tag14 kick. + fn register_queue(&mut self) -> Result { + let before = self.queue_probe_status()?; + dev_info!(self.device(), "G18_QUEUE before tag15-only {:?}\n", before); + if before != ([0; 3], [0, 0, 1], [0; 5]) { return Err(EBUSY); } + let address = self.queue_probe.as_ref().ok_or(EIO)?.queue_address(); + self.memory.as_mut().ok_or(EIO)?.publish_queue_probe(address, QUEUE_ID, 1)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + for _ in 0..CONTROL_POLLS { + self.check_faults("queue registration")?; + let (outer, inner, slot) = self.queue_probe_status()?; + if outer == [1; 3] && inner == [1; 3] && G18QueueProbe::installed(slot) { + dev_info!(self.device(), + "G18_QUEUE installed qid11 outer={:?} inner={:?} slot={:x?}; no kicks\n", + outer, inner, slot); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + let status = self.queue_probe_status()?; + dev_err!(self.device(), "G18_QUEUE registration timeout {:?}\n", status); + Err(ETIMEDOUT) + } + + fn bind_scheduler(&mut self) -> Result { + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + let before = probe.scheduler_status()?; + dev_info!(self.dev.as_ref(), "G18_SCHED before bind {:x?}\n", before); + probe.prepare_scheduler_bind()?; + let address = probe.queue_address(); + self.memory.as_mut().ok_or(EIO)?.publish_queue_probe(address, QUEUE_ID, 2)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + for _ in 0..CONTROL_POLLS { + self.check_faults("scheduler bind")?; + let (outer, inner, slot) = self.queue_probe_status()?; + let status = self.queue_probe.as_mut().ok_or(EIO)?.scheduler_status()?; + if outer == [2; 3] && inner == [2; 3] && G18QueueProbe::installed(slot) + && G18QueueProbe::scheduler_bound(status) { + dev_info!(self.device(), + "G18_SCHED bound qid11 outer={:?} inner={:?} status={:x?}; no kicks\n", + outer, inner, status); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + let queues = self.queue_probe_status()?; + let status = self.queue_probe.as_mut().ok_or(EIO)?.scheduler_status()?; + dev_err!(self.device(), "G18_SCHED bind timeout queues={:?} status={:x?}\n", queues, status); + Err(ETIMEDOUT) + } + + fn bind_pool(&mut self, binding: G18UmaPoolBinding) -> Result { + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + let before = probe.pool_status(); + dev_info!(self.dev.as_ref(), "G18_POOL before bind {:x?}; state={:#x} generation={} slot={}\n", + before, binding.state, binding.generation, binding.slot); + probe.prepare_pool_bind(binding)?; + let address = probe.queue_address(); + self.memory.as_mut().ok_or(EIO)?.publish_queue_probe(address, QUEUE_ID, 3)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + for _ in 0..CONTROL_POLLS { + self.check_faults("UMA pool bind")?; + let (outer, inner, slot) = self.queue_probe_status()?; + let status = self.queue_probe.as_ref().ok_or(EIO)?.pool_status(); + if outer == [3; 3] && inner == [3; 3] && G18QueueProbe::installed(slot) + && G18QueueProbe::pool_bound(status, binding) { + dev_info!(self.device(), + "G18_POOL bound qid11 outer={:?} inner={:?} status={:x?}; no kicks or growth\n", + outer, inner, status); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + let queues = self.queue_probe_status()?; + let status = self.queue_probe.as_ref().ok_or(EIO)?.pool_status(); + dev_err!(self.device(), "G18_POOL bind timeout queues={:?} status={:x?}\n", queues, status); + Err(ETIMEDOUT) + } + + /// Submit exactly one addition; a retired setup or changed stamp alone is + /// insufficient. Preserve the whole firmware owner after every outcome. + fn submit_addition(&mut self, command: G18PreparedComputeCommand, + pool: G18UmaPoolBinding) -> Result { + let memory = self.memory.as_mut().ok_or(EIO)?; + let output = memory.addition_output()?; + let completion = memory.compute().completion_snapshot()?; + let scheduler = memory.scheduler().snapshot(0)?; + if output.checked != 4096 || output.untouched != 4096 || output.correct != 0 + || completion.stamps != [0; 2] || memory.completion_masks() != [0; 2] + || completion.timestamps != [0; 2] || scheduler.submitted != 0 + || scheduler.completed != 0 || scheduler.linked != 0 { + return Err(EBUSY); + } + let roots = memory.compute().publish_context()?; + let probe = self.queue_probe.as_mut().ok_or(EIO)?; + probe.prepare_compute_job(command, pool)?; + let address = probe.queue_address(); + // From this counter onward, even a failed publication retains all + // memory. Firmware may already observe the host-owned counter. + let submitted = self.memory.as_mut().ok_or(EIO)?.scheduler().publish_kick(0)?; + self.queue_probe.as_mut().ok_or(EIO)?.publish_compute_job()?; + self.memory.as_mut().ok_or(EIO)?.publish_compute_queue(address, QUEUE_ID)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + dev_info!(self.device(), + "G18_COMPUTE submitted qid11 context={} roots={:x?} count={} expected_stamp={:#x}\n", + command.context_id, roots, submitted, command.expected_stamp); + // Bounded first-use run. Polling observes completion; it sends no + // additional commands and never retries a GPU job in this boot. + for step in 0..10_000 { + self.check_faults("compute addition")?; + let (outer, inner, _) = self.queue_probe_status()?; + let memory = self.memory.as_mut().ok_or(EIO)?; + let completion = memory.compute().completion_snapshot()?; + let scheduler = memory.scheduler().snapshot(0)?; + let masks = memory.completion_masks(); + if outer == [4; 3] && inner == [6; 3] + // Original b000 19234/19268 writes through the first qword. + // The second qword is carried into trace 0xe as an identity, + // not written by this completion path (stage29 hardware). + && completion.stamps[0] == command.expected_stamp + && masks[0] & (1u64 << QUEUE_ID) != 0 + && completion.timestamps[0] != 0 + && completion.timestamps[1] >= completion.timestamps[0] + && scheduler.submitted == 1 && scheduler.completed == 1 && scheduler.linked == 0 { + let output = memory.addition_output()?; + dev_info!(self.dev.as_ref(), + "G18_COMPUTE completion outer={:?} inner={:?} stamps={:x?} times={:x?} event_masks={:x?} scheduler={:?} output={:?}\n", + outer, inner, completion.stamps, completion.timestamps, masks, scheduler, output); + if *crate::module_parameters::g18_compute_variant.value() == 2 { + let pixels = memory.triangle_output()?; + dev_info!(self.dev.as_ref(), "G18_COMPUTE_TRIANGLE output={:?}\n", pixels); + if !pixels.passed() { return Err(EIO); } + self.check_faults("compute triangle output")?; + dev_info!(self.device(), "G18_COMPUTE_TRIANGLE_PASS: 4096 GPU pixels match after independent completion; compute rasterization, no TA/3D or Mesa\n"); + return Ok(()); + } + // g18_compute_variant=1 dispatches nothing: every output must stay poisoned. + let empty = *crate::module_parameters::g18_compute_variant.value() == 1; + let expected = if empty { (0, 4096) } else { (4096, 0) }; + if output.checked != 4096 || (output.correct, output.untouched) != expected { + if empty { + dev_err!(self.device(), "G18_COMPUTE_EMPTY_STREAM_FAIL: completed without a dispatch, but output changed {:?}\n", output); + } + return Err(EIO); + } + self.check_faults("compute output")?; + if empty { + dev_info!(self.device(), "G18_COMPUTE_EMPTY_STREAM_PASS: completion observed with no dispatch; all 4096 outputs still poisoned\n"); + } else { + dev_info!(self.device(), "G18_COMPUTE_ADD_PASS: all 4096 outputs match after independent completion\n"); + } + return Ok(()); + } + if step % 1000 == 999 { + dev_info!(self.dev.as_ref(), + "G18_COMPUTE wait={}ms outer={:?} inner={:?} stamps={:x?} scheduler={:?}\n", + step + 1, outer, inner, completion.stamps, scheduler); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.device(), "G18_COMPUTE timeout; owners retained, no resubmission\n"); + Err(ETIMEDOUT) + } + + /// A single bounded H1 pass; all owners already live in the retained session. + fn submit_render(&mut self) -> Result { + let pass = self.render_job.as_ref().ok_or(EIO)?.pass()?; + if self.memory.as_ref().ok_or(EIO)?.completion_masks()!=[0;2] {return Err(EBUSY);} + for phase in 1..=3 { + self.check_faults("render binding")?; + let queues = self.render_job.as_mut().ok_or(EIO)?.queues(); + let records = match phase { + 1 => queues.install_records(&pass)?, + 2 => queues.scheduler_records(&pass)?, + _ => queues.pool_records(&pass)?, + }; + self.memory.as_mut().ok_or(EIO)?.publish_outer_records(&records)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + let mut passed = false; + for _ in 0..CONTROL_POLLS { + self.check_faults("render binding")?; + let outer=self.memory.as_mut().ok_or(EIO)?.compute_outer_counters()?; + if outer==[phase*2;3] && self.render_job.as_mut().ok_or(EIO)?.queues().stage_gate(phase,&pass)? { + dev_info!(self.device(), "G18_RENDER_BIND_PASS phase={} outer={:?} stages=0,1\n",phase,outer); + passed=true;break; + } + fsleep(Delta::from_millis(1)); + } + if !passed { + let outer=self.memory.as_mut().ok_or(EIO)?.compute_outer_counters()?; + let job=self.render_job.as_mut().ok_or(EIO)?; + dev_err!(self.dev.as_ref(), "G18_RENDER_BIND_FAIL phase={} outer={:?} inner={:?} firmware={:x?} pool_pb={:x?}\n", + phase,outer,job.queues().counters()?,job.queues().firmware_status()?,job.queues().pool_and_pb_status(0,0)?); + return Err(ETIMEDOUT); + } + } + let job=self.render_job.as_mut().ok_or(EIO)?; + if job.parameter_buffer().count_submitted_render()?!=1 {return Err(EIO);} + self.memory.as_mut().ok_or(EIO)?.audit_render(&self.dev,job)?; + job.queues().prepare_pass(&pass)?; + job.queues().audit_prepared_pass(&self.dev,&pass)?; + self.check_faults("render pre-kick")?; + let job=self.render_job.as_mut().ok_or(EIO)?; + let roots=job.publish_h1()?; + let records=job.queues().publish_pass()?; + self.memory.as_mut().ok_or(EIO)?.publish_outer_records(&records)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, 0x0083_0000_0000_000a)?; + dev_info!(self.device(), "G18_RENDER_KICK context={} roots={:x?} order=3D,TA submitted=2 barrier=1 PB-generation=1\n",pass.context_id,roots); + for step in 0..10_000 { + self.check_faults("render completion")?; + let memory=self.memory.as_mut().ok_or(EIO)?; + let outer=memory.compute_outer_counters()?;let masks=memory.completion_masks(); + let job=self.render_job.as_mut().ok_or(EIO)?; + let inner=job.queues().counters()?;let stamps=job.stamps()?; + let scheduler=job.scheduler_snapshot()?;let list=job.queues().completion_list()?; + if outer==[8;3] && inner==[[6;3];2] && stamps==[0x100;2] + && scheduler.submitted==2 && scheduler.completed==2 && scheduler.linked==0 + && list==[0,job.queues().job_list()] && masks[0]&0x600==0x600 { + let pixels=job.pixels()?; + dev_info!(self.dev.as_ref(), "G18_RENDER_COMPLETION outer={:?} inner={:?} stamps={:x?} masks={:x?} scheduler={:?} list={:x?}\n",outer,inner,stamps,masks,scheduler,list); + dev_info!(self.dev.as_ref(), "G18_RENDER_PIXELS sha256={:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x} background={} poison={}\n", + pixels.0[0],pixels.0[1],pixels.0[2],pixels.0[3],pixels.0[4],pixels.0[5],pixels.0[6],pixels.0[7],pixels.1,pixels.2); + self.check_faults("render pixels")?; + dev_info!(self.device(), "G18_RENDER_H1_PASS variant={} completion=independent pixels=exact owners=retained\n",*crate::module_parameters::g18_render_variant.value()); + return Ok(()); + } + if step%1000==999 { + dev_info!(self.dev.as_ref(), "G18_RENDER_WAIT ms={} outer={:?} inner={:?} stamps={:x?} masks={:x?} scheduler={:?} list={:x?}\n",step+1,outer,inner,stamps,masks,scheduler,list); + } + fsleep(Delta::from_millis(1)); + } + Err(ETIMEDOUT) + } + + /// Stage 4..9: passive drain of both roles' event and trace rings. Sends + /// nothing. This is the milestone-1 heartbeat. + fn heartbeat(&mut self) -> Result { + dev_info!(self.device(), "G18_HEARTBEAT passive drain for {} ms, no host messages\n", HEARTBEAT_MS); + let steps = HEARTBEAT_MS / HEARTBEAT_STEP_MS; + for step in 0..steps { + self.check_faults("heartbeat")?; + if step % 100 == 99 { + let memory = self.memory.as_mut().ok_or(EIO)?; + dev_info!(self.dev.as_ref(), + "G18_HEARTBEAT t={} ms reports [role][event,trace]={:?} counters={:?} cursors={:?}\n", + (step + 1) * HEARTBEAT_STEP_MS, memory.report_counts(), + memory.control_counters()?, memory.completion_cursors()?); + } + fsleep(Delta::from_millis(HEARTBEAT_STEP_MS as i64)); + } + let counts = self.memory.as_ref().ok_or(EIO)?.report_counts(); + let alive = counts.iter().all(|role| role[1] != 0); + dev_info!(self.device(), "G18_HEARTBEAT done: reports={:?}; trace advanced on both roles={}; no recovery record\n", + counts, alive); + Ok(()) + } +} + +/// Holds a session that may have firmware running. Dropping it stops both +/// CPUs and then LEAKS every owner: the halt is not qualified, and the +/// firmware may still reference the UAT, the graph and the transports. +struct Retained(Option>); + +impl Retained { + fn stop(&mut self, label: &str) -> Result { + let Some(session) = self.0.as_mut() else { + return Ok(()); + }; + let stopped = session.stop_cpus(); + dev_info!(session.device(), "G18 {}: stop requests={:?}; owners retained until reboot\n", + label, stopped); + stopped + } +} + +impl Drop for Retained { + fn drop(&mut self) { + let _ = self.stop("session release"); + if let Some(session) = self.0.take() { + core::mem::forget(session); + } + } +} + +/// Stage-4 owner kept by the driver (`AsahiRuntime::G18`). +#[pin_data] +pub(crate) struct G18Owner { + #[pin] + session: Mutex, +} + +impl G18Runtime for G18Owner { + fn shutdown(&self) { + let _ = self.session.lock().stop("unbind"); + } +} + +// --------------------------------------------------------------------------- +// Stage 1: read-only probe +// --------------------------------------------------------------------------- + +fn stage_read_only(pdev: &platform::Device, probe: &T8152Probe) { + let dev: &kernel::device::Device = pdev.as_ref(); + for sample in probe.psd.iter() { + log_psd(pdev, "stage1 (probe sample)", sample); + } + match &probe.observed { + Some(id) => dev_info!(dev, "G18_STAGE1 SGX identity (sgx PSD proven powered): {:?}\n", id), + None => dev_info!(dev, "G18_STAGE1 SGX identity not read: sgx PSD not powered\n"), + } + dev_info!(dev, "G18_STAGE1 firmware receipt {}\n", + if probe.firmware_profile_admitted { "admitted" } else { "absent" }); + + for role in G18Role::ALL { + let provider = match G18Provider::get(dev, role) { + Ok(provider) => provider, + Err(error) => { + // G18Provider::get already explained an unbound provider. + dev_info!(dev, "G18_STAGE1 {} mailbox provider unavailable ({:?}); wrapper and post-office reads skipped\n", + role.name(), error); + continue; + } + }; + let lifecycle = match provider.validate(dev) { + Ok(lifecycle) => lifecycle, + Err(error) => { + dev_err!(dev, "G18_STAGE1 {} provider rejected ({:?}); no reads\n", role.name(), error); + continue; + } + }; + dev_info!(dev, "G18_STAGE1 {} provider wrapper={:x?} iop-vbar={:x?} post-office={:#x} missing={:#x} (bit0: CPU control not approved)\n", + role.name(), lifecycle.wrapper, lifecycle.iop_vbar, role.postoffice_window(), lifecycle.missing); + log_cpu_state(dev, role, "stage1", &provider); + match provider.postoffice_status(pdev) { + Ok((psd, Some(status))) => { + log_psd(pdev, "stage1 post-office gate", &psd); + dev_info!(dev, "G18_STAGE1 {} post-office status={:#x} (rx-occupancy={} tx-free={})\n", + role.name(), status, status & 0xf, (status >> 8) & 0xf); + } + Ok((psd, None)) => { + log_psd(pdev, "stage1 post-office gate", &psd); + dev_info!(dev, "G18_STAGE1 {} post-office status not read: role PSD not powered\n", role.name()); + } + Err(error) => dev_err!(dev, "G18_STAGE1 {} post-office read failed ({:?})\n", role.name(), error), + } + } + dev_info!(dev, "G18_STAGE1 done; nothing sent, no write issued; ENODEV\n"); +} + +// --------------------------------------------------------------------------- +// Stages 90..93: PSD requests (each REQUIRES USER APPROVAL) +// --------------------------------------------------------------------------- + +/// Native PSD::getPsdAck order: poll ACK, then read STATE once. STATE +/// clears ACK on J873; reading it while waiting loses the acknowledgement. +fn wait_psd_state(pdev: &platform::Device, block: &PsdBlock, target: u32) -> Result { + let started = Instant::::now(); + for poll in 0..PSD_POLLS { + if let Some(sample) = block.acknowledged_sample()? { + dev_info!(pdev.as_ref(), "G18_PSD service={} poll={} t={} us state={:#x} ack={:#x} (ACK read before STATE)\n", + sample.service.service_id(), poll, started.elapsed().as_micros_ceil(), + sample.state, sample.ack); + if sample.power_state() != target { + dev_err!(pdev.as_ref(), "G18_PSD service={} acknowledged unexpected state {}, wanted {}\n", + sample.service.service_id(), sample.power_state(), target); + return Err(EIO); + } + dev_info!(pdev.as_ref(), "G18_PSD service={} reached state {} after {} us; ack={:#x}\n", + sample.service.service_id(), target, started.elapsed().as_micros_ceil(), sample.ack); + return Ok(sample); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(pdev.as_ref(), "G18_PSD service={} did not acknowledge state {} within {} ms\n", + block.service.service_id(), target, PSD_POLLS); + Err(ETIMEDOUT) +} + +/// Claim one PSD request kind for this module instance. +fn claim_psd_request(dev: &kernel::device::Device, bit: u32) -> Result { + if G18_FIRMWARE_ATTEMPTED.load(Ordering::Acquire) { + dev_err!(dev, "G18_PSD refused: a firmware stage already ran this boot\n"); + return Err(EBUSY); + } + if G18_PSD_REQUESTS.fetch_or(bit, Ordering::AcqRel) & bit != 0 { + dev_err!(dev, "G18_PSD refused: this request was already written by this module instance\n"); + return Err(EBUSY); + } + Ok(()) +} + +fn map_psd_blocks(pdev: &platform::Device) -> Result> { + let mut blocks = KVec::new(); + for service in regs::T8152PsdService::ALL { + blocks.push(PsdBlock::map(pdev, service)?, GFP_KERNEL)?; + } + Ok(blocks) +} + +/// REQUIRES USER APPROVAL. Stages 90 (sgx), 91 (gfx-asc), 92 (gfx1-asc): +/// request the native active state (SGX 0, ASCs 2) on exactly one service. +/// +/// Refuses unless that service reads cold idle (state 6); gfx-asc and +/// gfx1-asc additionally need sgx to read state 0 first. Native order is +/// 3, 82, 81. Polls ACK before STATE. On a timeout nothing is restored: the service is +/// left as requested and the power state is reported UNKNOWN, to be +/// restored by stage 93 under its own approval. The SGX ID is read only if +/// sgx then proves powered. +fn stage_psd_power_on(pdev: &platform::Device, probe: &T8152Probe, service: regs::T8152PsdService) -> Result { + let dev: &kernel::device::Device = pdev.as_ref(); + dev_warn!(dev, "G18_PSD stage {}: PMGR2 PSD power-on of service {} only (REQUIRES USER APPROVAL)\n", + probe.stage, service.service_id()); + let blocks = map_psd_blocks(pdev)?; + let mut before = [regs::T8152PsdSample { service: regs::T8152PsdService::Sgx, state: 0, ack: 0 }; 3]; + for (slot, block) in before.iter_mut().zip(blocks.iter()) { + *slot = block.sample()?; + log_psd(pdev, "power-on before", slot); + } + let index = regs::T8152PsdService::ALL.iter().position(|s| *s == service).ok_or(EINVAL)?; + let target = &before[index]; + let active_state = service.active_state(); + if target.powered() { + dev_info!(dev, "G18_PSD service {} already reads state {} ({}); no request written\n", + service.service_id(), active_state, target.verdict()); + return Ok(()); + } + if target.power_state() != PSD_STATE_IDLE { + dev_err!(dev, "G18_PSD refused: service {} must read cold idle state {} first\n", + service.service_id(), PSD_STATE_IDLE); + return Err(EBUSY); + } + if service != regs::T8152PsdService::Sgx && !before[0].powered() { + dev_err!(dev, "G18_PSD refused: sgx (service 3) must read state 0 before service {} (run stage 90 first)\n", + service.service_id()); + return Err(EBUSY); + } + claim_psd_request(dev, psd_request_bit(service))?; + let block = &blocks[index]; + dev_warn!(dev, "G18_PSD write service={} request={} (REQUIRES USER APPROVAL)\n", + service.service_id(), active_state); + let result = block.request(active_state).and_then(|()| wait_psd_state(pdev, block, active_state)); + for block in blocks.iter() { + if let Ok(sample) = block.sample() { + log_psd(pdev, "power-on after", &sample); + } + } + if let Err(error) = result { + dev_err!(dev, "G18_PSD service {} power-on failed ({:?}); left as requested, GPU power state UNKNOWN; no automatic restore (stage 93 restores, REQUIRES USER APPROVAL)\n", + service.service_id(), error); + return Err(error); + } + if service == regs::T8152PsdService::Sgx { + match regs::t8152_prove_sgx_powered(pdev) { + Ok(powered) => { + let id = probe.registers.get_gpu_id_t8152(&powered, probe.soc)?; + dev_info!(dev, "G18_PSD sgx powered after acknowledged transition; SGX identity {:?}\n", id); + } + Err(error) => dev_err!(dev, "G18_PSD sgx reached state 0 but is not provably powered ({:?}); SGX not read\n", error), + } + } + dev_info!(dev, "G18_PSD stage {} done; service {} left in state {}, no firmware touched; ENODEV\n", + probe.stage, service.service_id(), active_state); + Ok(()) +} + +/// REQUIRES USER APPROVAL. Stage 93: request idle (6) on every GPU service +/// that reads its active state, in reverse native order (81, 82, 3), +/// polling ACK before STATE. Refuses if a service is neither active nor +/// idle, or if a firmware stage ran this boot. Stops at the first +/// failure and reports the power state UNKNOWN. +fn stage_psd_restore_idle(pdev: &platform::Device) -> Result { + let dev: &kernel::device::Device = pdev.as_ref(); + dev_warn!(dev, "G18_PSD stage 93: PMGR2 PSD idle restore (REQUIRES USER APPROVAL)\n"); + let blocks = map_psd_blocks(pdev)?; + let mut samples = [regs::T8152PsdSample { service: regs::T8152PsdService::Sgx, state: 0, ack: 0 }; 3]; + for (slot, block) in samples.iter_mut().zip(blocks.iter()) { + *slot = block.sample()?; + log_psd(pdev, "restore before", slot); + } + if samples.iter().any(|sample| !sample.powered() && sample.power_state() != PSD_STATE_IDLE) { + dev_err!(dev, "G18_PSD restore refused: a service is neither active nor idle\n"); + return Err(EBUSY); + } + if samples.iter().all(|sample| sample.power_state() == PSD_STATE_IDLE) { + dev_info!(dev, "G18_PSD every GPU service already reads idle; no request written\n"); + return Ok(()); + } + claim_psd_request(dev, PSD_REQUEST_RESTORE_BIT)?; + let mut result = Ok(()); + for index in [1, 2, 0] { + let block = &blocks[index]; + let sample = &samples[index]; + if sample.power_state() == PSD_STATE_IDLE { + continue; + } + dev_warn!(dev, "G18_PSD write service={} request={} (REQUIRES USER APPROVAL)\n", + block.service.service_id(), PSD_STATE_IDLE); + if let Err(error) = block.request(PSD_STATE_IDLE).and_then(|()| wait_psd_state(pdev, block, PSD_STATE_IDLE)) { + dev_err!(dev, "G18_PSD restore of service {} failed ({:?}); GPU power state is UNKNOWN; stopping\n", + block.service.service_id(), error); + result = Err(error); + break; + } + } + for block in blocks.iter() { + if let Ok(sample) = block.sample() { + log_psd(pdev, "restore after", &sample); + } + } + result +} + +// --------------------------------------------------------------------------- +// Stages 2..6 +// --------------------------------------------------------------------------- + +fn prepare_firmware_session(pdev: &platform::Device, probe: T8152Probe) -> Result { + let dev: &kernel::device::Device = pdev.as_ref(); + let stage = probe.stage; + dev_warn!(dev, "G18 stage {}: starts the GPU ASC CPUs (REQUIRES USER APPROVAL)\n", stage); + if !probe.firmware_profile_admitted { + return Err(ENODEV); + } + if G18_FIRMWARE_ATTEMPTED.load(Ordering::Acquire) { + dev_err!(dev, "G18 refused: the firmware was already started this boot; reboot first\n"); + return Err(EBUSY); + } + + // The GPU must already be powered (by iBoot, or by approved stages + // 90..92 on this boot). This path never requests PSD power. STATE reads + // consume transition ACK, so this steady-state gate checks the native + // active states (SGX 0, ASCs 2), not an already-consumed acknowledgement. + let psd = read_all_psd(pdev)?; + for sample in psd.iter() { + log_psd(pdev, "firmware-stage gate", sample); + } + if !psd.iter().all(|sample| sample.powered()) { + dev_err!(dev, "G18 stage {} refused: the GPU PSD services are not all powered; nothing started\n", stage); + return Err(ENODEV); + } + // Fresh SGX proof plus identity read. + let powered = regs::t8152_prove_sgx_powered(pdev)?; + let id = probe.registers.get_gpu_id_t8152(&powered, probe.soc)?; + dev_info!(dev, "G18 SGX identity {:?}\n", id); + + // J873 hardware, 2026-09-25: with SGX+0xe90000 left at 0x30000, the GFX1 + // firmware accepts the init data but takes an async bus error on + // SGX+0xbc08 right after the 0x89 power-transition-end. The register + // reads 0x30000 here too, the state M5 native sets to 0x30001 before its + // firmware runs; apply the same checked bit-0 step before any init data. + if stage >= STAGE_INITDATA { + probe.registers.m5_match_pmgr_boot_control()?; + } + + // Providers: exact v8 resources, CPU control approved, CPUs stopped. + let mut providers = KVec::new(); + for role in G18Role::ALL { + let provider = G18Provider::get(dev, role)?; + let lifecycle = provider.validate(dev)?; + if lifecycle.missing != 0 || provider.require_safe_cpu_lifecycle().is_err() { + dev_err!(dev, "G18 {} provider lifecycle missing={:#x}: boot with apple_mailbox.ascwrap_v8_cpu_control=1 (REQUIRES USER APPROVAL)\n", + role.name(), lifecycle.missing); + return Err(EPERM); + } + let (control, status) = provider.cpu_state()?; + dev_info!(dev, "G18_ASC {} before start CPU_CONTROL={:#x} CPU_STATUS={:#x}\n", + role.name(), control, status); + // J873 hardware, 2026-09-25: after the PSD power-on both ASCs read + // RUN=1 with one message waiting in the post-office RX FIFO (the + // firmware booted on power-up and posted its HELLO). Adopt them: + // boot_rtkit skips the CPU start and the RTKit core reads that HELLO. + if control & ASCWRAP_V8_CPU_RUN != 0 { + dev_info!(dev, "G18_ASC {} already running (RUN set); adopting the live CPU\n", + role.name()); + } + providers.push(provider, GFP_KERNEL)?; + } + + // Firmware __DATA reservations (crashlog scan source), as new_t8152 + // checks them. + let node = dev.of_node().ok_or(EINVAL)?; + let mut data = [(0u64, 0u64); 2]; + for role in G18Role::ALL { + let region = node.reserved_mem_region_to_resource_byname(role.data_region_name())?; + if region.size() != role.data_size() || region.start() & (crate::pgtable::UAT_PGMSK as u64) != 0 { + return Err(EINVAL); + } + data[role.index()] = (region.start(), region.size()); + } + let mut shared_ro = [(0u64, 0u64); 2]; + for role in G18Role::ALL { + if let Ok(region) = node.reserved_mem_region_to_resource_byname(role.shared_ro_region_name()) { + shared_ro[role.index()] = (region.start(), region.size()); + } + } + + let drm = probe.drm.clone(); + let uat = Arc::new(mmu::Uat::new_t8152(&drm, None)?, GFP_KERNEL)?; + let mut states = KVec::new(); + let mut slots = KVec::new(); + let mut providers = providers.into_iter(); + for role in G18Role::ALL { + let state = Arc::new(G18RoleState { + dev: drm.clone(), + role, + data: data[role.index()], + shared_ro: shared_ro[role.index()], + acked: AtomicBool::new(false), + crashed: AtomicBool::new(false), + crash_dumped: AtomicBool::new(false), + messages: AtomicU32::new(0), + }, GFP_KERNEL)?; + let rtkit = rtkit::RtKit::::new_g17p(drm.as_ref(), None, role.index(), state.clone()) + .map_err(|error| no_probe_defer(dev, role.name(), "RTKit mailbox", error))?; + let provider = providers.next().ok_or(EINVAL)?; + slots.push(G18RoleSlot { role, provider, rtkit, started: false }, GFP_KERNEL)?; + states.push(state, GFP_KERNEL)?; + } + let mut slots = slots.into_iter(); + let mut state_iter = states.into_iter(); + let roles = [slots.next().ok_or(EINVAL)?, slots.next().ok_or(EINVAL)?]; + let states = [state_iter.next().ok_or(EINVAL)?, state_iter.next().ok_or(EINVAL)?]; + if roles[0].role != G18Role::Gfx || roles[1].role != G18Role::Gfx1 { + return Err(EINVAL); + } + let session = KBox::new(G18Session { + roles, + states, + memory: None, + queue_probe: None, + render_job: None, + application_job: None, + uat, + _registers: probe.registers, + dev: drm, + stage, + }, GFP_KERNEL)?; + + // Point of no return: from here every path retains the session, and the + // module stays loaded so RTKit callbacks keep valid code until reboot. + G18_FIRMWARE_ATTEMPTED.store(true, Ordering::Release); + // SAFETY: THIS_MODULE is this module; the reference is never dropped. + unsafe { bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; + Ok(Retained(Some(session))) +} + +fn stage_firmware(pdev: &platform::Device, probe: T8152Probe) -> Result> { + let dev: &kernel::device::Device = pdev.as_ref(); + let stage = probe.stage; + let mut retained = prepare_firmware_session(pdev, probe)?; + if stage == STAGE_RENDER { + // A shallow workqueue stack avoids nesting render builders beneath + // platform probe/module loading. The closure owns the retained session + // before any ASC start, including on spawn failure and every exit. + kernel::workqueue::system_long().try_spawn(GFP_KERNEL, move || { + let result = match retained.0.as_mut() { + Some(session) => session.run(), None => Err(EIO), + }; + if let Some(session) = retained.0.as_mut() { + dev_info!(session.device(), "G18_RENDER_STAGE_DONE result={:?} owners=retained\n", result); + if result.is_err() { + dev_err!(session.device(), "G18_RENDER_H1_FAIL result={:?} owners=retained no-retry\n", result); + session.dump_fault_evidence("render worker"); + } + } + let _ = retained.stop("render worker end"); + if let Some(session) = retained.0.as_mut() { + for _ in 0..4 { if session.drain().is_err() { break; } } + if let Some(memory) = session.memory.as_ref() { memory.log_trace_journal(&session.dev); } + } + drop(retained); + })?; + dev_info!(dev, "G18_RENDER_WORKER queued; ENODEV is diagnostic, watch G18_RENDER_H1_PASS/FAIL and G18_RENDER_STAGE_DONE\n"); + return Err(ENODEV); + } + let session = retained.0.as_mut().ok_or(EIO)?; + let result = session.run(); + dev_info!(dev, "G18 stage {} transaction result={:?}\n", stage, result); + if result.is_err() { + if stage == STAGE_RENDER { dev_err!(dev, "G18_RENDER_H1_FAIL result={:?} owners=retained no-retry\n", result); } + session.dump_fault_evidence("failed stage"); + } + let _ = retained.stop("stage end"); + + // Bulk logging must not delay active job polling. Stop is not qualified, + // so drain only a bounded tail and retain all owners as before. The export + // accounts for consumed records; it does not assert the producer is halted. + if let Some(session) = retained.0.as_mut() { + for _ in 0..4 { + if session.drain().is_err() { + break; + } + } + if let Some(memory) = session.memory.as_ref() { + memory.log_trace_journal(&session.dev); + } + } + + match result { + Ok(()) if matches!(stage, STAGE_HEARTBEAT | STAGE_QUEUE_REGISTER | STAGE_SCHEDULER_BIND | STAGE_POOL_BIND) => { + let owner: Arc = Arc::pin_init( + try_pin_init!(G18Owner { session <- new_mutex!(retained, "G18Owner::session") }), + GFP_KERNEL, + )?; + dev_info!(dev, "G18 stage {} complete; owner retained by the driver\n", stage); + let owner: Arc = owner; + Ok(owner) + } + Ok(()) => { + dev_info!(dev, "G18 stage {} complete; exiting ENODEV with every owner retained\n", stage); + drop(retained); + Err(ENODEV) + } + Err(error) => { + drop(retained); + Err(error) + } + } +} + +/// Entry point called by the driver's T8152 probe gate for +/// `g18_initdata_test >= 1`. +pub(crate) fn start(pdev: &platform::Device, probe: T8152Probe) -> Result> { + let result = match probe.stage { + STAGE_READ_ONLY => { + stage_read_only(pdev, &probe); + Err(ENODEV) + } + stage if stage_writes_psd(stage) => { + let result = match psd_power_on_service(stage) { + Some(service) => stage_psd_power_on(pdev, &probe, service), + None => stage_psd_restore_idle(pdev), + }; + dev_info!(pdev.as_ref(), "G18 stage {} result={:?}\n", stage, result); + result.and(Err(ENODEV)) + } + stage if stage_runs_firmware(stage) => stage_firmware(pdev, probe), + _ => Err(EINVAL), + }; + // Belt and braces: no stage may ever defer the GPU probe. + let dev: &kernel::device::Device = pdev.as_ref(); + result.map_err(|error| no_probe_defer(dev, "GPU", "stage", error)) +} diff --git a/drivers/gpu/drm/asahi/g18/boot/recovery.rs b/drivers/gpu/drm/asahi/g18/boot/recovery.rs new file mode 100644 index 0000000000000..d60912f869a6f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/boot/recovery.rs @@ -0,0 +1,432 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! GPU fault and hang recovery on the retained firmware session. +//! +//! The 26A428 GFX firmware resets the GPU itself (recovery routine at b000 +//! 0x6ef0) and hands the host two edges of a status-B handshake: it writes +//! state 1 and spins until the host writes 2, resets, then writes 3 and spins +//! until the host writes 0. It begins one on its own after a fault or its +//! watchdog, posting a type-4 event first, or when the host asks with EP 0x21 +//! type 6 (`AGXArmFirmware::stopFirmwareForRecoveryInspection`), posting no +//! event. This is the halted/resume protocol upstream Asahi drives on +//! G13/G14, in its newer shape. Servicing it makes a fault or hang end one job +//! instead of the boot. +use super::*; +use crate::g18::initdata::{G18RecoveryCause, G18RecoveryWord}; + +/// Recoveries serviced this boot. There is one firmware session per boot. +static RECOVERIES: AtomicU32 = AtomicU32::new(0); + +/// EP 0x21 type 6: b000 0x255b0 requests a host recovery when the low 44 +/// bits equal the status-B recovery epoch. +const HOST_RECOVERY_TAG: u64 = 0x0086_0000_0000_0000; +const HOST_RECOVERY_EPOCH_MASK: u64 = (1 << 44) - 1; +/// The firmware's per-QID queue records, which its restart walk (b000 +/// 0x10760-0x10800) reads: +0x18 an activity gate it refreshes from the +/// scheduler, +0x40 its stamp position, +0x48 the count of stamps to flush. +/// They live in the GFX role's __DATA reservation, whose first byte is VA +/// 0xfffffc0000050000; the host reads them only while the firmware is halted. +const FW_DATA_VA: u64 = 0xffff_fc00_0005_0000; +const FW_QUEUE_RECORDS_VA: u64 = 0xffff_fc00_000b_b530; +const FW_QUEUE_RECORD_STRIDE: usize = 0x58; +/// Queues the application paths submit to: both graphics stages, compute. +const APPLICATION_QIDS: [usize; 3] = [9, 10, 11]; + +/// The firmware halts right after it posts its event or takes the request. +const RECOVERY_HALT_POLLS: u32 = 2000; // x 1 ms +/// Between the host's 2 and the firmware's 3 the GPU is reset. +const RECOVERY_RESET_POLLS: u32 = 5000; // x 1 ms +/// A job that has neither retired nor been recovered by the firmware after +/// this long is hung; the host then requests the recovery itself. +const JOB_WATCHDOG_MS: u32 = 10_000; +/// After a recovery, how long the ended job's queues may take to show that +/// the firmware consumed it. +const RECOVERED_RETIRE_MS: u32 = 2000; + +/// Poll interval for a wait that has lasted `us`: fine while a job is +/// likely to be about to retire, coarse once it has clearly not. +pub(super) const fn poll_interval_us(us: i64) -> i64 { + if us < 2_000 { 20 } else if us < 50_000 { 200 } else { 1_000 } +} + +/// Upper bounds, in microseconds, of the latency buckets; the last bucket is +/// open-ended. +const LATENCY_BOUNDS_US: [i64; 6] = [100, 300, 1_000, 3_000, 10_000, 100_000]; +const LATENCY_BUCKETS: usize = LATENCY_BOUNDS_US.len() + 1; + +/// A cumulative latency histogram for this boot. +pub(super) struct Latency { + buckets: [AtomicU32; LATENCY_BUCKETS], + max_us: AtomicU32, +} + +impl Latency { + const fn new() -> Self { + #[allow(clippy::declare_interior_mutable_const)] + const EMPTY: AtomicU32 = AtomicU32::new(0); + Self { buckets: [EMPTY; LATENCY_BUCKETS], max_us: EMPTY } + } + + /// Count one sample; returns the samples counted so far. + pub(super) fn record(&self, us: i64) -> u32 { + let index = LATENCY_BOUNDS_US.iter().position(|&bound| us <= bound).unwrap_or(LATENCY_BOUNDS_US.len()); + self.buckets[index].fetch_add(1, Ordering::Relaxed); + self.max_us.fetch_max(us.clamp(0, u32::MAX as i64) as u32, Ordering::Relaxed); + self.buckets.iter().map(|bucket| bucket.load(Ordering::Relaxed)).sum() + } + + fn snapshot(&self) -> ([u32; LATENCY_BUCKETS], u32) { + let mut buckets = [0; LATENCY_BUCKETS]; + for (count, bucket) in buckets.iter_mut().zip(&self.buckets) { + *count = bucket.load(Ordering::Relaxed); + } + (buckets, self.max_us.load(Ordering::Relaxed)) + } +} + +/// Kick to retirement of jobs that retired normally, as the session worker +/// sees it. +static JOB_LATENCY: Latency = Latency::new(); +/// A submitting thread's whole wait on the session worker: queueing, +/// preparation, kick, completion and retirement. +pub(super) static SUBMIT_LATENCY: Latency = Latency::new(); + +/// Microseconds after `since` at which each condition was first seen true +/// (0: not yet). Diagnostic: shows which firmware-owned condition a job's +/// retirement waits on. +pub(super) fn note_phases(first: &mut [i64], since: &Instant, conditions: &[bool]) { + let us = since.elapsed().as_micros_ceil().max(1); + for (seen, &holds) in first.iter_mut().zip(conditions) { + if holds && *seen == 0 { + *seen = us; + } + } +} + +/// Whether the `n`th job of a path is sampled for phase timing. +pub(super) fn phases_due(n: u32) -> bool { + n <= 3 || n % 1024 == 0 +} + +/// Whether the `jobs`th retired job is one whose latency summary is logged. +fn latency_report_due(jobs: u32) -> bool { + (jobs >= 64 && jobs.is_power_of_two()) || jobs % 4096 == 0 +} + +/// One serviced firmware recovery. +#[derive(Debug, Copy, Clone)] +pub(super) struct G18Recovery { + pub(super) epoch: u64, + pub(super) host: bool, + pub(super) cause: G18RecoveryCause, + pub(super) us: u64, +} + +/// How one job's bounded wait ended. +pub(super) enum JobEnd { + /// The job retired normally. + Retired, + /// A recovery ended the job; its queues then showed it consumed. + Recovered(G18Recovery), + /// A recovery ended the job; its queues never showed it consumed, so the + /// firmware may still reference its memory. + Stranded(G18Recovery), +} + +/// The two application job paths; each can be quarantined alone. +#[derive(Debug, Copy, Clone)] +pub(super) enum JobPath { + Render, + Compute, +} + +impl JobPath { + pub(super) const fn bit(self) -> u32 { + match self { + JobPath::Render => 1, + JobPath::Compute => 2, + } + } +} + +/// A job failure that leaves the session running. +pub(super) enum JobFailure { + /// Ended by a recovery; the queues were retired and stay usable. + Recovered(G18Recovery), + /// Ended by a recovery; its owners stay retained and its path refuses + /// further jobs. + Stranded(G18Recovery, JobPath), + /// Ended by a recovery; its queues retired and its memory was released, + /// but the path's firmware bindings did not survive the recovery, so the + /// path refuses further jobs. The client is not poisoned. + Unbound(G18Recovery, JobPath, &'static str), + /// [`Self::Recovered`] as reported back to the submitting thread, which + /// needs only whether the job's memory was released. + Released, + /// [`Self::Stranded`] as reported back to the submitting thread. + Retained, +} + +/// How far a failed task reaches. +pub(super) enum Failure { + /// The session can no longer be trusted and is stopped. + Fatal(Error), + /// Only this job failed; the session keeps serving. + Job(JobFailure), +} + +impl From for Failure { + fn from(error: Error) -> Self { + Failure::Fatal(error) + } +} + +impl From for Failure { + fn from(error: kernel::alloc::AllocError) -> Self { + Failure::Fatal(error.into()) + } +} + +pub(super) fn recoveries() -> u32 { + RECOVERIES.load(Ordering::Relaxed) +} + +impl G18Session { + /// Drain the report rings and service any recovery the firmware has + /// begun. The state word, not the event, is authoritative: a host + /// recovery posts no event. + pub(super) fn poll_faults(&mut self, label: &str) -> Result> { + let event = self.drain()?; + if self.any_crashed() { + dev_err!(self.device(), "G18 {}: firmware crashed\n", label); + return Err(EIO); + } + let halted = match self.memory.as_mut() { + Some(memory) => memory.recovery_word()?.state != 0, + None => false, + }; + if !event && !halted { + return Ok(None); + } + if event { + dev_err!(self.device(), "G18 {}: firmware posted a recovery event\n", label); + } + self.recover(label, false).map(Some) + } + + /// Poll before a kick. A recovery with no job in flight blames nothing + /// this job owns, so the job proceeds. + pub(super) fn poll_idle_faults(&mut self, label: &str) -> Result { + if let Some(recovery) = self.poll_faults(label)? { + dev_warn!(self.device(), "G18_RECOVERY_IDLE {} epoch={} host={}: no job was in flight\n", + label, recovery.epoch, recovery.host); + } + Ok(()) + } + + /// Complete one recovery handshake. With `request`, first ask the idle + /// firmware for one. Resumes from whichever edge the firmware is at, so + /// an interrupted handshake can be finished. + pub(super) fn recover(&mut self, label: &str, request: bool) -> Result { + let started = Instant::::now(); + let word = self.recovery_word()?; + if request && word.state == 0 { + let message = HOST_RECOVERY_TAG | (word.epoch & HOST_RECOVERY_EPOCH_MASK); + dev_err!(self.device(), "G18_RECOVERY_REQUEST {} epoch={} message={:#018x}\n", + label, word.epoch, message); + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit) + .send_message(EP_GFX_INTERRUPTS, message)?; + } + let mut word = word; + if word.state == 0 { + word = self.await_recovery_state(label, "halt", 0, 1, RECOVERY_HALT_POLLS)?; + } + let mut cause = G18RecoveryCause::default(); + if word.state == 1 { + cause = self.memory.as_mut().ok_or(EIO)?.recovery_cause()?; + dev_err!(self.device(), + "G18_RECOVERY_HALT {} epoch={} host={} source-count={} command={:#x} reason={:#x} detail={:#x} slot-mask={:#x} request={:x?}\n", + label, word.epoch, word.host, cause.source_count, cause.command, cause.reason, + cause.detail, cause.slot_mask, cause.request); + let memory = self.memory.as_mut().ok_or(EIO)?; + let progress = [memory.queue_progress(9)?, memory.queue_progress(10)?, memory.queue_progress(11)?]; + dev_err!(self.device(), "G18_RECOVERY_PROGRESS {} qid9={:x?} qid10={:x?} qid11={:x?}\n", + label, progress[0], progress[1], progress[2]); + self.log_firmware_queue_records(label, "halted"); + self.log_parameter_buffer(label, "halted"); + self.publish_queue_progress(label)?; + self.memory.as_mut().ok_or(EIO)?.acknowledge_recovery(1, 2)?; + word.state = 2; + } + if word.state == 2 { + word = self.await_recovery_state(label, "reset", 2, 3, RECOVERY_RESET_POLLS)?; + } + if word.state != 3 { + dev_err!(self.device(), "G18_RECOVERY_FAIL {} phase=clear state={} epoch={}\n", + label, word.state, word.epoch); + return Err(EIO); + } + let info = self.memory.as_mut().ok_or(EIO)?.recovery_cause()?; + self.log_firmware_queue_records(label, "restarted"); + self.log_parameter_buffer(label, "restarted"); + self.memory.as_mut().ok_or(EIO)?.acknowledge_recovery(3, 0)?; + self.drain()?; + // The restart walk retired every stamp up to the last-submitted table + // published before 1->2, so each queue's next kick names its last + // kicked stamp as its parent, as native encodeDependencyList does + // (queue +0xa0 survives restartWorkQueue unchanged). + let serviced = RECOVERIES.fetch_add(1, Ordering::Relaxed) + 1; + let recovery = G18Recovery { + epoch: word.epoch, + host: word.host != 0, + cause: G18RecoveryCause { valid_entries: info.valid_entries, first_entry: info.first_entry, + pending: info.pending, ..cause }, + us: started.elapsed().as_micros_ceil() as u64, + }; + dev_err!(self.device(), + "G18_RECOVERY_DONE {} epoch={} host={} us={} recoveries={} info-valid={} info-first={:x?}\n", + label, recovery.epoch, recovery.host, recovery.us, serviced, + info.valid_entries, info.first_entry); + let logged = (info.valid_entries as usize).min(info.pending.len()); + for &(index, raw) in &info.pending[..logged] { + let entry = raw as u32; + dev_err!(self.device(), "G18_RECOVERY_PENDING {} index={} slot={} flags={:#x} value={:#x} raw={:#018x}\n", + label, index, (entry >> 4) & 0x7f, entry & 0xf, (raw >> 32) as u32, raw); + } + Ok(recovery) + } + + /// Diagnostic: the render PB's firmware-visible words and scene record. + fn log_parameter_buffer(&mut self, label: &str, phase: &str) { + let Some(job) = self.application_job.as_mut() else { return }; + match job.job.parameter_buffer().recovery_words() { + Ok((state, scene_words, counter, scene)) => dev_err!(self.device(), + "G18_RECOVERY_PB {} {} counter={} scene={} state={:x?} scene_record={:x?}\n", + label, phase, counter, scene, state, scene_words), + Err(error) => dev_err!(self.device(), "G18_RECOVERY_PB {} {} unreadable={:?}\n", label, phase, error), + } + } + + /// What native `restartWorkQueue` does before resuming a halted + /// firmware: publish each queue's last submitted kick stamp. The restart + /// walk flushes every stamp from the queue's own position up to this one + /// (count = stamp - position + 1), which is what retires a stamp the + /// recovery killed; without it the queue's next kick waits on that stamp + /// forever. Queues that never kicked keep their zero entry. + fn publish_queue_progress(&mut self, label: &str) -> Result { + let render = self.application_job.as_ref().map_or(0, |job| job.submitted_generation()); + let memory = self.memory.as_mut().ok_or(EIO)?; + let compute = memory.application_compute_generation(); + for (qid, stamp) in [(9, render), (10, render), (11, compute)] { + if stamp != 0 { + memory.publish_queue_progress(qid, 1, u64::from(stamp))?; + } + } + let progress = [memory.queue_progress(9)?, memory.queue_progress(10)?, memory.queue_progress(11)?]; + dev_err!(self.device(), "G18_RECOVERY_PUBLISH {} qid9={:x?} qid10={:x?} qid11={:x?}\n", + label, progress[0], progress[1], progress[2]); + Ok(()) + } + + /// Log the firmware's own records for the application queues. Read-only + /// and diagnostic: a failure to map is logged, never fatal. + fn log_firmware_queue_records(&self, label: &str, phase: &str) { + let (base, size) = self.states[G18Role::Gfx.index()].data; + let map = match usize::try_from(size).map_err(|_| EINVAL).and_then(|size| PhysMap::new(base, size)) { + Ok(map) => map, + Err(error) => { + dev_warn!(self.device(), "G18_RECOVERY_FWQ {} {}: __DATA map failed ({:?})\n", label, phase, error); + return; + } + }; + for qid in APPLICATION_QIDS { + let offset = (FW_QUEUE_RECORDS_VA - FW_DATA_VA) as usize + qid * FW_QUEUE_RECORD_STRIDE; + let mut raw = [0u8; FW_QUEUE_RECORD_STRIDE]; + if let Err(error) = map.copy(offset, &mut raw) { + dev_warn!(self.device(), "G18_RECOVERY_FWQ {} {} qid={}: read failed ({:?})\n", label, phase, qid, error); + continue; + } + let qword = |at: usize| u64::from_le_bytes(raw[at..at + 8].try_into().unwrap_or([0; 8])); + let words: [u64; FW_QUEUE_RECORD_STRIDE / 8] = core::array::from_fn(|index| qword(index * 8)); + dev_err!(self.device(), + "G18_RECOVERY_FWQ {} {} qid={} gate={:#x} position={:#x} count={:#x} raw={:x?}\n", + label, phase, qid, words[3] as u32, words[8], words[9], words); + } + } + + fn recovery_word(&mut self) -> Result { + self.memory.as_mut().ok_or(EIO)?.recovery_word() + } + + fn await_recovery_state(&mut self, label: &str, phase: &str, from: u32, to: u32, polls: u32) + -> Result { + let mut word = self.recovery_word()?; + for _ in 0..polls { + if word.state == to || word.state != from || self.any_crashed() { + break; + } + // Keep the report rings moving while the firmware works. + self.drain()?; + fsleep(Delta::from_millis(1)); + word = self.recovery_word()?; + } + if word.state == to { + return Ok(word); + } + dev_err!(self.device(), "G18_RECOVERY_FAIL {} phase={} state={} epoch={} host={} crashed={}\n", + label, phase, word.state, word.epoch, word.host, self.any_crashed()); + Err(ETIMEDOUT) + } + + /// Wait for one kicked job. `retired(session, ms, recovered, report)` + /// reports whether the job's queues have reached their retired state; + /// once a recovery has ended the job it checks only what the firmware + /// owns, since no stage will complete. `report` is set once per second + /// of waiting (every 250 ms after a recovery) for progress logging. The + /// host watchdog requests a recovery for a job the firmware never flags. + /// Timing is wall clock; most jobs retire within a few hundred + /// microseconds, so polling starts fine and backs off. + pub(super) fn wait_job(&mut self, label: &str, mut retired: F) -> Result + where + F: FnMut(&mut Self, u32, bool, bool) -> Result, + { + let mut recovery: Option = None; + let mut since = Instant::::now(); + let mut reported = 0u32; + loop { + if let Some(serviced) = self.poll_faults(label)? { + if recovery.is_none() { + since = Instant::::now(); + reported = 0; + recovery = Some(serviced); + } + } + let us = since.elapsed().as_micros_ceil(); + let ms = (us / 1000) as u32; + let period = if recovery.is_some() { 250 } else { 1000 }; + let report = ms / period > reported; + if report { + reported = ms / period; + } + if retired(self, ms, recovery.is_some(), report)? { + if recovery.is_none() && latency_report_due(JOB_LATENCY.record(us)) { + let (job, job_max) = JOB_LATENCY.snapshot(); + let (submit, submit_max) = SUBMIT_LATENCY.snapshot(); + dev_info!(self.device(), "G18_JOB_LATENCY bounds_us={:?} job={:?} job_max_us={} submit={:?} submit_max_us={}\n", + LATENCY_BOUNDS_US, job, job_max, submit, submit_max); + } + return Ok(recovery.map_or(JobEnd::Retired, JobEnd::Recovered)); + } + if let Some(ended) = recovery { + if ms >= RECOVERED_RETIRE_MS { + return Ok(JobEnd::Stranded(ended)); + } + } else if ms >= JOB_WATCHDOG_MS { + dev_err!(self.device(), "G18_JOB_WATCHDOG {} ms={}: requesting recovery\n", label, ms); + recovery = Some(self.recover(label, true)?); + since = Instant::::now(); + reported = 0; + } + fsleep(Delta::from_micros(poll_interval_us(us))); + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/boot/render_stage.rs b/drivers/gpu/drm/asahi/g18/boot/render_stage.rs new file mode 100644 index 0000000000000..51e2ff13c3369 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/boot/render_stage.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! H1 admission and retained-owner construction. H2 is deliberately excluded. +//! The caller must execute every runtime readback gate before publication. +use crate::{driver::AsahiDevice, g18::runtime::graphics_job::G18GraphicsJob, + g18::fw::render_assembly::RenderVariant, mmu}; +use kernel::prelude::*; + +pub(crate) const STAGE_RENDER: u32 = 9; +pub(crate) const H1_ADMITTED: bool = true; + +pub(crate) fn options(variant: u32, profile: u32) -> Result { + let variant = RenderVariant::parse(variant).ok_or(EINVAL)?; + let admitted = ((1..=2).contains(&profile) + && matches!(variant, RenderVariant::Color | RenderVariant::Clear)) + || (profile == 4 && variant == RenderVariant::Color); + if !admitted { return Err(EINVAL); } + Ok(variant) +} + +/// Called before prepare_t8152_device: rejection precedes MMIO, allocation, +/// firmware startup and the init-data publication boundary. +pub(crate) fn admit(dev: &kernel::device::Device, variant: u32, profile: u32) -> Result { + let variant = options(variant, profile)?; + if !H1_ADMITTED { + dev_err!(dev, "G18_RENDER_BLOCKED variant={:?} profile={} frames={} stages={}: native command policy, graphics UMA backing, PB/queue binding and complete pointer/difference ledger are not admitted; refused before MMIO\n", + variant, profile, variant.frames(), variant.stage_count()); + return Err(ENOTSUPP); + } + dev_info!(dev, "G18_RENDER_ADMIT variant={:?} profile={} policy=captured-M5-H1 single-pass\n", variant, profile); + dev_info!(dev, "G18_RENDER_RESOURCE_PAIRS {:?}\n", crate::g18::uapi::render_policy::resource_pairs()); + dev_info!(dev, "G18_RENDER_TA_BARRIER_FLAG {}\n", crate::g18::uapi::render_policy::ta_barrier_flag()); + Ok(()) +} + +#[inline(never)] +pub(crate) fn prepare(dev: &AsahiDevice, uat: &mmu::Uat) -> Result> { + let variant = *crate::module_parameters::g18_render_variant.value(); + let profile = *crate::module_parameters::g18_render_profile.value(); + let options = options(variant, profile)?; + G18GraphicsJob::from_payload(dev, uat, profile, options) +} diff --git a/drivers/gpu/drm/asahi/g18/boot/session.rs b/drivers/gpu/drm/asahi/g18/boot/session.rs new file mode 100644 index 0000000000000..7ca41483103cb --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/boot/session.rs @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Stage10 owner: synchronous DRM boundary, shallow worker stack. A job that a +//! firmware recovery ends fails alone; any other fault is sticky. +use super::*; +#[path = "compute_session.rs"] +mod compute_session; +#[path = "recovery.rs"] +mod recovery; +use recovery::{Failure, JobEnd, JobFailure, JobPath}; +use core::{ops::Range,sync::atomic::AtomicI32}; +use crate::{g18::uapi::application::{G18ApplicationRender,G18RenderJobSink,G18RenderRetirement}, + g18::drm::{self as g18_drm,G18DrmRuntime},gem, + g18::runtime::graphics_job::application::ApplicationJob}; + +pub(crate) struct ApplicationOwner { + session: Arc>, + failed: Arc, + /// [`JobPath::bit`]s of paths a stranded job quarantined. + quarantined: Arc, +} + +impl ApplicationOwner { + fn admit(&self,path:JobPath)->Result { + if self.quarantined.load(Ordering::Acquire)&path.bit()==0 {return Ok(());} + if let Some(session)=self.session.lock().0.as_ref() { + dev_err!(session.device(),"G18_APP_JOB_FAILED error={:?} refused path={:?} quarantined session=live\n",EIO,path); + } + Err(EIO) + } + + /// Run one task on the session worker. The outcome crosses back as one + /// word: [`TICKET_RUNNING`], 0, a negative errno for a fatal failure, or + /// a positive [`JobOutcome`] code for a job that failed alone. + fn work(&self, task: F) -> core::result::Result<(),Failure> + where F: FnOnce(&mut G18Session)->core::result::Result<(),Failure> + Send + 'static { + if self.failed.load(Ordering::Acquire) {return Err(Failure::Fatal(EIO));} + let session=self.session.clone();let failed=self.failed.clone(); + let quarantined=self.quarantined.clone(); + let ticket=Arc::new(AtomicI32::new(TICKET_RUNNING),GFP_KERNEL)?;let done=ticket.clone(); + kernel::workqueue::system_long().try_spawn(GFP_KERNEL,move || { + let mut owner=session.lock(); + let result=if failed.load(Ordering::Acquire) {Err(Failure::Fatal(EIO))} else { + owner.0.as_mut().ok_or(Failure::Fatal(EIO)).and_then(|session|task(session)) + }; + // A job that a recovery ended fails alone; the firmware session + // keeps serving other jobs. Anything else ends the session. + let result=match result { + Ok(()) => Ok(()), + Err(Failure::Job(failure)) => match owner.0.as_ref() { + Some(session) if !session.any_crashed() => { + if let JobFailure::Stranded(_,path) | JobFailure::Unbound(_,path,_)=&failure { + quarantined.fetch_or(path.bit(),Ordering::AcqRel); + } + log_job_failure(session,&failure); + Err(Failure::Job(failure)) + } + _ => Err(Failure::Fatal(EIO)), + }, + Err(fatal) => Err(fatal), + }; + if let Err(Failure::Fatal(error))=result { + failed.store(true,Ordering::Release); + if let Some(session)=owner.0.as_mut() { + if let Some(job)=session.application_job.as_mut() {job.quarantine();} + if let Some(memory)=session.memory.as_mut() {memory.quarantine_application_compute();} + dev_err!(session.device(),"G18_APP_FAIL error={:?} owners=retained no-retry\n",error); + session.dump_fault_evidence("application worker"); + if let Some(memory)=session.memory.as_ref() {memory.log_trace_journal(&session.dev);} + } + let _=owner.stop("application fault"); + } + done.store(match result { + Ok(())=>0, + Err(Failure::Fatal(error))=>error.to_errno(), + Err(Failure::Job(JobFailure::Recovered(_) | JobFailure::Unbound(..) | JobFailure::Released))=>JobOutcome::Recovered as i32, + Err(Failure::Job(JobFailure::Stranded(..) | JobFailure::Retained))=>JobOutcome::Stranded as i32, + },Ordering::Release); + })?; + let waited=Instant::::now(); + loop { + match ticket.load(Ordering::Acquire) { + TICKET_RUNNING => fsleep(Delta::from_micros(recovery::poll_interval_us(waited.elapsed().as_micros_ceil()))), + 0 => {recovery::SUBMIT_LATENCY.record(waited.elapsed().as_micros_ceil());return Ok(())} + code if code==JobOutcome::Recovered as i32 => return Err(Failure::Job(JobFailure::Released)), + code if code==JobOutcome::Stranded as i32 => return Err(Failure::Job(JobFailure::Retained)), + errno => return Err(Failure::Fatal(Error::from_errno(errno))), + } + } + } + + /// Fail one client's job. Its VM is poisoned only while the firmware may + /// still reference the job's memory; a job whose queues retired after a + /// recovery fails alone, as a faulting job does upstream. + fn finish(vm:&mmu::Vm,result:core::result::Result<(),Failure>)->Result { + match result { + Ok(()) => Ok(()), + Err(Failure::Job(JobFailure::Released)) | Err(Failure::Job(JobFailure::Recovered(_))) => Err(EIO), + Err(Failure::Job(_)) => {if let Some(status)=vm.status(){status.record(EIO);}Err(EIO)} + Err(Failure::Fatal(error)) => {if let Some(status)=vm.status(){status.record(error);}Err(error)} + } + } +} + +/// Worker ticket values besides 0 and negative errnos. +const TICKET_RUNNING: i32 = 1; +#[repr(i32)] +enum JobOutcome { + Recovered = 2, + Stranded = 3, +} + +impl G18Runtime for ApplicationOwner { + fn shutdown(&self) { + self.failed.store(true,Ordering::Release); + let _=self.session.lock().stop("DRM unbind"); + } +} +impl G18DrmRuntime for ApplicationOwner { + fn submit_compute(&self,vm:&mmu::Vm,command:&crate::g18::uapi::command::TranslatedComputeCommand) + ->Result { + let command=KBox::new(*command,GFP_KERNEL)?;let job_vm=vm.clone(); + self.admit(JobPath::Compute)?; + Self::finish(vm,self.work(move |session|session.submit_application_compute(&job_vm,&command)))?; + Ok(G18RenderRetirement {stage_ticks:None}) + } + fn is_crashed(&self)->bool { + self.failed.load(Ordering::Acquire) + || self.session.lock().0.as_ref().map_or(true,|s|s.any_crashed()) + } + fn new_user_vm(&self,id:u64,kernel_range:Range)->Result { + if self.failed.load(Ordering::Acquire) {return Err(EIO);} + self.session.lock().0.as_ref().ok_or(EIO)?.uat.new_vm(id,kernel_range)?.with_t8152_lifetime() + } + fn map_timestamp_buffer(&self,bo:gem::ObjectRef,range:Range)->Result { + if self.failed.load(Ordering::Acquire) {return Err(EIO);} + g18_drm::map_timestamp_buffer_in(&self.session.lock().0.as_ref().ok_or(EIO)?.uat,bo,range) + } +} +impl G18RenderJobSink for ApplicationOwner { + fn submit_render(&self,vm:&mmu::Vm,render:&G18ApplicationRender)->Result { + let render=KBox::new(*render,GFP_KERNEL)?;let job_vm=vm.clone(); + self.admit(JobPath::Render)?; + Self::finish(vm,self.work(move |session|session.submit_application(&job_vm,&render)))?; + // K8 uses the adapter's architectural-counter bracket until firmware + // timestamp units and stage boundaries have hardware qualification. + Ok(G18RenderRetirement {stage_ticks:None}) + } +} + +pub(crate) fn start_drm(pdev:&platform::Device,probe:T8152Probe) + ->Result<(Arc,Arc)> { + if probe.stage!=STAGE_DRM {return Err(EINVAL);} + let retained=prepare_firmware_session(pdev,probe)?; + let session=Arc::pin_init(kernel::new_mutex!(retained,"G18 application session"),GFP_KERNEL)?; + let owner=Arc::new(ApplicationOwner {session,failed:Arc::new(AtomicBool::new(false),GFP_KERNEL)?, + quarantined:Arc::new(AtomicU32::new(0),GFP_KERNEL)?},GFP_KERNEL)?; + owner.work(|session|Ok(session.run()?)).map_err(|failure| match failure { + Failure::Fatal(error) => error, + Failure::Job(_) => EIO, + })?; + dev_info!(pdev.as_ref(),"G18_DRM_READY firmware=live graphics=on-first-submit timestamps=K8\n"); + crate::g18::debug::g18_mark_drm_live(); + Ok((owner.clone(),owner)) +} + +fn log_job_failure(session:&G18Session,failure:&JobFailure) { + match failure { + JobFailure::Recovered(r) => dev_err!(session.device(), + "G18_APP_JOB_FAILED error={:?} recovery epoch={} host={} us={} reason={:#x} slot-mask={:#x} queues=retired session=live recoveries={}\n", + EIO,r.epoch,r.host,r.us,r.cause.reason,r.cause.slot_mask,recovery::recoveries()), + JobFailure::Stranded(r,path) => dev_err!(session.device(), + "G18_APP_JOB_FAILED error={:?} recovery epoch={} host={} us={} reason={:#x} slot-mask={:#x} queues=stranded path={:?} quarantined session=live recoveries={}\n", + EIO,r.epoch,r.host,r.us,r.cause.reason,r.cause.slot_mask,path,recovery::recoveries()), + JobFailure::Unbound(r,path,unready) => dev_err!(session.device(), + "G18_APP_JOB_FAILED error={:?} recovery epoch={} host={} us={} reason={:#x} slot-mask={:#x} queues=retired unbound={} path={:?} quarantined session=live recoveries={}\n", + EIO,r.epoch,r.host,r.us,r.cause.reason,r.cause.slot_mask,unready,path,recovery::recoveries()), + // Only the submitting thread's summaries; the worker logs the rest. + JobFailure::Released | JobFailure::Retained => {} + } +} + +impl G18Session { + #[inline(never)] + fn submit_application(&mut self,vm:&mmu::Vm,render:&G18ApplicationRender)->core::result::Result<(),Failure> { + self.poll_idle_faults("application prepare")?; + if self.application_job.is_none() { + self.application_job=Some(ApplicationJob::new(self.render_job.take().ok_or(EIO)?)?); + } + let job=self.application_job.as_mut().ok_or(EIO)?; + job.prepare(&self.dev,&self.uat,vm,render)?; + if job.pool_aliases.is_empty() { + job.pool_aliases=self.memory.as_mut().ok_or(EIO)?.application_aliases(vm)?; + } + let frame=job.frame();let pass=job.pass()?; + if frame.generation==1 { + for phase in 1..=3 { + self.check_faults("application binding")?; + let queues=self.application_job.as_mut().ok_or(EIO)?.job.queues(); + let records=match phase {1=>queues.install_records(&pass)?,2=>queues.scheduler_records(&pass)?,_=>queues.pool_records(&pass)?}; + let outer_after=self.memory.as_mut().ok_or(EIO)?.publish_application_records(&records)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit).send_message(EP_GFX_INTERRUPTS,0x0083_0000_0000_000a)?; + let mut passed=false; + for _ in 0..CONTROL_POLLS { + self.check_faults("application binding")?; + if self.memory.as_mut().ok_or(EIO)?.compute_outer_counters()?==[outer_after;3] + && self.application_job.as_mut().ok_or(EIO)?.job.queues().stage_gate(phase,&pass)? {passed=true;break;} + fsleep(Delta::from_millis(1)); + } + if !passed {return Err(ETIMEDOUT.into());} + dev_info!(self.device(),"G18_APP_BIND_PASS phase={}\n",phase); + } + } + let job=self.application_job.as_mut().ok_or(EIO)?; + if job.job.parameter_buffer().count_submitted_render()?!=frame.generation {return Err(EIO.into());} + self.memory.as_mut().ok_or(EIO)?.audit_application(&self.dev,job)?; + job.job.queues().prepare_application_pass(&self.dev,&pass,frame)?; + self.poll_idle_faults("application pre-kick")?; + let job=self.application_job.as_mut().ok_or(EIO)?; + let traced=job.traced() || *crate::module_parameters::g18_graphics_stall_trace.value()!=0; + let roots=job.publish()?; + let records=job.job.queues().publish_application_pass(frame)?; + let outer_after=self.memory.as_mut().ok_or(EIO)?.publish_application_outer(&records,frame)?; + Pin::new(&mut self.roles[G18Role::Gfx.index()].rtkit).send_message(EP_GFX_INTERRUPTS,0x0083_0000_0000_000a)?; + // The gallery needs a kick/retirement pair for every isolated client: + // each client's first frames are logged. Other frames stay sparse; + // explicit stall tracing records each job. + if traced { + crate::g18_kick_info!(self.device(),"G18_APP_KICK frame={} context={} roots={:x?} order=3D,TA stamp={:#x}\n",frame.generation,pass.context_id,roots,frame.stamp()); + } + // Phases: outer, inner, scheduler submitted, completed, linked, list, + // stamps, completion masks. + let kicked=Instant::::now();let mut first=[0i64;8]; + let end=self.wait_job("application completion",|session,ms,recovered,report| { + let memory=session.memory.as_mut().ok_or(EIO)?; + let outer=memory.compute_outer_counters()?;let masks=memory.completion_masks(); + let job=session.application_job.as_mut().ok_or(EIO)?; + let inner=job.job.queues().counters()?;let stamps=job.job.stamps()?; + let scheduler=job.job.scheduler_snapshot()?;let list=job.job.queues().completion_list()?; + // What the firmware owns: both queues consumed and the scheduler + // retired. After a recovery no stage completes, so the stamps and + // completion event are not required. + let consumed=outer==[outer_after;3] && inner==[[frame.inner_after();3];2] + && scheduler.submitted==frame.submitted() && scheduler.completed==frame.submitted() + && scheduler.linked==0 && list==[0,job.job.queues().job_list()]; + let completed=stamps==[frame.stamp();2] && masks[0]&0x600==0x600; + recovery::note_phases(&mut first,&kicked,&[outer==[outer_after;3],inner==[[frame.inner_after();3];2], + scheduler.submitted==frame.submitted(),scheduler.completed==frame.submitted(),scheduler.linked==0, + list==[0,job.job.queues().job_list()],stamps==[frame.stamp();2],masks[0]&0x600==0x600]); + if report { + dev_info!(session.device(),"G18_APP_WAIT frame={} ms={} recovered={} outer={:?} inner={:?} stamps={:x?} masks={:x?} scheduler={:?} list={:x?}\n", + frame.generation,ms,recovered,outer,inner,stamps,masks,scheduler,list); + } + if report && !recovered && (5000..6000).contains(&ms) { + let (state,scene_words,counter,scene)=session.application_job.as_mut().ok_or(EIO)?.job.parameter_buffer().recovery_words()?; + dev_info!(session.device(),"G18_APP_PB_STALL frame={} counter={} scene={} state={:x?} scene_record={:x?}\n", + frame.generation,counter,scene,state,scene_words); + } + Ok(consumed && (completed || recovered)) + })?; + let recovery=match end { + JobEnd::Retired => match self.poll_faults("application retired")? { + None => { + if recovery::phases_due(frame.generation) { + dev_info!(self.device(),"G18_JOB_PHASES frame={} us={:?} (outer inner submitted completed linked list stamps masks)\n", + frame.generation,first); + } + self.application_job.as_mut().ok_or(EIO)?.retired(); + if traced || frame.generation%256==0 { + crate::g18_kick_info!(self.device(),"G18_APP_FRAME_PASS frame={} stamp={:#x} scheduler={} outer={} inner={}\n", + frame.generation,frame.stamp(),frame.submitted(),outer_after,frame.inner_after()); + } + return Ok(()); + } + // Retired, but a recovery followed at once: do not vouch for + // the frame. Its queues are retired either way. + Some(recovery) => recovery, + }, + JobEnd::Recovered(recovery) => recovery, + JobEnd::Stranded(recovery) => { + self.application_job.as_mut().ok_or(EIO)?.quarantine(); + dev_err!(self.device(),"G18_APP_STRANDED frame={} epoch={} host={}: queues never retired; render quarantined, owners retained\n", + frame.generation,recovery.epoch,recovery.host); + return Err(Failure::Job(JobFailure::Stranded(recovery,JobPath::Render))); + } + }; + // No stage will write this frame's completion stamps; the next + // frame's audit requires them as its previous stamp. + let job=self.application_job.as_mut().ok_or(EIO)?; + job.job.publish_recovered_stamps(frame.previous_stamp(),frame.stamp())?; + job.retired(); + dev_err!(self.device(),"G18_APP_RECOVERED frame={} epoch={} host={}: queues retired\n", + frame.generation,recovery.epoch,recovery.host); + let pb=self.application_job.as_mut().ok_or(EIO)?.job.parameter_buffer(); + let (state,scene_words,counter,scene)=pb.recovery_words()?; + dev_err!(self.device(),"G18_APP_PB_POSTRECOVERY frame={} counter={} scene={} state={:x?} scene_record={:x?}\n", + frame.generation,counter,scene,state,scene_words); + // The next frame needs the queues' firmware bindings as they were; a + // recovery may reset them. Refuse the path rather than kick into it. + let queues=self.application_job.as_mut().ok_or(EIO)?.job.queues(); + let unready=queues.render_unready(&pass,frame)?; + let (status,pool,slot)=queues.render_state(&pass)?; + dev_err!(self.device(),"G18_APP_RENDER_POSTRECOVERY frame={} unready={:?} status={:x?} pool={:x?} pb={:x?}\n", + frame.generation,unready,status,pool,slot); + match unready { + None => Err(Failure::Job(JobFailure::Recovered(recovery))), + Some(what) => Err(Failure::Job(JobFailure::Unbound(recovery,JobPath::Render,what))), + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/live.rs b/drivers/gpu/drm/asahi/g18/buffer/live.rs new file mode 100644 index 0000000000000..a9f7ed88b8e54 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/live.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded volatile stores to host-owned shared firmware words. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. +use kernel::prelude::*; + +// Live shared control words are read asynchronously by firmware, including while +// older kicks retire. Byte copies are not a single-copy publication on strict- +// alignment builds. The Runtime lock serializes host RMWs; volatile accesses +// preserve native aligned LDR/STR width, not an atomic RMW across processors. +#[inline] +fn g17p_live_word_bounds(bytes: &[u8], offset: usize, width: usize) -> Result { + if !matches!(width, 4 | 8) + || offset.checked_add(width).is_none_or(|end| end > bytes.len()) + || (bytes.as_ptr() as usize).checked_add(offset) + .is_none_or(|address| address & (width - 1) != 0) + { + return Err(EINVAL); + } + Ok(()) +} + +#[inline] +pub(crate) fn g17p_live_write_u32(bytes: &mut [u8], offset: usize, value: u32) -> Result { + g17p_live_word_bounds(bytes, offset, 4)?; + // SAFETY: checked bounds and natural alignment; host owns this shared word. + unsafe { core::ptr::write_volatile(bytes.as_mut_ptr().add(offset).cast::(), value.to_le()) }; + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/mod.rs b/drivers/gpu/drm/asahi/g18/buffer/mod.rs new file mode 100644 index 0000000000000..5dd07cc47e1f5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 memory owners: kernel-mapped objects and the UMA/compute pools. + +#[allow(dead_code)] +pub(crate) mod pool; +#[allow(dead_code)] +pub(crate) mod object; +#[allow(dead_code)] +pub(crate) mod live; diff --git a/drivers/gpu/drm/asahi/g18/buffer/object.rs b/drivers/gpu/drm/asahi/g18/buffer/object.rs new file mode 100644 index 0000000000000..006e57f9647cc --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/object.rs @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Kernel-owned GEM objects with fixed firmware-VA mappings (`MappedObject`). +//! +//! Items copied from the J873 module-1134 sources (provenance in +//! the port's extraction ledger), then adapted: +//! the type and the methods G18 calls are `pub(crate)` (in module 1134 they +//! were private to g17_resources, the parent of every G18 resource module); +//! the G17P firmware-cached object pool (fields `fw_pool`/`fw_node`, its +//! Drop path and `new_fw_pooled`) and the constructors and capture helpers +//! only G17P uses are not carried. G18 never creates pooled objects, so its +//! objects always took the non-pooled path, which is the one kept. + +use core::ops::Range; + +use kernel::prelude::*; + +use super::live::g17p_live_write_u32; +use crate::{driver::AsahiDevice, gem, mmu}; + +pub(crate) struct MappedObject { + /// Present for an owning GPU mapping and absent for a subview retained by + /// another `MappedObject` in the same resource bundle. + pub(crate) mapping: Option, + pub(crate) object: gem::ObjectRef, + pub(crate) iova: u64, + pub(crate) object_offset: usize, + pub(crate) logical_size: usize, + /// Records the existing owning CPU mapping; capture never creates a + /// physical alias or upgrades a WB observation to GPU-coherent evidence. + pub(crate) cpu_wc: bool, +} + +impl MappedObject { + pub(crate) fn new_wc_in_range_with_guards( + dev: &AsahiDevice, vm: &mmu::Vm, range: Range, size: usize, + alignment: u64, prot: mmu::Prot, guard_pages: usize, + ) -> Result { + let mut object = gem::new_kernel_object_wc(dev, size)?; + object.vmap()?.memset(0); + let mapping = vm.map_in_range_with_guards(&object.gem, 0..object.size(), + alignment, range, prot, guard_pages)?; + let iova = mapping.iova(); + Ok(Self { mapping: Some(mapping), object, iova, object_offset: 0, + logical_size: size, cpu_wc: true }) + } + + pub(crate) fn map_alias_at_with_guards(&mut self, vm: &mmu::Vm, address: u64, + prot: mmu::Prot, guard_pages: usize) -> Result { + if self.object_offset != 0 { return Err(EINVAL); } + self.object.map_at_with_guards(vm, address, prot, guard_pages) + } + + + pub(crate) fn new_wc_in_range( + dev: &AsahiDevice, + vm: &mmu::Vm, + range: Range, + size: usize, + alignment: u64, + prot: mmu::Prot, + ) -> Result { + let mut object = gem::new_kernel_object_wc(dev, size)?; + object.vmap()?.memset(0); + let mapping = object.map_into_range(vm, range, alignment, prot, true)?; + let iova = mapping.iova(); + Ok(Self { + mapping: Some(mapping), + object, + iova, + object_offset: 0, + logical_size: size, + cpu_wc: true, + }) + } + + pub(crate) fn new_wc_at( + dev: &AsahiDevice, + vm: &mmu::Vm, + address: u64, + size: usize, + prot: mmu::Prot, + ) -> Result { + let mut object = gem::new_kernel_object_wc(dev, size)?; + object.vmap()?.memset(0); + let mapping = object.map_at(vm, address, prot, false)?; + let iova = mapping.iova(); + Ok(Self { + mapping: Some(mapping), + object, + iova, + object_offset: 0, + logical_size: size, + cpu_wc: true, + }) + } + + /// Replace this object's backing with a larger one at the SAME address. + /// + /// The replacement is allocated and zeroed before the old aperture is + /// released, so an allocation failure leaves the caller's object intact + /// and mapped. Only the caller can know that nothing is reading the old + /// backing; this helper does not synchronize. + pub(crate) fn regrow_wc_at( + &mut self, + dev: &AsahiDevice, + vm: &mmu::Vm, + address: u64, + size: usize, + ) -> Result { + if size <= self.logical_size { + return Ok(()); + } + let mut object = gem::new_kernel_object_wc(dev, size)?; + object.vmap()?.memset(0); + self.mapping = None; + let mapping = object.map_at(vm, address, mmu::PROT_GPU_FW_SHARED_RW, false)?; + self.iova = mapping.iova(); + self.mapping = Some(mapping); + self.object = object; + self.object_offset = 0; + self.logical_size = size; + self.cpu_wc = true; + Ok(()) + } + + pub(crate) fn iova(&self) -> u64 { + self.iova + } + + pub(crate) fn with_bytes_mut(&mut self, f: impl FnOnce(&mut [u8]) -> Result) -> Result { + let vmap = self.object.vmap()?; + let bytes = unsafe { + // SAFETY: the VMap covers this object's complete, page-rounded + // allocation; construction bounds-checks the view offset and size. + core::slice::from_raw_parts_mut( + vmap.as_mut_ptr().add(self.object_offset), + self.logical_size, + ) + }; + f(bytes) + } + + pub(crate) fn read_u32(&mut self, address: u64) -> Result { + let offset = address.checked_sub(self.iova()).ok_or(EINVAL)? as usize; + let end = offset.checked_add(4).ok_or(EINVAL)?; + if end > self.logical_size { + return Err(EINVAL); + } + self.with_bytes_mut(|bytes| { + Ok(u32::from_le_bytes( + bytes[offset..end].try_into().map_err(|_| EINVAL)?, + )) + }) + } + + pub(crate) fn read_u64(&mut self, address: u64) -> Result { + let offset = address.checked_sub(self.iova()).ok_or(EINVAL)? as usize; + let end = offset.checked_add(8).ok_or(EINVAL)?; + if end > self.logical_size { + return Err(EINVAL); + } + self.with_bytes_mut(|bytes| { + Ok(u64::from_le_bytes( + bytes[offset..end].try_into().map_err(|_| EINVAL)?, + )) + }) + } + + pub(crate) fn write_u32(&mut self, address: u64, value: u32) -> Result { + let offset: usize = address.checked_sub(self.iova()).ok_or(EINVAL)?.try_into()?; + self.with_bytes_mut(|bytes| g17p_live_write_u32(bytes, offset, value)) + } + + pub(crate) fn map_alias_at( + &mut self, + vm: &mmu::Vm, + address: u64, + prot: mmu::Prot, + ) -> Result { + if self.object_offset != 0 { + return Err(EINVAL); + } + self.object.map_at(vm, address, prot, false) + } + + pub(crate) fn map_alias_in_range( + &mut self, + vm: &mmu::Vm, + range: Range, + alignment: u64, + prot: mmu::Prot, + ) -> Result { + if self.object_offset != 0 { + return Err(EINVAL); + } + self.object + .map_into_range(vm, range, alignment, prot, false) + } + +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/pool.rs b/drivers/gpu/drm/asahi/g18/buffer/pool.rs new file mode 100644 index 0000000000000..4db53eace3bb2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/pool.rs @@ -0,0 +1,708 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned initial G18 UMA pool resources. No firmware binding is published here. +//! +//! Flags 3 use a sparse directory: a page at each USC stride plus the final +//! page, with hardware addressing starting one KiB before the first page end. +//! Initial mappings reserve their trailing holes using the existing fixed +//! mapping guards. The fixed ideal-witness probe precommits native tail extents +//! before publication; runtime growth is not implemented. +//! The state must be fully encoded before a caller may bind or submit work. +//! A job's pool also needs its hardware UMA descriptor loaded into the table +//! slot before the tag15 that names the pool is published. + +#![allow(dead_code)] + +#[path = "pool_application.rs"] +mod application; +#[path = "pool_shared_compute.rs"] +mod shared_compute; +#[path = "pool_owner_probe.rs"] +mod owner_probe; + +use crate::g18::buffer::object::MappedObject; +use crate::{driver::AsahiDevice, g18::fw::uma as g18_uma, mmu}; +use core::ops::Range; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +// Host-selected addresses in the shared GPU view, disjoint from the descriptor +// and completion aliases. Actual extents are checked before allocation. +const DIRECTORY_BASE: u64 = 0x70_0200_0000; +const BLOCK_LIST_BASE: u64 = 0x70_0400_0000; +const BLOCK_DATA_BASE: u64 = 0x70_0500_0000; +const BLOCK_DATA_STRIDE: u64 = 0x20_0000; +const STATE_BYTES: usize = g18_uma::G18_POOL_STATE_BYTES; +/// Hardware UMA slot of this boot's single pool. Native +/// `AGXUSCPrivMemPool::prepareLocked` takes it from the accelerator's ID +/// manager (accelerator+0x13b98, 0x100 IDs, init 0xfffffe0008c458f4); the +/// first grant on a fresh accelerator is 0. The same ID is the tag15 +0x46 +/// firmware pool slot (descriptor+0x7a8) and the low half of register 0x10241. +pub(crate) const G18_UMA_HARDWARE_SLOT: u32 = 0; +/// prepareLocked rejects IDs >= 0x100 (0xfffffe0008c8f334): 0x8000 bytes. +const UMA_TABLE_SLOTS: usize = 0x100; + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18UmaPoolBinding { + pub(crate) state: u64, + pub(crate) generation: u64, + pub(crate) slot: u32, +} + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18UmaPoolView { + pub(crate) directory_base: u64, + pub(crate) reservation_bytes: u64, + pub(crate) stride: u64, + pub(crate) page_count: usize, + pub(crate) gpu_base_offset: u64, + pub(crate) state_high: u64, + pub(crate) block_list_high: u64, + pub(crate) block_list_low: u64, +} + +pub(crate) struct G18UmaPoolMemory { + // Drop the GPU aliases and their reservations before the CPU/FW mappings. + // Every mapping independently retains its GEM backing and owning VM. + // The job-context aliases go first: that VM's roots may still be live. + process_aliases: KVec, + aliases: KVec, + scratch_aliases: KVec, + scratch: KVec, + scratch_plan: Option, + directory: KVec, + directory_growth: KVec, + directory_growth_offsets: KVec, + shared_compute_plan: Option, + block_list: MappedObject, + state: MappedObject, + // Firmware dereferences state+0x88 during pool retirement. Keep the + // independently allocated counter alive for the entire pool lifetime. + preempt_counter: MappedObject, + geometry: g18_uma::G18UmaGeometry, + layout: g18_uma::G18UmaSparseDirectory, + encoded: bool, + descriptor_loaded: bool, + render: bool, + low_bias: u64, + hardware_slot: u32, + block_stride: u64, +} + +impl G18UmaPoolMemory { + /// Separate application compute pool. The four native gallery compute + /// captures use four two-MiB blocks; reserve their maximum observed result + /// pair/counter for the initial bounded, non-spilling dispatch path. + pub(crate) fn new_application_compute(dev: &AsahiDevice, uat: &mmu::Uat, + high_range: Range) -> Result { + let mut pool = Self::new_layout(dev, uat, high_range.clone(), 12, + true, false, 0x40000000, 1, 0x200000)?; + pool.prepare_compute_backing(dev, uat, high_range, g18_uma::G18ComputeScratchPlan { + block_bytes: 0x200000, block_count: 4, + register_pair: [30, 12], chunk_counters: [0, 260], + })?; + Ok(pool) + } + + pub(crate) fn compute_application_aliases(&mut self, dev: &AsahiDevice, + uat: &mmu::Uat, vm: &mmu::Vm) -> Result> { + if self.render || ![0x40000000, 0x2_4000_0000, 0x3_4000_0000].contains(&self.low_bias) + || !self.process_aliases.is_empty() { + return Err(EINVAL); + } + self.alias_into_process(dev, uat, vm)?; + Ok(core::mem::replace(&mut self.process_aliases, KVec::new())) + } + + /// The retained backing plan; application compute bounds its SMA pair by it. + pub(crate) fn scratch_plan(&self) -> Option { + self.scratch_plan + } + + pub(crate) fn shared_compute_plan(&self) -> Option { + self.shared_compute_plan + } + + pub(crate) fn new( + dev: &AsahiDevice, + uat: &mmu::Uat, + high_range: Range, + uscs: u32, + ) -> Result { + Self::new_with_guard(dev, uat, high_range, uscs, true, false) + } + + /// `guard` reserves one unmapped page after each firmware-view object. + /// The render pool packs its 16 objects into the fixed 0x40000 window + /// of g18_address_map (list/state/counter at +0x34000/+0x38000/+0x3c000), + /// which has no room for guards: with them the directory pages alone + /// exhaust the window (stage33 H1, ENOSPC). Compute keeps its guards. + fn new_with_guard( + dev: &AsahiDevice, + uat: &mmu::Uat, + high_range: Range, + uscs: u32, + guard: bool, + ideal: bool, + ) -> Result { + Self::new_layout(dev, uat, high_range, uscs, guard, ideal, 0, 0, 0x100000) + } + + fn new_layout(dev: &AsahiDevice, uat: &mmu::Uat, high_range: Range, + uscs: u32, guard: bool, ideal: bool, low_bias: u64, hardware_slot: u32, + block_bytes: u64) -> Result { + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 { + return Err(ENODEV); + } + let directory_base = DIRECTORY_BASE + low_bias; + let block_list_base = BLOCK_LIST_BASE + low_bias; + // A 1 MiB block is one native accessor's admitted size. This is the + // initial host policy, not a claim to reproduce every native override. + let geometry = g18_uma::g18_uma_geometry(0x8000_0000, block_bytes, + uscs, false, true, 14).ok_or(EINVAL)?; + let layout = g18_uma::g18_uma_sparse_directory(&geometry, uscs, 14).ok_or(EINVAL)?; + use crate::{g18::address_map as va, g18::uapi::render_policy as policy}; + if ideal && (guard || uscs != 12 || geometry.directory_bytes_per_usc != 0x158000 + || layout.initial_page_count != 13 || layout.page_bytes != 0x4000) { + return Err(EINVAL); + } + if directory_base.checked_add(layout.reservation_bytes).ok_or(EOVERFLOW)? > block_list_base { + return Err(EINVAL); + } + let fixed = |size| MappedObject::new_wc_in_range_with_guards(dev, uat.kernel_vm(), + high_range.clone(), size, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, + usize::from(guard)); + let mut directory = KVec::new(); + let mut directory_growth = KVec::new(); + let mut directory_growth_offsets = KVec::new(); + let mut aliases = KVec::new(); + for index in 0..layout.initial_page_count { + let offset = layout.page_offset(index).ok_or(EINVAL)?; + let end = if index + 1 == layout.initial_page_count { + layout.reservation_bytes + } else { + layout.page_offset(index + 1).ok_or(EINVAL)? + - if ideal { policy::IDEAL_GROWTH_BYTES } else { 0 } + }; + let guards: usize = ((end - offset - layout.page_bytes) / layout.page_bytes).try_into()?; + let mut page = fixed(mmu::UAT_PGSZ)?; + aliases.push(page.map_alias_at_with_guards(uat.kernel_lower_vm(), + directory_base + offset, mmu::PROT_GPU_SHARED_RW, guards)?, GFP_KERNEL)?; + if uat.kernel_vm().translate_iova(page.iova())? + != uat.kernel_lower_vm().translate_iova(directory_base + offset)? { + return Err(EFAULT); + } + directory.push(page, GFP_KERNEL)?; + } + if ideal { + // Original 8c8b8f8..8c8ba08 selects 21 tail pages per USC for + // 40 one-MiB blocks. Preserve the guarded middle of every stride. + for index in 0..12u64 { + let high = va::GRAPHICS_UMA_GROWTH + index * policy::IDEAL_GROWTH_BYTES; + let low = directory_base + index * geometry.directory_bytes_per_usc + + policy::IDEAL_GROWTH_OFFSET; + let mut extent = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, + policy::IDEAL_GROWTH_BYTES as usize, mmu::PROT_FW_SHARED_RW)?; + aliases.push(extent.map_alias_at(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..policy::IDEAL_GROWTH_BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset)? + != uat.kernel_lower_vm().translate_iova(low + offset)? { return Err(EFAULT); } + } + directory_growth.push(extent, GFP_KERNEL)?; + directory_growth_offsets.push(low - directory_base, GFP_KERNEL)?; + } + } + // Validate every PTE in the aperture, including the intentionally + // absent entries. Do this before a firmware owner can see any address. + let mut holes = 0usize; + for offset in (0..layout.reservation_bytes).step_by(mmu::UAT_PGSZ) { + let mapped = uat.kernel_lower_vm().translate_iova(directory_base + offset).is_ok(); + let expected = offset % geometry.directory_bytes_per_usc == 0 + || (ideal && offset < 12 * geometry.directory_bytes_per_usc + && offset % geometry.directory_bytes_per_usc >= policy::IDEAL_GROWTH_OFFSET); + if mapped != expected { return Err(EFAULT); } + if !mapped { holes += 1; } + } + // A missing PTE is not enough: the allocator must also reject a new + // owner inside both the first and last gap. On unexpected success, + // dropping the temporary alias unwinds it before returning an error. + if geometry.directory_bytes_per_usc > layout.page_bytes { + let last_hole = if ideal { 11 * geometry.directory_bytes_per_usc + layout.page_bytes } + else { layout.reservation_bytes - 2 * layout.page_bytes }; + for offset in [layout.page_bytes, last_hole] { + match directory[0].map_alias_at(uat.kernel_lower_vm(), + directory_base + offset, mmu::PROT_GPU_SHARED_RW) { + Err(error) if error == EBUSY || error == ENOSPC => {}, + Err(error) => return Err(error), + Ok(alias) => { drop(alias); return Err(EFAULT); }, + } + } + } + let block_bytes = usize::try_from(u64::from(geometry.block_count) * 4)?; + let block_bytes = block_bytes.checked_add(mmu::UAT_PGSZ - 1).ok_or(EOVERFLOW)? + & !(mmu::UAT_PGSZ - 1); + let mut block_list = fixed(block_bytes)?; + aliases.push(block_list.map_alias_at(uat.kernel_lower_vm(), block_list_base, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..block_bytes).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(block_list.iova() + offset as u64)? + != uat.kernel_lower_vm().translate_iova(block_list_base + offset as u64)? { + return Err(EFAULT); + } + } + let mut state = fixed(STATE_BYTES)?; + state.with_bytes_mut(|bytes| { + // Native initial hardware slot is unassigned. All other fields + // remain unpublished until the native pool-state encoder runs. + bytes[8..12].copy_from_slice(&u32::MAX.to_le_bytes()); + Ok(()) + })?; + // Native FList::init allocates eight bytes at 8c8d28c..8c8d298; + // serialize at 8c8c19c/1b0 publishes its GPU address/current value. + // A null address crashes b000 at 1e5c4 (stage28); zero is a valid + // initial value of a backed counter, never a valid counter address. + let mut preempt_counter = fixed(8)?; + preempt_counter.with_bytes_mut(|bytes| { + if bytes.len() != 8 || bytes.iter().any(|&value| value != 0) { + return Err(EINVAL); + } + Ok(()) + })?; + uat.kernel_vm().translate_iova(preempt_counter.iova())?; + crate::g18_kick_info!(dev.as_ref(), + "G18 UMA retirement counter: owned FW address={:#x} initial=0; unpublished\n", + preempt_counter.iova()); + crate::g18_kick_info!(dev.as_ref(), + "G18 UMA sparse owner: reserved={:#x} backing_pages={} holes={} bias={:#x} aliases=verified guards=verified unpublished\n", + layout.reservation_bytes, layout.initial_page_count, holes, layout.gpu_base_offset); + Ok(Self { + process_aliases: KVec::new(), + aliases, scratch_aliases: KVec::new(), scratch: KVec::new(), scratch_plan: None, + directory, directory_growth, block_list, state, preempt_counter, geometry, layout, encoded: false, + directory_growth_offsets, shared_compute_plan: None, + descriptor_loaded: false, + render: false, low_bias, hardware_slot, + block_stride: if low_bias == 0 { BLOCK_DATA_STRIDE } else { 0x300000 }, + }) + } + + /// Separate deterministic owner for H1. The firmware-memory constructor + /// selects this INSTEAD OF the compute pool; their low windows cannot + /// coexist. All compute constructor and backing policy remain unchanged. + pub(crate) fn new_render(dev: &AsahiDevice, uat: &mmu::Uat) -> Result { + use crate::g18::address_map as va; + let mut owner = Self::new_with_guard(dev, uat, + va::GRAPHICS_UMA_HIGH..va::GRAPHICS_UMA_HIGH + 0x40000, 12, false, + crate::g18::uapi::render_policy::ideal_enabled())?; + if owner.state.iova() != va::GRAPHICS_UMA_STATE + || owner.block_list.iova() != va::GRAPHICS_UMA_LIST + || owner.preempt_counter.iova() != va::GRAPHICS_UMA_COUNTER { return Err(EFAULT); } + owner.render = true; + Ok(owner) + } + + pub(crate) fn prepare_render_backing(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + vm: &mmu::Vm) -> Result { + use crate::{g18::address_map as va, g18::uapi::render_policy as policy}; + let blocks = policy::uma_blocks(); + let directory_ready = if policy::ideal_enabled() { + blocks == 40 && self.directory_growth.len() == 12 + && self.directory_growth.iter().all(|extent| extent.logical_size == policy::IDEAL_GROWTH_BYTES as usize) + && self.geometry.directory_bytes_per_usc == 0x158000 + } else { + self.directory_growth.is_empty() + && g18_uma::g18_initial_directory_fits(blocks, 0x100000, 12, 3, 14) == Some(true) + }; + if !self.render || self.encoded || self.scratch_plan.is_some() || !directory_ready { + return Err(EINVAL); + } + let mut entries = KVec::new(); + for index in 0..blocks { + let low = policy::block_base() + u64::from(index) * BLOCK_DATA_STRIDE; + let high = va::GRAPHICS_UMA_BACKING + u64::from(index) * 0x100000; + let mut block = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, 0x100000, + mmu::PROT_FW_SHARED_RW)?; + let alias = block.map_alias_at_with_guards(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW, 1)?; + for offset in (0..0x100000).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset)? + != uat.kernel_lower_vm().translate_iova(low + offset)? { return Err(EFAULT); } + } + entries.push(g18_uma::g18_uma_block_entry(low, 0).ok_or(EINVAL)?, GFP_KERNEL)?; + self.scratch_aliases.push(alias, GFP_KERNEL)?; + self.scratch.push(block, GFP_KERNEL)?; + } + self.block_list.with_bytes_mut(|out| { + for (index, value) in entries.iter().enumerate() { + out[index*4..index*4+4].copy_from_slice(&value.to_le_bytes()); + } + Ok(()) + })?; + self.scratch_plan = Some(g18_uma::G18ComputeScratchPlan { + block_bytes: 0x100000, block_count: blocks, + register_pair: policy::resource_pairs()[0], chunk_counters: policy::uma_chunks(), + }); + self.alias_into_process(dev, uat, vm)?; + dev_info!(dev.as_ref(), "G18_RENDER_UMA backing={}MiB chunks={:?} resource_pairs={:?} ideal={} growth_extents={}; mapped in job context\n", + blocks, policy::uma_chunks(), policy::resource_pairs(), policy::ideal_enabled(), self.directory_growth.len()); + Ok(()) + } + + /// Retain the exact addition fixture's initial backing before publication. + /// A failed allocation/alias check leaves the prior empty pool untouched. + /// No firmware command, process root or hardware pool binding is published. + pub(crate) fn prepare_compute_backing( + &mut self, dev: &AsahiDevice, uat: &mmu::Uat, high_range: Range, + plan: g18_uma::G18ComputeScratchPlan, + ) -> Result<()> { + if self.encoded || self.scratch_plan.is_some() { return Err(EBUSY); } + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 + || plan.block_bytes != self.geometry.block_bytes + || !((self.low_bias == 0 && plan.block_bytes == 0x100000 + && plan.block_count == 5 && plan.register_pair == [16, 6] + && plan.chunk_counters == [0, 350]) + || (self.low_bias == 0x40000000 && plan.block_bytes == 0x200000 + && plan.block_count == 4 && plan.register_pair == [30, 12] + && plan.chunk_counters == [0, 260])) + || self.geometry.pool_bytes != 0x8000_0000 + || self.geometry.directory_bytes != self.geometry.directory_bytes_per_usc * 12 + || g18_uma::g18_initial_directory_fits(plan.block_count, plan.block_bytes, 12, 3, 14) + != Some(true) { + return Err(EINVAL); + } + let mut backing = KVec::new(); + let mut aliases = KVec::new(); + let mut entries = KVec::new(); + for index in 0..plan.block_count { + let gpu = BLOCK_DATA_BASE + self.low_bias + u64::from(index) * self.block_stride; + entries.push(g18_uma::g18_uma_block_entry(gpu, 0).ok_or(EINVAL)?, GFP_KERNEL)?; + let mut block = MappedObject::new_wc_in_range( + dev, uat.kernel_vm(), high_range.clone(), plan.block_bytes as usize, + mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, + )?; + // GPU-address alignment, not physical contiguity: the native list + // stores GPU virtual addresses and the UAT retains each GEM page. + let alias = block.map_alias_at_with_guards( + uat.kernel_lower_vm(), gpu, mmu::PROT_GPU_SHARED_RW, 1, + )?; + for offset in (0..plan.block_bytes).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(block.iova() + offset)? + != uat.kernel_lower_vm().translate_iova(gpu + offset)? { + return Err(EFAULT); + } + } + if uat.kernel_lower_vm().translate_iova(gpu + plan.block_bytes).is_ok() { + return Err(EFAULT); + } + aliases.push(alias, GFP_KERNEL)?; + backing.push(block, GFP_KERNEL)?; + } + // All allocations and translations are validated before the first + // publication data changes. Nothing fallible follows this write. + self.block_list.with_bytes_mut(|bytes| { + if bytes.len() < entries.len() * 4 || bytes.iter().any(|&v| v != 0) { + return Err(EBUSY); + } + for (index, entry) in entries.iter().enumerate() { + bytes[index * 4..index * 4 + 4].copy_from_slice(&entry.to_le_bytes()); + } + Ok(()) + })?; + self.scratch_aliases = aliases; + self.scratch = backing; + self.scratch_plan = Some(plan); + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), + "G18 compute scratch: bytes={} blocks={} pair={:?}; retained translations verified unpublished\n", + plan.block_bytes * u64::from(plan.block_count), plan.block_count, plan.register_pair); + Ok(()) + } + + /// Alias the backed directory pages, block list and scratch blocks into the + /// job's VM at their unchanged GPU addresses. The USC walks the descriptor's + /// directory/block-list/block pointers in the job's context: stage27 took a + /// GMMU fault (vm_slot 3, Unmapped read of the block list 0x7004000000) + /// while these existed only in the kernel lower VM. Holes stay unmapped. + /// Call after the backing exists and before the job's roots are published. + pub(crate) fn alias_into_process( + &mut self, dev: &AsahiDevice, uat: &mmu::Uat, vm: &mmu::Vm, + ) -> Result<()> { + let plan = self.scratch_plan.ok_or(EINVAL)?; + if !self.process_aliases.is_empty() { return Err(EBUSY); } + let block_list_bytes = (u64::from(self.geometry.block_count) * 4) + .checked_add(mmu::UAT_PGSZ as u64 - 1).ok_or(EOVERFLOW)? & !(mmu::UAT_PGSZ as u64 - 1); + let mut aliases = KVec::new(); + let mut ranges = KVec::new(); + for (index, page) in self.directory.iter_mut().enumerate() { + let gpu = DIRECTORY_BASE + self.low_bias + self.layout.page_offset(index as u32).ok_or(EINVAL)?; + let guards = if let Some(shared) = self.shared_compute_plan { + let commitment = crate::g18::uapi::compute_plan::G18ComputeDirectory::for_blocks( + shared.ideal.block_count).ok_or(EINVAL)?; + if index < 12 { + (self.geometry.directory_bytes_per_usc / self.layout.page_bytes) as usize + - commitment.tail_pages_per_usc as usize - 1 + } else { 0 } + } else { 0 }; + aliases.push(page.map_alias_at_with_guards(vm, gpu, mmu::PROT_GPU_SHARED_RW, guards)?, GFP_KERNEL)?; + ranges.push((gpu, mmu::UAT_PGSZ as u64), GFP_KERNEL)?; + } + for (index, extent) in self.directory_growth.iter_mut().enumerate() { + let gpu = DIRECTORY_BASE + self.low_bias + + *self.directory_growth_offsets.get(index).ok_or(EINVAL)?; + aliases.push(extent.map_alias_at(vm, gpu, mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + ranges.push((gpu, extent.logical_size as u64), GFP_KERNEL)?; + } + aliases.push(self.block_list.map_alias_at(vm, BLOCK_LIST_BASE + self.low_bias, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + ranges.push((BLOCK_LIST_BASE + self.low_bias, block_list_bytes), GFP_KERNEL)?; + let block_base = if self.render { crate::g18::uapi::render_policy::block_base() } else { BLOCK_DATA_BASE + self.low_bias }; + for (index, block) in self.scratch.iter_mut().enumerate() { + let gpu = block_base + index as u64 * self.block_stride; + aliases.push(block.map_alias_at_with_guards(vm, gpu, mmu::PROT_GPU_SHARED_RW, + usize::from(self.shared_compute_plan.is_some()))?, GFP_KERNEL)?; + ranges.push((gpu, plan.block_bytes), GFP_KERNEL)?; + } + // Every job-context page must reach the same memory as the kernel + // lower VM view that the descriptor and block list were encoded from. + let mut pages = 0usize; + for &(gpu, bytes) in ranges.iter() { + for offset in (0..bytes).step_by(mmu::UAT_PGSZ) { + if vm.translate_iova(gpu + offset)? + != uat.kernel_lower_vm().translate_iova(gpu + offset)? { + return Err(EFAULT); + } + pages += 1; + } + } + fence(Ordering::SeqCst); + self.process_aliases = aliases; + crate::g18_kick_info!(dev.as_ref(), + "G18 UMA pool aliased into the job VM: {} mappings, {} pages verified (directory {:#x}, block list {:#x}, blocks {:#x}+{}x{:#x})\n", + ranges.len(), pages, DIRECTORY_BASE + self.low_bias, BLOCK_LIST_BASE + self.low_bias, block_base, + self.scratch.len(), self.block_stride); + Ok(()) + } + + /// Prepare this boot's single compute pool, empty unless backing was + /// explicitly prepared first. Ownership must survive firmware access. + /// Runtime pool growth is not implemented. + pub(crate) fn prepare_binding(&mut self) -> Result { + if self.encoded { return Err(EBUSY); } + let counter_address = self.preempt_counter.iova(); + if counter_address == 0 || counter_address & 7 != 0 { return Err(EINVAL); } + let counter = self.preempt_counter.with_bytes_mut(|bytes| { + if bytes.len() != 8 { return Err(EINVAL); } + let value = u64::from_le_bytes(bytes.try_into().map_err(|_| EINVAL)?); + if value != 0 { return Err(EBUSY); } + Ok(value) + })?; + let encoded = if let Some(plan) = self.shared_compute_plan { + plan.pool_state(1, DIRECTORY_BASE + self.low_bias, + BLOCK_LIST_BASE + self.low_bias, counter_address, counter).ok_or(EINVAL)? + } else { g18_uma::encode_g18_pool_state(&g18_uma::G18UmaPoolState { + // Host identities are local to this one fixed-VA pool and boot. + // The statistics ID and FList generation have different meanings. + statistics_id: 1, + // Native halUMAGetGrowType returns 2; implicit engine init also + // stores 2. This records the protocol, not an implemented grow path. + grow_engine: 2, + priority: 2, + directory_gpu: DIRECTORY_BASE + self.low_bias + self.layout.gpu_base_offset, + directory_stride: self.geometry.directory_bytes_per_usc, + chunk_counters: self.scratch_plan.map_or([0, 0], |plan| plan.chunk_counters), + block_list_gpu: BLOCK_LIST_BASE + self.low_bias, + populated_blocks: self.scratch_plan.map_or(0, |plan| plan.block_count), + block_units_64k: (self.geometry.block_bytes >> 16).try_into()?, + // Native base configureDevice initializes this mode to zero. + sma_mode: 0, + preempt_counter_gpu: counter_address, + preempt_counter: counter, + compute: !self.render, + // Fresh Pool::init clears +0x1f0; prepareBufferResources writes 1. + policy_value: 1, + }) }; + self.state.with_bytes_mut(|bytes| { + if bytes.iter().enumerate().any(|(i, &v)| v != if (8..12).contains(&i) { 0xff } else { 0 }) { + return Err(EBUSY); + } + bytes.copy_from_slice(&encoded); + Ok(()) + })?; + fence(Ordering::SeqCst); + self.encoded = true; + Ok(G18UmaPoolBinding { state: self.state.iova(), generation: 0, slot: self.hardware_slot }) + } + + /// Native `prepareLocked` 0xfffffe0008c8f308..0x8c8f390 on a pool's first + /// use: store the hardware slot at state+0x08 (only when it differs), then + /// `halLoadUMADesc` packs state[..0x7c] with the four SMA words into + /// table[slot]. Plain host stores; native has no barrier or cache + /// maintenance here. The submit's tag15 names the pool only afterwards, so + /// call this before the binding is published. Refuses an occupied slot, a + /// foreign binding or a second load. + pub(crate) fn load_hardware_descriptor( + &mut self, + dev: &AsahiDevice, + table: &mut MappedObject, + binding: G18UmaPoolBinding, + device_config: [u64; 4], + ) -> Result<[u64; g18_uma::G18_UMA_DESCRIPTOR_BYTES / 8]> { + if binding.generation != 0 { return Err(EINVAL); } + self.write_hardware_descriptor(dev, table, binding, device_config, [0; 16], false) + } + + /// Requires the caller's retired scheduler and old firmware binding check. + /// This only replaces our own still-retained descriptor; it cannot reclaim + /// an arbitrary occupied slot or reset an already encoded view in place. + pub(crate) fn replace_hardware_descriptor(&mut self, dev: &AsahiDevice, + table: &mut MappedObject, binding: G18UmaPoolBinding, + previous: &mut Self, old: G18UmaPoolBinding, device_config: [u64; 4]) + -> Result<[u64; 16]> + { + if !previous.descriptor_loaded || previous.render || self.render + || old.state != previous.state.iova() || old.slot != previous.hardware_slot + || binding.slot != old.slot || binding.state == old.state + || old.generation.checked_add(1) != Some(binding.generation) + || binding.generation == u64::MAX { return Err(EINVAL); } + let state = previous.render_state()?; + if state[8..12] != old.slot.to_le_bytes() { return Err(EIO); } + let packed: [u8; g18_uma::G18_UMA_STATE_BYTES] = + state[..g18_uma::G18_UMA_STATE_BYTES].try_into().map_err(|_| EINVAL)?; + let expected = g18_uma::pack_g18_uma_descriptor(&packed, device_config); + self.write_hardware_descriptor(dev, table, binding, device_config, expected, true) + } + + fn write_hardware_descriptor(&mut self, dev: &AsahiDevice, table: &mut MappedObject, + binding: G18UmaPoolBinding, device_config: [u64; 4], expected: [u64; 16], + retired_view: bool) + -> Result<[u64; 16]> + { + const DESCRIPTOR: usize = g18_uma::G18_UMA_DESCRIPTOR_BYTES; + if !self.encoded || self.descriptor_loaded { + dev_err!(dev.as_ref(), "G18_POOL_DESCRIPTOR_REFUSED owner encoded={} loaded={}\n", + self.encoded, self.descriptor_loaded); + return Err(EBUSY); + } + if binding.state != self.state.iova() || binding.slot != self.hardware_slot + || binding.generation == u64::MAX || device_config == [0; 4] { + return Err(EINVAL); + } + let slot = binding.slot as usize; + if slot >= UMA_TABLE_SLOTS { return Err(EINVAL); } + let mut packed = [0u8; g18_uma::G18_UMA_STATE_BYTES]; + self.state.with_bytes_mut(|bytes| { + // The encoded state must still carry the native unassigned slot. + if bytes.len() < STATE_BYTES || bytes[8..12] != u32::MAX.to_le_bytes() { + dev_err!(dev.as_ref(), "G18_POOL_DESCRIPTOR_REFUSED candidate slot is not unassigned\n"); + return Err(EBUSY); + } + packed.copy_from_slice(&bytes[..g18_uma::G18_UMA_STATE_BYTES]); + Ok(()) + })?; + packed[8..12].copy_from_slice(&binding.slot.to_le_bytes()); + let words = g18_uma::pack_g18_uma_descriptor(&packed, device_config); + let offset = slot * DESCRIPTOR; + table.with_bytes_mut(|out| { + if out.len() != UMA_TABLE_SLOTS * DESCRIPTOR { + return Err(EBUSY); + } + for (index, word) in expected.iter().enumerate() { + let at = offset + index * 8; + let actual = u64::from_le_bytes(out[at..at + 8].try_into().map_err(|_| EINVAL)?); + if actual == *word { continue; } + // Native FList prepareBufferResources resets these cursor + // fields separately from immutable identity/geometry: + // state+0x24..0x43 and +0x5c..0x7b. They may have advanced + // in the used descriptor. A retired replacement has its own + // fresh cursors; the first-load check still admits only zero. + if retired_view && matches!(index, 1 | 2 | 7 | 8 | 12..=15) { + crate::g18_kick_info!(dev.as_ref(), + "G18_POOL_DESCRIPTOR_CURSOR slot={} word={} host={:#x} retired={:#x}\n", + binding.slot, index, word, actual); + } else { + dev_err!(dev.as_ref(), + "G18_POOL_DESCRIPTOR_REFUSED slot={} word={} expected={:#x} actual={:#x} retired_view={}\n", + binding.slot, index, word, actual, retired_view); + return Err(EBUSY); + } + } + Ok(()) + })?; + // All checks passed. A later failure still aborts before tag15. + // Native order: state+0x08 first, then the sixteen descriptor words. + self.state.with_bytes_mut(|bytes| { + bytes[8..12].copy_from_slice(&binding.slot.to_le_bytes()); + Ok(()) + })?; + table.with_bytes_mut(|out| { + for (index, word) in words.iter().enumerate() { + let at = offset + index * 8; + out[at..at + 8].copy_from_slice(&word.to_le_bytes()); + } + Ok(()) + })?; + fence(Ordering::SeqCst); + self.descriptor_loaded = true; + crate::g18_kick_info!(dev.as_ref(), + "G18 UMA descriptor loaded: slot={} entry={:#x} state={:#x} state+0x08={} sma={:x?} words={:x?}; tag15 unpublished\n", + binding.slot, table.iova() + offset as u64, binding.state, binding.slot, + device_config, words); + Ok(words) + } + + pub(crate) fn directory_page_high(&self, index: usize) -> Option { + self.directory.get(index).map(MappedObject::iova) + } + + pub(crate) fn verify_render_descriptor(&mut self, table: &mut MappedObject, + device_config: [u64; 4]) -> Result<[u64; 16]> { + if !self.render || !self.descriptor_loaded { return Err(EINVAL); } + let state: [u8; g18_uma::G18_UMA_STATE_BYTES] = self.state.with_bytes_mut(|out| + out[..g18_uma::G18_UMA_STATE_BYTES].try_into().map_err(|_| EINVAL))?; + if u32::from_le_bytes(state[8..12].try_into().map_err(|_| EINVAL)?) != G18_UMA_HARDWARE_SLOT { return Err(EIO); } + let expected = g18_uma::pack_g18_uma_descriptor(&state, device_config); + let mut actual = [0u64; 16]; + for (i, value) in actual.iter_mut().enumerate() { *value = table.read_u64(table.iova() + i as u64 * 8)?; } + if actual != expected || actual == [0; 16] { return Err(EIO); } + Ok(actual) + } + + pub(crate) fn render_state(&mut self) -> Result<[u8; STATE_BYTES]> { + self.state.with_bytes_mut(|out| out.try_into().map_err(|_| EINVAL)) + } + + pub(crate) fn audit_render_mappings(&self, uat: &mmu::Uat, vm: &mmu::Vm) -> Result { + if !self.render || self.scratch.len() != crate::g18::uapi::render_policy::uma_blocks() as usize { return Err(EINVAL); } + let mut pages = 0; + let mut audit = |high: u64, low: u64, bytes: usize| -> Result { + for offset in (0..bytes).step_by(mmu::UAT_PGSZ) { + let pa = uat.kernel_vm().translate_iova(high + offset as u64)?; + if vm.translate_iova(low + offset as u64)? != pa + || uat.kernel_lower_vm().translate_iova(low + offset as u64)? != pa { return Err(EFAULT); } + pages += 1; + } + Ok(()) + }; + for (i, page) in self.directory.iter().enumerate() { + audit(page.iova(), DIRECTORY_BASE + self.low_bias + self.layout.page_offset(i as u32).ok_or(EINVAL)?, mmu::UAT_PGSZ)?; + } + for (i, extent) in self.directory_growth.iter().enumerate() { + audit(extent.iova(), DIRECTORY_BASE + i as u64 * self.geometry.directory_bytes_per_usc + + crate::g18::uapi::render_policy::IDEAL_GROWTH_OFFSET, extent.logical_size)?; + } + audit(self.block_list.iova(), BLOCK_LIST_BASE, self.block_list.logical_size)?; + for (i, block) in self.scratch.iter().enumerate() { + audit(block.iova(), crate::g18::uapi::render_policy::block_base() + i as u64 * BLOCK_DATA_STRIDE, block.logical_size)?; + } + Ok(pages) + } + + pub(crate) fn view(&self) -> G18UmaPoolView { + G18UmaPoolView { + directory_base: DIRECTORY_BASE, reservation_bytes: self.layout.reservation_bytes, + stride: self.geometry.directory_bytes_per_usc, page_count: self.directory.len(), + gpu_base_offset: self.layout.gpu_base_offset, state_high: self.state.iova(), + block_list_high: self.block_list.iova(), block_list_low: BLOCK_LIST_BASE, + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/pool_application.rs b/drivers/gpu/drm/asahi/g18/buffer/pool_application.rs new file mode 100644 index 0000000000000..3e1e012966a0c --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/pool_application.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +use super::*; +impl G18UmaPoolMemory { + pub(crate) fn verify_application_descriptor(&mut self, table: &mut MappedObject, + device_config: [u64;4], first: bool) -> Result<[u64;16]> { + if first {return self.verify_render_descriptor(table,device_config);} + if !self.render || !self.descriptor_loaded {return Err(EINVAL);} + let state: [u8;g18_uma::G18_UMA_STATE_BYTES]=self.state.with_bytes_mut(|out| + out[..g18_uma::G18_UMA_STATE_BYTES].try_into().map_err(|_|EINVAL))?; + let expected=g18_uma::pack_g18_uma_descriptor(&state,device_config); + let mut actual=[0;16]; + for (i,v) in actual.iter_mut().enumerate() {*v=table.read_u64(table.iova()+i as u64*8)?;} + // Directory address/stride, device masks, modes, block-list address + // and block geometry stay fixed. Allocation cursors/counters remain + // hardware-owned; never restore the initial row over their progress. + for i in [0,3,4,5,6,10] { + if actual[i]!=expected[i] {return Err(EIO);} + } + if actual[11]>u64::from(crate::g18::uapi::render_policy::UMA_BLOCKS) {return Err(EIO);} + Ok(actual) + } + + pub(crate) fn application_aliases(&mut self, uat: &mmu::Uat, vm: &mmu::Vm) + -> Result> { + if !self.render || self.scratch_plan.is_none() {return Err(EINVAL);} + let mut aliases=KVec::new(); + for (index,page) in self.directory.iter_mut().enumerate() { + let address=DIRECTORY_BASE+self.layout.page_offset(index as u32).ok_or(EINVAL)?; + aliases.push(page.map_alias_at(vm,address,mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?; + } + aliases.push(self.block_list.map_alias_at(vm,BLOCK_LIST_BASE,mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?; + for (index,block) in self.scratch.iter_mut().enumerate() { + aliases.push(block.map_alias_at(vm,BLOCK_DATA_BASE+index as u64*BLOCK_DATA_STRIDE, + mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?; + } + self.audit_render_mappings(uat,vm)?;Ok(aliases) + } +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/pool_owner_probe.rs b/drivers/gpu/drm/asahi/g18/buffer/pool_owner_probe.rs new file mode 100644 index 0000000000000..3b33415bd6ce7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/pool_owner_probe.rs @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Opt-in allocation E2E on the real UAT, before firmware publication. +//! This never submits GPU work or claims shader/CTS qualification. + +use super::*; +use crate::g18::uapi::compute_plan::{G18ComputeDirectory, G18ComputePlan, G18_COMPUTE_BLOCK_BYTES}; + +struct Snapshot { + translations: KVec<(u64, u64, u64)>, + list: KVec, + state: [u8; STATE_BYTES], + plan: Option, +} + +impl G18UmaPoolMemory { + fn owner_snapshot(&mut self, uat: &mmu::Uat) -> Result { + let plan = self.shared_compute_plan; + let blocks = self.scratch_plan.ok_or(EINVAL)?.block_count; + let directory = G18ComputeDirectory::for_blocks(blocks).ok_or(EINVAL)?; + let mut translations = KVec::new(); + let mut capture = |high: u64, low: u64, size: usize| -> Result { + for offset in (0..size).step_by(mmu::UAT_PGSZ) { + let high = high + offset as u64; + let low = low + offset as u64; + let physical = uat.kernel_vm().translate_iova(high)?; + if uat.kernel_lower_vm().translate_iova(low)? != physical { return Err(EFAULT); } + translations.push((high, low, physical as u64), GFP_KERNEL)?; + } + Ok(()) + }; + let base = DIRECTORY_BASE + self.low_bias; + for (index, page) in self.directory.iter().enumerate() { + capture(page.iova(), base + self.layout.page_offset(index as u32).ok_or(EINVAL)?, + mmu::UAT_PGSZ)?; + } + for (extent, offset) in self.directory_growth.iter().zip(self.directory_growth_offsets.iter()) { + capture(extent.iova(), base + offset, extent.logical_size)?; + } + capture(self.block_list.iova(), BLOCK_LIST_BASE + self.low_bias, self.block_list.logical_size)?; + for (index, block) in self.scratch.iter().enumerate() { + capture(block.iova(), BLOCK_DATA_BASE + self.low_bias + index as u64 * self.block_stride, + block.logical_size)?; + } + let tail_bytes = u64::from(directory.tail_pages_per_usc) * self.layout.page_bytes; + for offset in (0..directory.reservation_bytes()).step_by(mmu::UAT_PGSZ) { + let in_stride = offset % self.geometry.directory_bytes_per_usc; + let expected = in_stride == 0 || (offset < 12 * self.geometry.directory_bytes_per_usc + && in_stride >= self.geometry.directory_bytes_per_usc - tail_bytes); + if uat.kernel_lower_vm().translate_iova(base + offset).is_ok() != expected { + return Err(EFAULT); + } + } + // The sparse holes must still belong to the allocator, not merely + // have absent PTEs. Probe both ends of the aperture when holes exist. + if directory.tail_pages_per_usc < 85 { + for offset in [self.layout.page_bytes, + 11 * self.geometry.directory_bytes_per_usc + self.layout.page_bytes] { + match self.directory[0].map_alias_at(uat.kernel_lower_vm(), base + offset, + mmu::PROT_GPU_SHARED_RW) { + Err(error) if error == EBUSY || error == ENOSPC => {}, + _ => return Err(EFAULT), + } + } + } + let list = self.block_list.with_bytes_mut(|bytes| { + let mut copy = KVec::with_capacity(bytes.len(), GFP_KERNEL)?; + for &byte in bytes.iter() { copy.push(byte, GFP_KERNEL)?; } + Ok(copy) + })?; + Ok(Snapshot { translations, list, state: self.render_state()?, plan }) + } + + fn unchanged(&mut self, uat: &mmu::Uat, before: &Snapshot) -> Result { + let after = self.owner_snapshot(uat)?; + if before.translations[..] != after.translations[..] || before.list[..] != after.list[..] + || before.state != after.state || before.plan != after.plan { return Err(EFAULT); } + Ok(()) + } + + /// E2E injection at the last allocation boundary: a complete replacement + /// is discarded before descriptor publication. Compare every old mapping, + /// state byte and list entry, then let the real preparation reuse the bank. + pub(crate) fn verify_replacement_rollback(&mut self, dev: &AsahiDevice, + uat: &mmu::Uat, plan: G18ComputePlan) -> Result { + use crate::g18::address_map as va; + let before = self.owner_snapshot(uat)?; + let mut staged = self.replacement_shared_compute(dev, uat, + va::DYNAMIC_HIGH_START..va::DYNAMIC_HIGH_END, plan)?; + let _binding = staged.prepare_binding()?; + let staged_before = staged.owner_snapshot(uat)?; + let bias = staged.low_bias; + let retained = self.scratch.len().min(plan.ideal.block_count as usize); + drop(staged); + self.unchanged(uat, &before)?; + for &(high, low, _) in staged_before.translations.iter() { + if uat.kernel_vm().translate_iova(high).is_ok() + || uat.kernel_lower_vm().translate_iova(low).is_ok() { return Err(EFAULT); } + } + dev_info!(dev.as_ref(), + "G18_COMPUTE_POOL_ROLLBACK old_pages={} staged_pages={} retained_blocks={} released_bias={:#x} state=list=translations=unchanged unpublished=1\n", + before.translations.len(), staged_before.translations.len(), retained, bias); + Ok(()) + } + + #[inline(never)] + pub(crate) fn verify_shared_compute_owner(dev: &AsahiDevice, uat: &mmu::Uat) -> Result { + use crate::g18::address_map as va; + let high = va::DYNAMIC_HIGH_START..va::DYNAMIC_HIGH_END; + let mut words = [48,12,0,0,2304,24,0,0,2,1,0,0,0,0,0,0,1,0,0,128,4,0,0]; + let inverse = G18ComputePlan::from_descriptor(&words).ok_or(EINVAL)?; + words[4] = 48; + let small = G18ComputePlan::from_descriptor(&words).ok_or(EINVAL)?; + if inverse.ideal.bytes != 30 * 1024 * 1024 || inverse.ideal.block_count != 15 + || small.ideal.block_count != 2 { return Err(EINVAL); } + dev_info!(dev.as_ref(), "G18_COMPUTE_OWNER_E2E begin unpublished=1 gpu_kicks=0\n"); + let mut pool = Self::new_shared_compute(dev, uat, high.clone(), small)?; + let before = pool.owner_snapshot(uat)?; + // Refuse an actual collision after five additional blocks are staged. + let blocked_low = BLOCK_DATA_BASE + pool.low_bias + 7 * pool.block_stride; + let blocker = pool.directory[0].map_alias_at(uat.kernel_lower_vm(), blocked_low, + mmu::PROT_GPU_SHARED_RW)?; + match pool.prepare_shared_compute(dev, uat, high.clone(), inverse) { + Err(error) if error == EBUSY || error == ENOSPC => {}, + _ => return Err(EFAULT), + } + pool.unchanged(uat, &before)?; + drop(blocker); + for index in 2..15 { + if uat.kernel_lower_vm().translate_iova(BLOCK_DATA_BASE + pool.low_bias + + index * pool.block_stride).is_ok() { return Err(EFAULT); } + } + dev_info!(dev.as_ref(), "G18_COMPUTE_OWNER_E2E scratch_collision=pass previous_pages={}\n", + before.translations.len()); + + // In a fresh 64-MiB subarena, leave exactly thirteen guarded blocks + // and two guarded directory extents free. Third extent allocation + // must fail after two real guard splits, causing their rollback. + let constrained = high.end - 64 * 1024 * 1024..high.end; + let page = mmu::UAT_PGSZ as u64; + let free = 13 * (G18_COMPUTE_BLOCK_BYTES + page) + 2 * (21 * page + page); + let cut = constrained.start + free; + let guard_pages = ((constrained.end - cut) / page - 1) as usize; + let blocker = MappedObject::new_wc_in_range_with_guards(dev, uat.kernel_vm(), + cut..constrained.end, mmu::UAT_PGSZ, page, mmu::PROT_FW_SHARED_RW, guard_pages)?; + if blocker.iova() != cut { return Err(EFAULT); } + match pool.prepare_shared_compute(dev, uat, constrained.clone(), inverse) { + Err(error) if error == ENOSPC => {}, + _ => return Err(EFAULT), + } + pool.unchanged(uat, &before)?; + for at in (constrained.start..cut).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(at).is_ok() { return Err(EFAULT); } + } + drop(blocker); + dev_info!(dev.as_ref(), "G18_COMPUTE_OWNER_E2E directory_allocation_rollback=pass\n"); + + pool.prepare_shared_compute(dev, uat, high.clone(), inverse)?; + let grown = pool.owner_snapshot(uat)?; + for &(high, low, physical) in before.translations.iter() { + if uat.kernel_vm().translate_iova(high)? as u64 != physical + || uat.kernel_lower_vm().translate_iova(low)? as u64 != physical { return Err(EFAULT); } + } + let vm = uat.new_vm(0x267, 0x70_4000_0000..0x70_6000_0000)?.with_t8152_lifetime()?; + pool.alias_into_process(dev, uat, &vm)?; + for &(_, low, physical) in grown.translations.iter() { + if vm.translate_iova(low)? as u64 != physical { return Err(EFAULT); } + } + for usc in [0,11] { + let hole = DIRECTORY_BASE + pool.low_bias + usc * pool.geometry.directory_bytes_per_usc + page; + if vm.translate_iova(hole).is_ok() { return Err(EFAULT); } + match pool.directory[0].map_alias_at(&vm, hole, mmu::PROT_GPU_SHARED_RW) { + Err(error) if error == EBUSY || error == ENOSPC => {}, + _ => return Err(EFAULT), + } + } + // Publication state has the exact pure serializer output, and once + // encoded the allocation path must refuse further growth/reset. + let expected = inverse.pool_state(1, DIRECTORY_BASE + pool.low_bias, + BLOCK_LIST_BASE + pool.low_bias, pool.preempt_counter.iova(), 0).ok_or(EINVAL)?; + pool.prepare_binding()?; + if pool.render_state()? != expected { return Err(EFAULT); } + if pool.prepare_shared_compute(dev, uat, high.clone(), inverse) != Err(EBUSY) { + return Err(EFAULT); + } + dev_info!(dev.as_ref(), + "G18_COMPUTE_OWNER_E2E inverse_pool=pass bytes={} directory_pages=265 translated_pages={} process_aliases=pass state=pass\n", + inverse.ideal.bytes, grown.translations.len()); + drop(pool); + for &(_, low, _) in grown.translations.iter() { + if uat.kernel_lower_vm().translate_iova(low).is_ok() || vm.translate_iova(low).is_ok() { + return Err(EFAULT); + } + } + drop(vm); + // Reusing the exact low addresses catches leaked allocator reservations. + drop(Self::new_shared_compute(dev, uat, high.clone(), inverse)?); + Self::verify_guard_tiers(dev, uat, high)?; + dev_info!(dev.as_ref(), "G18_COMPUTE_OWNER_E2E pass gpu_kicks=0 hardware_descriptor_loaded=0\n"); + Ok(()) + } + + #[inline(never)] + fn verify_guard_tiers(dev: &AsahiDevice, uat: &mmu::Uat, high: Range) -> Result { + let low = DIRECTORY_BASE + 0x40000000; + let mut anchor_object = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), high.clone(), + mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)?; + let mut anchor = anchor_object.map_alias_at_with_guards(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW, 85)?; + let original = uat.kernel_lower_vm().translate_iova(low)?; + let mut tails = KVec::with_capacity(3, GFP_KERNEL)?; + for pages in [21,22,42] { + let extent = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), high.clone(), + pages * mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)?; + let alias = extent.mapping.as_ref().ok_or(EINVAL)?.map_alias_into_guard(&mut anchor, + mmu::PROT_GPU_SHARED_RW)?; + for offset in (0..pages * mmu::UAT_PGSZ).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(extent.iova() + offset as u64)? + != uat.kernel_lower_vm().translate_iova(alias.iova() + offset as u64)? { + return Err(EFAULT); + } + } + tails.push((alias, extent), GFP_KERNEL)?; + } + match anchor_object.mapping.as_ref().ok_or(EINVAL)?.map_alias_into_guard(&mut anchor, + mmu::PROT_GPU_SHARED_RW) { + Err(error) if error == ENOSPC => {}, + _ => return Err(EFAULT), + } + while let Some((tail, extent)) = tails.pop() { + let address = tail.iova(); + anchor.rollback_guard_tail(tail)?; + for offset in (0..extent.logical_size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_lower_vm().translate_iova(address + offset as u64).is_ok() { + return Err(EFAULT); + } + } + if uat.kernel_lower_vm().translate_iova(low)? != original { return Err(EFAULT); } + for (retained, backing) in tails.iter() { + for offset in (0..backing.logical_size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_lower_vm().translate_iova(retained.iova() + offset as u64)? + != uat.kernel_vm().translate_iova(backing.iova() + offset as u64)? { + return Err(EFAULT); + } + } + } + match anchor_object.map_alias_at(uat.kernel_lower_vm(), address, mmu::PROT_GPU_SHARED_RW) { + Err(error) if error == EBUSY || error == ENOSPC => {}, + _ => return Err(EFAULT), + } + } + dev_info!(dev.as_ref(), "G18_COMPUTE_OWNER_E2E directory_tiers=21,43,85 rollback=pass\n"); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g18/buffer/pool_shared_compute.rs b/drivers/gpu/drm/asahi/g18/buffer/pool_shared_compute.rs new file mode 100644 index 0000000000000..69e57d0bb4623 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/buffer/pool_shared_compute.rs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Transactional allocation for a fresh, unpublished shared compute pool. +//! Previously submitted firmware state cannot be grown or reset here. + +use super::*; +use crate::g18::uapi::compute_plan::{G18ComputeDirectory, G18ComputePlan, G18_COMPUTE_BLOCK_BYTES}; + +/// Reserve first, then append without any fallible operation after publication. +fn append_reserved(out: &mut KVec, values: KVec) { + let length = out.len() + values.len(); + assert!(out.capacity() >= length); + for (slot, value) in out.spare_capacity_mut().iter_mut().zip(values) { slot.write(value); } + // SAFETY: capacity was checked and every added slot was initialized once. + unsafe { out.set_len(length); } +} + +struct DirectoryTail { + mapping: Option, + backing: MappedObject, + offset: u64, + anchor: usize, +} + +/// Until commit, every tail restores its original anchor reservation on Drop. +/// Mapping teardown then removes its PTEs before releasing its backing. +struct DirectoryTransaction<'a> { + anchors: &'a mut KVec, + tails: KVec, +} + +impl DirectoryTransaction<'_> { + fn commit(mut self) -> KVec { + core::mem::replace(&mut self.tails, KVec::new()) + } +} + +impl Drop for DirectoryTransaction<'_> { + fn drop(&mut self) { + for tail in self.tails.iter_mut().rev() { + if let Some(mapping) = tail.mapping.take() { + // Each tail came from this exact anchor under exclusive access. + // Reverse order preserves adjacency; rollback allocates nothing. + assert!(self.anchors[tail.anchor].rollback_guard_tail(mapping).is_ok()); + } + } + } +} + +impl G18UmaPoolMemory { + pub(crate) fn new_shared_compute(dev: &AsahiDevice, uat: &mmu::Uat, + high: Range, plan: G18ComputePlan) -> Result + { + let mut pool = Self::new_layout(dev, uat, high.clone(), 12, + true, false, 0x40000000, 1, G18_COMPUTE_BLOCK_BYTES)?; + pool.prepare_shared_compute(dev, uat, high, plan)?; + Ok(pool) + } + + /// A fresh FList view, retaining the previous scratch GEM objects while + /// giving the directory, list, state and counter independent ownership. + /// The caller must keep the old view until a verified firmware rebind. + pub(crate) fn replacement_shared_compute(&self, dev: &AsahiDevice, uat: &mmu::Uat, + high: Range, plan: G18ComputePlan) -> Result + { + if self.render || !self.encoded || !self.descriptor_loaded + || self.hardware_slot != 1 || !self.process_aliases.is_empty() + { return Err(EBUSY); } + let bias = if self.low_bias == 0x2_4000_0000 { 0x3_4000_0000 } + else { 0x2_4000_0000 }; + let mut pool = Self::new_layout(dev, uat, high.clone(), 12, + true, false, bias, 1, G18_COMPUTE_BLOCK_BYTES)?; + pool.prepare_shared_compute_retaining(dev, uat, high, plan, Some(self))?; + Ok(pool) + } + + /// Increase an unpublished pool without moving any existing mapping. + /// Every backing object, alias and list entry is staged. A failure restores + /// all directory guards and leaves the previous plan and block list intact. + pub(crate) fn prepare_shared_compute(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + high: Range, plan: G18ComputePlan) -> Result + { + self.prepare_shared_compute_retaining(dev, uat, high, plan, None) + } + + fn prepare_shared_compute_retaining(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + high: Range, plan: G18ComputePlan, retained: Option<&Self>) -> Result + { + if self.encoded || self.descriptor_loaded || !self.process_aliases.is_empty() { + return Err(EBUSY); + } + let previous = self.shared_compute_plan.map_or(0, |p| p.ideal.block_count); + let new = plan.ideal; + let directory = G18ComputeDirectory::for_blocks(new.block_count).ok_or(EINVAL)?; + let old_tail = if previous == 0 { 0 } else { + G18ComputeDirectory::for_blocks(previous).ok_or(EINVAL)?.tail_pages_per_usc + }; + if self.render || ![0x40000000, 0x2_4000_0000, 0x3_4000_0000].contains(&self.low_bias) + || self.hardware_slot != 1 + || self.geometry.block_bytes != G18_COMPUTE_BLOCK_BYTES + || self.geometry.directory_bytes_per_usc != 0x158000 + || self.layout.reservation_bytes != directory.reservation_bytes() + || self.scratch.len() != previous as usize + || (previous == 0 && self.scratch_plan.is_some()) + || new.block_count < previous + || new.bytes != u64::from(new.block_count) * G18_COMPUTE_BLOCK_BYTES + || directory.tail_pages_per_usc < old_tail + || high.start >= high.end + { return Err(EINVAL); } + let added = (new.block_count - previous) as usize; + let extra_pages = directory.tail_pages_per_usc - old_tail; + let extent_count = if extra_pages == 0 { 0 } else { 12 }; + let extent_bytes = extra_pages as usize * mmu::UAT_PGSZ; + // Reject impossible address-space demands before any GEM allocations. + // Actual free ranges and collisions are still checked by the allocator. + let needed = (added as u64).checked_mul(G18_COMPUTE_BLOCK_BYTES).ok_or(EOVERFLOW)? + .checked_add((extent_count * extent_bytes) as u64).ok_or(EOVERFLOW)? + .checked_add(((added + extent_count) * mmu::UAT_PGSZ) as u64).ok_or(EOVERFLOW)?; + if needed > high.end - high.start { return Err(ENOSPC); } + let low_end = (BLOCK_DATA_BASE + self.low_bias) + .checked_add(u64::from(new.block_count) * self.block_stride).ok_or(EOVERFLOW)?; + if low_end > uat.kernel_lower_vm().kernel_range().end { return Err(ENOSPC); } + self.scratch.reserve(added, GFP_KERNEL)?; + self.scratch_aliases.reserve(added, GFP_KERNEL)?; + self.directory_growth.reserve(extent_count, GFP_KERNEL)?; + self.directory_growth_offsets.reserve(extent_count, GFP_KERNEL)?; + self.aliases.reserve(extent_count, GFP_KERNEL)?; + let mut backing = KVec::with_capacity(added, GFP_KERNEL)?; + let mut aliases = KVec::with_capacity(added, GFP_KERNEL)?; + let mut entries = KVec::with_capacity(added, GFP_KERNEL)?; + for index in previous..new.block_count { + let low = BLOCK_DATA_BASE + self.low_bias + u64::from(index) * self.block_stride; + let mut block = if let Some(old) = retained.and_then(|p| p.scratch.get(index as usize)) { + if !old.cpu_wc || old.object_offset != 0 + || old.logical_size != G18_COMPUTE_BLOCK_BYTES as usize { return Err(EINVAL); } + let mut object = crate::gem::ObjectRef::new(old.object.gem.clone()); + let mapping = object.map_into_range(uat.kernel_vm(), high.clone(), + mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, true)?; + let iova = mapping.iova(); + MappedObject { mapping: Some(mapping), object, iova, object_offset: 0, + logical_size: old.logical_size, cpu_wc: true } + } else { + MappedObject::new_wc_in_range(dev, uat.kernel_vm(), high.clone(), + G18_COMPUTE_BLOCK_BYTES as usize, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)? + }; + let alias = block.map_alias_at_with_guards(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW, 1)?; + for offset in (0..G18_COMPUTE_BLOCK_BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(block.iova() + offset)? + != uat.kernel_lower_vm().translate_iova(low + offset)? { return Err(EFAULT); } + if let Some(old) = retained.and_then(|p| p.scratch.get(index as usize)) { + if uat.kernel_vm().translate_iova(old.iova() + offset)? + != uat.kernel_vm().translate_iova(block.iova() + offset)? { return Err(EFAULT); } + } + } + if uat.kernel_lower_vm().translate_iova(low + G18_COMPUTE_BLOCK_BYTES).is_ok() { + return Err(EFAULT); + } + entries.push(g18_uma::g18_uma_block_entry(low, 0).ok_or(EINVAL)?, GFP_KERNEL)?; + aliases.push(alias, GFP_KERNEL)?; + backing.push(block, GFP_KERNEL)?; + } + let mut transaction = DirectoryTransaction { + anchors: &mut self.aliases, + tails: KVec::with_capacity(extent_count, GFP_KERNEL)?, + }; + for usc in 0..extent_count { + let extent = match MappedObject::new_wc_in_range(dev, uat.kernel_vm(), high.clone(), + extent_bytes, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW) { + Ok(extent) => extent, + Err(error) => { + dev_info!(dev.as_ref(), "G18_SHARED_COMPUTE_ROLLBACK directory_tails={} error={:?}\n", + transaction.tails.len(), error); + return Err(error); + } + }; + let offset = (usc as u64 + 1) * self.geometry.directory_bytes_per_usc + - u64::from(directory.tail_pages_per_usc) * self.layout.page_bytes; + let mapping = extent.mapping.as_ref().ok_or(EINVAL)?.map_alias_into_guard( + &mut transaction.anchors[usc], mmu::PROT_GPU_SHARED_RW)?; + let low = DIRECTORY_BASE + self.low_bias + offset; + // Retain rollback ownership before any fallible translation check. + transaction.tails.push(DirectoryTail { mapping: Some(mapping), backing: extent, + offset, anchor: usc }, GFP_KERNEL)?; + let tail = &transaction.tails[usc]; + if tail.mapping.as_ref().ok_or(EINVAL)?.iova() != low { return Err(EFAULT); } + for page in (0..extent_bytes).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(tail.backing.iova() + page as u64)? + != uat.kernel_lower_vm().translate_iova(low + page as u64)? { return Err(EFAULT); } + } + } + // vmap may fail, and list validation may fail: both precede mutation. + // No fallible operation follows the complete block-list update. + self.block_list.with_bytes_mut(|bytes| { + let start = previous as usize * 4; + let end = new.block_count as usize * 4; + if end > bytes.len() || bytes[start..].iter().any(|&v| v != 0) { return Err(EBUSY); } + for (index, value) in entries.iter().enumerate() { + bytes[start + index*4..start + index*4 + 4].copy_from_slice(&value.to_le_bytes()); + } + Ok(()) + })?; + let tails = transaction.commit(); + for mut tail in tails { + // All destination capacity was reserved before staging began. + self.aliases.push(tail.mapping.take().expect("owned directory tail"), GFP_KERNEL) + .expect("reserved alias capacity"); + self.directory_growth.push(tail.backing, GFP_KERNEL).expect("reserved backing capacity"); + self.directory_growth_offsets.push(tail.offset, GFP_KERNEL).expect("reserved offset capacity"); + } + append_reserved(&mut self.scratch_aliases, aliases); + append_reserved(&mut self.scratch, backing); + self.scratch_plan = Some(g18_uma::G18ComputeScratchPlan { + block_bytes: G18_COMPUTE_BLOCK_BYTES, block_count: new.block_count, + register_pair: new.register_pair.map(u64::from), chunk_counters: new.chunk_counters, + }); + self.shared_compute_plan = Some(plan); + fence(Ordering::SeqCst); + crate::g18_kick_info!(dev.as_ref(), + "G18_SHARED_COMPUTE_OWNER bytes={} blocks={} directory_pages={} previous_blocks={} pair={:?} counters={:?} unpublished=1\n", + new.bytes, new.block_count, directory.committed_pages(), previous, + new.register_pair, new.chunk_counters); + if let Some(old) = retained { + crate::g18_kick_info!(dev.as_ref(), + "G18_SHARED_COMPUTE_REPLACEMENT old_bias={:#x} new_bias={:#x} retained_blocks={} blocks={} unpublished=1\n", + old.low_bias, self.low_bias, old.scratch.len().min(new.block_count as usize), new.block_count); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g18/compat/dep_stats.rs b/drivers/gpu/drm/asahi/g18/compat/dep_stats.rs new file mode 100644 index 0000000000000..d8f1efc45ccce --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/dep_stats.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Live render-dependency resolution counters (`g17p_render_dep_stats`, +//! root-only, read-only sysfs). Counter-only: relaxed atomics, no printk. +//! +//! `g17p_live_render_dependency` turns a still-pending predecessor's native +//! token into an explicit kick dependency. Each failure is counted by +//! `SubmissionError` variant, with the last failure's predecessor kick word, +//! fence status and decoded QID. `high_qid` counts dependencies resolved on +//! a predecessor whose fragment QID is 64..127: its kick word carries QID bit +//! 6 in bit 46, the firmware's valid-queue mask-word selector (ubfx x6,x5, +//! #46,#1), which the resolver used to refuse as unmodeled bits. +use core::fmt::Write; +use core::sync::atomic::{AtomicI64, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +use crate::g18::fw::submission::SubmissionError; + +pub(crate) static RESOLVED: AtomicU64 = AtomicU64::new(0); +pub(crate) static HIGH_QID: AtomicU64 = AtomicU64::new(0); +static PREDECESSOR_FAILED: AtomicU64 = AtomicU64::new(0); +static MISSING_TOKEN: AtomicU64 = AtomicU64::new(0); +static UNMODELED_BITS: AtomicU64 = AtomicU64::new(0); +static TIMESTAMP_RANGE: AtomicU64 = AtomicU64::new(0); +static QID_RANGE: AtomicU64 = AtomicU64::new(0); +static OTHER: AtomicU64 = AtomicU64::new(0); +static LAST_WORD: AtomicU64 = AtomicU64::new(0); +static LAST_STATUS: AtomicI64 = AtomicI64::new(0); +static LAST_VARIANT: AtomicU64 = AtomicU64::new(0); + +/// Count one resolved dependency (Ok with a dependency). +#[inline] +pub(crate) fn note_resolved(native_word: u64) { + RESOLVED.fetch_add(1, Ordering::Relaxed); + if (native_word >> crate::g18::fw::submission::KICK_SELECTOR_BIT) & 1 != 0 { + HIGH_QID.fetch_add(1, Ordering::Relaxed); + } +} + +/// Count one resolver failure. Cold: only failing submissions reach it. +#[cold] +#[inline(never)] +pub(crate) fn note_error(error: SubmissionError, status: i32, native_word: u64) { + let (counter, variant) = match error { + SubmissionError::RenderBarrierPredecessorFailed => (&PREDECESSOR_FAILED, 1), + SubmissionError::RenderBarrierMissingToken => (&MISSING_TOKEN, 2), + SubmissionError::KickWordUnmodeledBits => (&UNMODELED_BITS, 3), + SubmissionError::KickTimestampOutOfRange => (&TIMESTAMP_RANGE, 4), + SubmissionError::KickDependencyQueueIdOutOfRange => (&QID_RANGE, 5), + _ => (&OTHER, 6), + }; + counter.fetch_add(1, Ordering::Relaxed); + LAST_WORD.store(native_word, Ordering::Relaxed); + LAST_STATUS.store(i64::from(status), Ordering::Relaxed); + LAST_VARIANT.store(variant, Ordering::Relaxed); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn stats_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let r = |c: &AtomicU64| c.load(Ordering::Relaxed); + let word = LAST_WORD.load(Ordering::Relaxed); + let _ = writeln!(out, + "resolved={} high_qid={} predecessor_failed={} missing_token={} unmodeled_bits={} \ + timestamp_range={} qid_range={} other={} last_variant={} last_status={} \ + last_word={:#x} last_qid={}", + r(&RESOLVED), r(&HIGH_QID), r(&PREDECESSOR_FAILED), r(&MISSING_TOKEN), + r(&UNMODELED_BITS), r(&TIMESTAMP_RANGE), r(&QID_RANGE), r(&OTHER), + r(&LAST_VARIANT), LAST_STATUS.load(Ordering::Relaxed), word, + (word >> crate::g18::fw::submission::KICK_QID_SHIFT) & u64::from(crate::g18::fw::submission::KICK_QID_MASK)); + out.used as core::ffi::c_int +} + +unsafe extern "C" fn stats_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + -1 // EPERM: read-only counters +} + +static STATS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(stats_set), get: Some(stats_get), free: None, +}; +#[link_section = "__param"] +#[used] +static STATS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_render_dep_stats" + } else { c"asahi.g17p_render_dep_stats" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&STATS_OPS), perm: 0o400, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/dvfs_parity.rs b/drivers/gpu/drm/asahi/g18/compat/dvfs_parity.rs new file mode 100644 index 0000000000000..16d09f9309b5a --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/dvfs_parity.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Root-only runtime control `g17p_dvfs_parity` for the opt-in native +//! DVFS-input parity diagnostic. Default "0": no RegionC or message change. +//! +//! Write `MASK [HOLD_US [RAMP_US [CE_MAX]]]` (decimal or 0x hex): +//! bit0 work-tied CLPC deadline control effort (RegionC+0xe4), +//! bit1 native submit bracket (RegionC+0x7c/+0x80), +//! bit2 native-gated 0x87 note (requires bit1). +//! Reading prints the configuration, the live effort and event counters. +//! A change takes effect at the next submission or completion service. +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +use crate::g18::compat::g17p::{self as policy, Config}; + +static MASK: AtomicU32 = AtomicU32::new(policy::DEFAULT_CONFIG.mask); +static HOLD_NS: AtomicU64 = AtomicU64::new(policy::DEFAULT_CONFIG.hold_ns); +static RAMP_NS: AtomicU64 = AtomicU64::new(policy::DEFAULT_CONFIG.ramp_ns); +static CE_MAX: AtomicU32 = AtomicU32::new(policy::DEFAULT_CONFIG.ce_max); + +/// Counters for hardware verification. Never reset except by module load. +pub(crate) static CE_PUBLISHES: AtomicU64 = AtomicU64::new(0); +pub(crate) static CE_CLEARS: AtomicU64 = AtomicU64::new(0); +pub(crate) static CE_LIVE: AtomicU32 = AtomicU32::new(0); +pub(crate) static CE_LAST_SET_NS: AtomicU64 = AtomicU64::new(0); +pub(crate) static CE_LAST_CLEAR_NS: AtomicU64 = AtomicU64::new(0); +pub(crate) static RECHECKS: AtomicU64 = AtomicU64::new(0); +pub(crate) static BRACKET_OPENS: AtomicU64 = AtomicU64::new(0); +pub(crate) static KICKS_SENT: AtomicU64 = AtomicU64::new(0); +pub(crate) static KICKS_GATED: AtomicU64 = AtomicU64::new(0); +pub(crate) static STATS_66C_LAST: AtomicU32 = AtomicU32::new(0); +pub(crate) static STATS_66C_NONZERO: AtomicU64 = AtomicU64::new(0); +pub(crate) static PUBLISH_ERRORS: AtomicU64 = AtomicU64::new(0); + +pub(crate) fn config() -> Config { + Config { + mask: MASK.load(Ordering::Relaxed), + hold_ns: HOLD_NS.load(Ordering::Relaxed), + ramp_ns: RAMP_NS.load(Ordering::Relaxed), + ce_max: CE_MAX.load(Ordering::Relaxed), + } +} + +pub(crate) fn enabled(bit: u32) -> bool { + MASK.load(Ordering::Relaxed) & bit != 0 +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn parity_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 64]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let Ok(text) = core::str::from_utf8(&command[..len]) else { return -22; }; + let Some(parsed) = policy::parse(text.trim()) else { return -22; }; + // Order: parameters before the mask, so an enabling write never runs + // with stale timing values. + HOLD_NS.store(parsed.hold_ns, Ordering::Relaxed); + RAMP_NS.store(parsed.ramp_ns, Ordering::Relaxed); + CE_MAX.store(parsed.ce_max, Ordering::Relaxed); + MASK.store(parsed.mask, Ordering::Release); + kernel::pr_info!( + "G17P dvfs parity mask={:#x} hold_us={} ramp_us={} ce_max={:#x}\n", + parsed.mask, parsed.hold_ns / 1000, parsed.ramp_ns / 1000, parsed.ce_max + ); + 0 +} + +unsafe extern "C" fn parity_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let cfg = config(); + let _ = writeln!(out, "mask={:#x} hold_us={} ramp_us={} ce_max={:#x}", + cfg.mask, cfg.hold_ns / 1000, cfg.ramp_ns / 1000, cfg.ce_max); + let _ = writeln!(out, "ce_live={:#x} ce_publishes={} ce_clears={} rechecks={} publish_errors={}", + CE_LIVE.load(Ordering::Relaxed), CE_PUBLISHES.load(Ordering::Relaxed), + CE_CLEARS.load(Ordering::Relaxed), RECHECKS.load(Ordering::Relaxed), + PUBLISH_ERRORS.load(Ordering::Relaxed)); + let _ = writeln!(out, "ce_last_set_ns={} ce_last_clear_ns={} now_ns={}", + CE_LAST_SET_NS.load(Ordering::Relaxed), CE_LAST_CLEAR_NS.load(Ordering::Relaxed), + crate::g18::compat::timing_buffer::now()); + let _ = writeln!(out, "bracket_opens={} kicks_sent={} kicks_gated={} stats_66c_last={:#x} stats_66c_nonzero={}", + BRACKET_OPENS.load(Ordering::Relaxed), KICKS_SENT.load(Ordering::Relaxed), + KICKS_GATED.load(Ordering::Relaxed), STATS_66C_LAST.load(Ordering::Relaxed), + STATS_66C_NONZERO.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} + +static PARITY_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(parity_set), get: Some(parity_get), free: None, +}; +#[link_section = "__param"] +#[used] +static PARITY_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_dvfs_parity" + } else { c"asahi.g17p_dvfs_parity" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PARITY_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/e2e.rs b/drivers/gpu/drm/asahi/g18/compat/e2e.rs new file mode 100644 index 0000000000000..d150f57305c18 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/e2e.rs @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Opt-in end-to-end timeline ring (v81 diagnostic, default OFF). +//! +//! `g17p_e2e` (root-only sysfs parameter) arms a preallocated ring of 64-byte +//! records. While disarmed, every hook is a single relaxed atomic load and a +//! branch; no production state, ordering, locking or firmware access changes. +//! While armed, `record` reserves one slot with an atomic increment and does +//! eight relaxed stores (the first word last, with Release). No allocation, +//! printk, sleeping lock or wait is ever taken from a hook. +//! +//! Record layout (u64 words): +//! w0: CNTPCT_EL0 at the record point (host 1 GHz on T8140); written last. +//! w1: kind | cpu << 8 | generation << 16 | aux << 32 +//! w2: identity (submission id, slot, ...; kind specific) +//! w3..w7: payload a..e (kind specific; see the KIND_* constants). +//! +//! Commands written to `g17p_e2e`: +//! "arm " start a new generation from slot 0 (mask bit0 base events, +//! bit1 mailbox send/IRQ events, bit2 read-only firmware power +//! word + GPC pstate samples, bit3 device segment reads, +//! bit4 v91 fine submit-path laps, kinds 27..31) +//! "stop" stop recording (slots stay readable) +//! "mark " insert a marker record (only while armed) +//! "seek " set the read cursor +//! Reading returns up to 24 records from the cursor and advances it. +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const OPT_BASE: u32 = 1 << 0; +pub(crate) const OPT_MAILBOX: u32 = 1 << 1; +pub(crate) const OPT_POWER: u32 = 1 << 2; +pub(crate) const OPT_SEGMENTS: u32 = 1 << 3; +/// v91: fine submit-path laps (K_FINE_*). Default off; with the ring armed +/// without this bit, each lap site is one relaxed load and a branch. +pub(crate) const OPT_FINE: u32 = 1 << 4; + +// Kernel event kinds. +/// submit ioctl entry (head = file.rs submit entry). id=submission, aux=queue id, +/// a=in_syncs | out_syncs << 32, b=cmdbuf bytes, c=tgid, d=cmdbuf copied, +/// e=syncs parsed (both CNTPCT). +pub(crate) const K_IOCTL_ENTER: u8 = 1; +/// render command parsed. id=submission, a=command count, +/// b=render_barrier | compute_barrier << 32, c=render wait_through (MAX none), +/// d=compute wait_through (MAX none), e=render sequence. +pub(crate) const K_IOCTL_PARSED: u8 = 2; +/// render job pushed to drm_sched. id=submission, a=dependencies, +/// b=checked (input) dependencies, c=firmware fences, d=render sequence. +pub(crate) const K_SCHED_PUSHED: u8 = 3; +/// submit ioctl exit. id=submission, a=errno (0 ok). +pub(crate) const K_IOCTL_EXIT: u8 = 4; +/// run_job (publish) entry. id=submission, a=input fences, b=firmware fences, +/// c=input fences already signalled at entry. +pub(crate) const K_RUN_JOB: u8 = 5; +/// input fences waited/checked. id=submission. +pub(crate) const K_DEPS_DONE: u8 = 6; +/// physical lease attached. id=submission, a=render slot. +pub(crate) const K_LEASE: u8 = 7; +/// publication lock/prepare times (CNTPCT). id=submission, a=work state +/// acquired, b=first Runtime lock requested, c=first Runtime lock acquired, +/// d=prepare begin (Runtime dropped), e=prepare end. head=Runtime reacquired. +pub(crate) const K_PUB_LOCKS: u8 = 8; +/// publication stage summary. id=submission, aux=slot|retained<<8|hit<<9|ok<<10, +/// a=runtime_lock_ns | state_polls << 32, b=handoff_ns | prepare_ns << 32, +/// c=restage_ns | reachability_ns << 32, d=flist_request_ns | tvb_blocks << 32, +/// e=doorbell CNTPCT. head=submit_translated_render return (Runtime held). +pub(crate) const K_PUB_STAGES: u8 = 9; +/// publication sub-stage nanoseconds sub_ns[0..10] packed pairwise into a..e, +/// sub_ns[10] in aux. id=submission. +pub(crate) const K_PUB_SUB: u8 = 10; +/// publish() return after Runtime release. id=submission, a=ok. +pub(crate) const K_PUB_EXIT: u8 = 11; +/// mailbox send. a=endpoint, b=message, c=RTKit instance. +pub(crate) const K_SEND: u8 = 13; +/// hard-IRQ mailbox receive. a=endpoint | processor << 8, b=message. +pub(crate) const K_IRQ: u8 = 14; +/// event-worker firmware-event service wrapper. a=entry, b=Runtime acquired, +/// c=Runtime released, d=deferred completions signalled. head=after signals. +pub(crate) const K_WORKER: u8 = 15; +/// one render observation iteration. id=slot, a=Runtime acquired, +/// b=observation taken (Runtime released), c=observe() done, +/// d=Runtime reacquired, e=commit done (before release), aux=signalled|ok<<16. +pub(crate) const K_OBS_LOOP: u8 = 16; +/// completion proven under Runtime. id=submission, aux=slot, a=primary +/// notification ingress, b=secondary ingress, c=service-pass entry, +/// d=pass start (GPU 24 MHz), e=pass end (GPU 24 MHz). +pub(crate) const K_OBSERVED: u8 = 17; +/// G17PRenderCompletion::complete. id=submission, a=entry, b=UAPI timestamps +/// written, c=lease dropped, d=VM job dropped, e=hardware fence signalled. +/// head=aggregate member completed. +pub(crate) const K_COMPLETE: u8 = 18; +/// aggregate (output syncobj) fence signalled. id=submission, a=before signal, +/// aux=errno. head=after dma_fence_signal returned. +pub(crate) const K_AGG_SIGNAL: u8 = 19; +/// marker. id=user value. +pub(crate) const K_MARK: u8 = 20; +/// device identity at retirement. id=pair slot << 32 | ordinal, a=TA QID | +/// FRG QID << 32, b=observed TA stamp | FRG stamp << 32, c=expected stamp, +/// d=pass start, e=pass end. +pub(crate) const K_OBS_DEVICE: u8 = 21; +/// descriptor segments (opt bit3). id=pair slot << 32 | ordinal, a..d=TA +/// start/end, FRG start/end (24 MHz), e=read ok. +pub(crate) const K_SEGMENTS: u8 = 22; +/// read-only power samples (opt bit2). id=site (1 doorbell, 2 observed), +/// aux=slot, a=firmware power-up count, b=firmware power state word, +/// c=GPC performance state (MMIO, low 4 bits), d=read error mask. +pub(crate) const K_POWER: u8 = 23; +/// v84 feed-perf actuator change. aux=new pct (0 = withdrawn), id=score, +/// a=previous pct, b=starved passes, c=fed passes (cumulative). +pub(crate) const K_FEED_PERF: u8 = 26; +/// v91 ioctl laps (opt bit4), recorded after the scheduler push. id=submission, +/// aux=validation requests | proof hit << 8 | batch walks << 9 | pending cap +/// << 16. a=queue submit entry, b=output fences split, c=VM guard + parser + +/// pending vector, d=translate begin, e=translate end. +pub(crate) const K_FINE_IOCTL: u8 = 27; +/// v91 ioctl laps, continued. id=submission. a=parse finish + VM job guards, +/// b=aggregate fence + output syncobjs installed, c=timestamp destinations, +/// d=hardware fence + completion + dependency vectors, e=packet built. +pub(crate) const K_FINE_QUEUE: u8 = 28; +/// v91 locked reservation laps inside take_render_restage (Runtime held). +/// id=pair slot, a=entry, b=recovery gate done, c=dependency snapshot done, +/// d=VM handoff done, e=manager reservation done (graph taken). +pub(crate) const K_FINE_RESERVE: u8 = 29; +/// v91 restage_command laps (unlocked). id=pair slot << 32 | ordinal. +/// a=entry, b=render-state/AXFB/status/timestamp encode done, c=PM scene + +/// objects selected, d=TA descriptor + retained fields done, e=3D descriptor, +/// tail and MCache done. head=support record + fence done. +pub(crate) const K_FINE_RESTAGE: u8 = 30; +/// v91 SUB_CONFIG laps (Runtime held). id=pair slot << 32 | ordinal. +/// a=entry, b=3D QoS published, c=3D consistency validated, d=TA QoS +/// published, e=TA consistency validated. head=pair configure done. +pub(crate) const K_FINE_CONFIG: u8 = 31; +/// v94 retained-graph reachability revalidation (opt bit4), one per render +/// publication. id=render descriptor storage IOVA (per pair graph), +/// aux=path | reason << 4 | generation << 8 (path 0 exact, 1 lock-free walk, +/// 2 locked walk, 3 full walk; reason for a full walk 1 no proof, 2 no key, +/// 3 refused, 4 touched, 5 key moved), a=entry, b=app changes walked | kernel +/// changes walked << 32, c=app exec lock held (0 unlocked), d=kernel exec +/// lock held (0 unlocked), e=walks done (all CNTPCT). head=exit. +pub(crate) const K_FINE_REACH: u8 = 32; + +const CAPACITY: usize = 1 << 17; +#[repr(C, align(64))] +struct Slot { w: [AtomicU64; 8] } +static SLOTS: [Slot; CAPACITY] = [const { Slot { w: [const { AtomicU64::new(0) }; 8] } }; CAPACITY]; +static ARMED: AtomicU32 = AtomicU32::new(0); +static GENERATION: AtomicU32 = AtomicU32::new(0); +static NEXT: AtomicUsize = AtomicUsize::new(0); +static BOUND: AtomicUsize = AtomicUsize::new(0); +static CURSOR: AtomicUsize = AtomicUsize::new(0); +static DROPPED: AtomicU64 = AtomicU64::new(0); + +#[inline(always)] +pub(crate) fn armed() -> bool { ARMED.load(Ordering::Relaxed) != 0 } +#[inline(always)] +pub(crate) fn option(bit: u32) -> bool { ARMED.load(Ordering::Relaxed) & bit != 0 } + +/// Host CNTPCT_EL0 (same register the existing timing buffer uses). +#[inline(always)] +pub(crate) fn now() -> u64 { crate::g18::compat::timing_buffer::arch_counter() } + +/// Only reads the counter while armed; otherwise returns 0. +#[inline(always)] +pub(crate) fn stamp() -> u64 { if armed() { now() } else { 0 } } + +/// v91: counter for a fine lap (option bit4), otherwise 0. +#[inline(always)] +pub(crate) fn fine() -> u64 { if option(OPT_FINE) { now() } else { 0 } } + +#[inline(always)] +fn cpu() -> u64 { + // SAFETY: raw_smp_processor_id has no preconditions; a stale value after + // migration is acceptable for a diagnostic label. + u64::from(unsafe { bindings::raw_smp_processor_id() }) & 0xff +} + +pub(crate) fn record_at(head: u64, kind: u8, aux: u32, id: u64, payload: [u64; 5]) { + if !armed() { return; } + let generation = GENERATION.load(Ordering::Relaxed); + let index = NEXT.fetch_add(1, Ordering::Relaxed); + if index >= CAPACITY { DROPPED.fetch_add(1, Ordering::Relaxed); return; } + let slot = &SLOTS[index]; + slot.w[1].store(u64::from(kind) | cpu() << 8 | (u64::from(generation) & 0xffff) << 16 + | u64::from(aux) << 32, Ordering::Relaxed); + slot.w[2].store(id, Ordering::Relaxed); + for (word, value) in slot.w[3..].iter().zip(payload) { word.store(value, Ordering::Relaxed); } + slot.w[0].store(head.max(1), Ordering::Release); +} + +#[inline(always)] +pub(crate) fn record(kind: u8, aux: u32, id: u64, payload: [u64; 5]) { + if !armed() { return; } + record_at(now(), kind, aux, id, payload); +} + +// Read-only power samples (option bit2). Mappings are created on the first +// arm with the bit set and kept until module unload (never torn down while a +// hook could still use them). Same physical words and access method as the +// existing `g17p_diag_knobs` read (memremap WB + dc ivac + load), and the +// same always-on GPC register as `g17p_gpc_sample`. +static POWERUPS_PAGE: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); +static STATE_PAGE: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); +static GPC_REG: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); +const POWERUPS_LINK: u64 = 0xffff_fc00_0012_4ba0; +const STATE_LINK: u64 = 0xffff_fc00_0017_71f8; +const GPC_STATE_PHYS: u64 = 0x4_80e0_1000; +const PAGE: u64 = crate::pgtable::UAT_PGSZ as u64; + +fn map_power_sources() -> bool { + let map_page = |slot: &AtomicPtr, link: u64| -> bool { + if !slot.load(Ordering::Acquire).is_null() { return true; } + let physical = crate::g18::compat::g17p::g17p_gfx_link_to_physical(link); + // SAFETY: same page as the existing bounded diag_knobs probe; WB + // read-only use; retained until module unload. + let ptr = unsafe { bindings::memremap(physical & !(PAGE - 1), PAGE as usize, + bindings::MEMREMAP_WB as _) }; + if ptr.is_null() { return false; } + slot.store(ptr.cast::(), Ordering::Release); + true + }; + let ok = map_page(&POWERUPS_PAGE, POWERUPS_LINK) && map_page(&STATE_PAGE, STATE_LINK); + if GPC_REG.load(Ordering::Acquire).is_null() { + // SAFETY: the always-on 4-byte register already sampled by + // `g17p_gpc_sample`; retained until module unload. + let base = unsafe { bindings::ioremap(GPC_STATE_PHYS as bindings::phys_addr_t, 4) }; + if base.is_null() { return false; } + GPC_REG.store(base.cast::(), Ordering::Release); + } + ok +} + +fn read_firmware_word(page: &AtomicPtr, link: u64) -> Option { + let base = page.load(Ordering::Acquire); + if base.is_null() { return None; } + let physical = crate::g18::compat::g17p::g17p_gfx_link_to_physical(link); + let offset = (physical & (PAGE - 1)) as usize; + // SAFETY: `base` maps the whole containing page for the module lifetime; + // `offset` is 8-aligned (both links are) and below the page size. The + // invalidate touches only AP cachelines of this read-only mapping, + // exactly like LiveFirmwareU64Probe::observe_mask. + unsafe { + let address = base.add(offset); + core::arch::asm!("dsb osh", "dc ivac, {a}", "dsb osh", a = in(reg) address, + options(nostack, preserves_flags)); + Some(core::ptr::read_volatile(address.cast::())) + } +} + +/// Opt-in bit2 sample at a named site. Read-only; never called otherwise. +pub(crate) fn power_sample(site: u64, slot: u8) { + if !option(OPT_POWER) { return; } + let powerups = read_firmware_word(&POWERUPS_PAGE, POWERUPS_LINK); + let state = read_firmware_word(&STATE_PAGE, STATE_LINK); + let gpc = { + let base = GPC_REG.load(Ordering::Acquire); + // SAFETY: live 4-byte MMIO mapping retained until module unload. + if base.is_null() { None } else { Some(u64::from(unsafe { bindings::readl(base.cast()) })) } + }; + let errors = u64::from(powerups.is_none()) | u64::from(state.is_none()) << 1 + | u64::from(gpc.is_none()) << 2; + record(K_POWER, u32::from(slot), site, + [powerups.unwrap_or(0), state.unwrap_or(0), gpc.unwrap_or(0) & 0xf, errors, 0]); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} + +fn bounded_command(value: *const core::ffi::c_char, out: &mut [u8; 32]) -> Option { + let mut len = 0; + while len < out.len() { + // SAFETY: the parameter core supplies a NUL-terminated value; at most + // 32 bytes are examined and never beyond the first NUL. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { return Some(len); } + out[len] = byte; + len += 1; + } + None +} + +unsafe extern "C" fn e2e_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 32]; + let Some(len) = bounded_command(value, &mut command) else { return -22; }; + let Ok(text) = core::str::from_utf8(&command[..len]) else { return -22; }; + let text = text.trim(); + let (verb, argument) = match text.split_once(' ') { + Some((verb, argument)) => (verb, argument.trim()), + None => (text, ""), + }; + let hex = |argument: &str| u64::from_str_radix(argument.strip_prefix("0x").unwrap_or(argument), 16).ok(); + match verb { + "arm" => { + let Some(mask) = hex(if argument.is_empty() { "1" } else { argument }) else { return -22; }; + let mask = (mask as u32) | OPT_BASE; + if mask & OPT_POWER != 0 && !map_power_sources() { return -12; } + ARMED.store(0, Ordering::Relaxed); + GENERATION.fetch_add(1, Ordering::Relaxed); + DROPPED.store(0, Ordering::Relaxed); + CURSOR.store(0, Ordering::Relaxed); + NEXT.store(0, Ordering::Relaxed); + ARMED.store(mask, Ordering::Release); + } + "stop" => { + ARMED.store(0, Ordering::Release); + BOUND.store(NEXT.load(Ordering::Relaxed).min(CAPACITY), Ordering::Relaxed); + } + "mark" => { + let Some(value) = hex(argument) else { return -22; }; + record(K_MARK, 0, value, [0; 5]); + } + "seek" => { + let Some(index) = hex(argument) else { return -22; }; + CURSOR.store((index as usize).min(CAPACITY), Ordering::Relaxed); + } + _ => return -22, + } + 0 +} + +unsafe extern "C" fn e2e_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4095) }; + let mut out = Output { bytes, used: 0 }; + let armed = ARMED.load(Ordering::Acquire); + let bound = if armed != 0 { NEXT.load(Ordering::Relaxed).min(CAPACITY) } else { BOUND.load(Ordering::Relaxed) }; + let generation = GENERATION.load(Ordering::Relaxed) as u64 & 0xffff; + let mut cursor = CURSOR.load(Ordering::Relaxed); + let _ = writeln!(out, "s {:x} {:x} {:x} {:x} {:x} {}", armed, generation, cursor, bound, + DROPPED.load(Ordering::Relaxed), crate::g18::compat::timing_buffer::arch_frequency()); + for _ in 0..24 { + if cursor >= bound { break; } + let slot = &SLOTS[cursor]; + let head = slot.w[0].load(Ordering::Acquire); + let meta = slot.w[1].load(Ordering::Relaxed); + if head == 0 || (meta >> 16) & 0xffff != generation { + // Reserved but not yet published (or a stale late writer). + let _ = writeln!(out, "u {:x}", cursor); + } else { + let _ = write!(out, "r {:x} {:x} {:x}", cursor, head, meta); + for word in &slot.w[2..] { let _ = write!(out, " {:x}", word.load(Ordering::Relaxed)); } + let _ = writeln!(out); + } + cursor += 1; + } + CURSOR.store(cursor, Ordering::Relaxed); + out.used as core::ffi::c_int +} + +static E2E_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(e2e_set), get: Some(e2e_get), free: None, +}; +#[link_section = "__param"] +#[used] +static E2E_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_e2e" + } else { c"asahi.g17p_e2e" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&E2E_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/g17p.rs b/drivers/gpu/drm/asahi/g18/compat/g17p.rs new file mode 100644 index 0000000000000..c1b2663123477 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/g17p.rs @@ -0,0 +1,574 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Module 1134 G17P items read by carried translation and diagnostic code. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. +use core::ops::Range; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +#[cfg(CONFIG_DEV_COREDUMP)] +use core::ptr::NonNull; +#[cfg(CONFIG_DEV_COREDUMP)] +use kernel::{bindings, ffi::c_void}; +use kernel::{addr::PhysicalAddr, prelude::*}; +use crate::pgtable::{UAT_PGMSK, UAT_PGSZ}; +// diagnostic_accounting_clocks names the probe as `pgtable::LiveFirmwareU64Probe`. +use crate::g18::compat::g17p as pgtable; + +// Pinned J700 T8140 GFX A000 scheduler root used by the exact EP type-4 +// thunk. The extracted image has SHA-256 +// 3e2821bb40d3abffe30488a06ede5f523cfa47544d6a0ad7654a08be6b5dc30f. +// Its __DATA vmaddr/size are 0xfffffc000004c000/0x134000, exactly matching +// the J700 gfx-data carveout size. These addresses are diagnostic only. +const G17P_GFX_LINK_DATA_BASE: u64 = 0xffff_fc00_0004_c000; + +const G17P_GFX_LINK_DATA_END: u64 = 0xffff_fc00_0018_0000; + +const G17P_GFX_PHYSICAL_DATA_BASE: u64 = 0x0000_0100_01d7_c000; + +/// Translate a GFX firmware LINK address into the physical address the host +/// can map. Same arithmetic the render status-page probes use. +pub(crate) fn g17p_gfx_link_to_physical(link: u64) -> u64 { + G17P_GFX_PHYSICAL_DATA_BASE + (link - G17P_GFX_LINK_DATA_BASE) +} + +fn g17p_binding_row_address(base: u64, id: u32, stride: u64) -> Option { + if stride == 0 || stride & 7 != 0 { return None; } + let start = base.checked_add(u64::from(id).checked_mul(stride)?)?; + let end = start.checked_add(stride)?; + if start < G17P_GFX_LINK_DATA_BASE || start & 7 != 0 || end > G17P_GFX_LINK_DATA_END { + return None; + } + Some(start) +} + +/// Explicit diagnostic read of retained GFX accounting clocks. No MMIO and no +/// firmware writes. Caller requires a frozen, nonempty G17 timing capture. +/// Sample timestamps can advance independently; frequency/epoch fields must +/// remain stable throughout the read or the result is rejected. +pub(crate) fn diagnostic_accounting_clocks() -> Result<[u64; 8]> { + const OFFSETS: [u64; 8] = [ + 0x124ad8, 0x124af8, 0x124b18, 0x124b38, + 0x6e598, 0x6e5b0, 0x6e5d0, 0x179320, + ]; + let read = |index: usize| -> Result { + let link = 0xffff_fc00_0000_0000u64.checked_add(OFFSETS[index]).ok_or(EOVERFLOW)?; + let start = g17p_binding_row_address(link, 0, 8).ok_or(EINVAL)?; + pgtable::LiveFirmwareU64Probe::new(g17p_gfx_link_to_physical(start)) + .and_then(|probe| probe.observe_mask(u64::MAX, 1)) + .map(|sample| sample.first) + }; + let mut values = [0; 8]; + for index in 0..8 { values[index] = read(index)?; } + for index in 4..8 { + if read(index)? != values[index] { return Err(EAGAIN); } + } + Ok(values) +} + +/// Platform-derived SKSM scratch offsets within the mapped accelerator +/// register window. +/// +/// Apple's host loads these three values from accelerator object fields +/// `+0x10730`, `+0x10734`, and `+0x10738`. Those are object offsets, not SGX +/// MMIO offsets. This type carries the loaded values and no physical base. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17SksmScratchGeometry { + pub(crate) write0_base: u32, + pub(crate) write1_base: u32, + pub(crate) stride: u32, +} + +/// The two offsets used by the fixed G17P scratch FIFO slot. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17SksmScratchOffsets { + pub(crate) write0: u32, + pub(crate) write1: u32, +} + +impl G17SksmScratchGeometry { + /// Apply the exact host equation for fixed FIFO slot 9. + /// + /// The slot is independent of the SKSM queue ID carried in each write. + pub(crate) const fn fifo_offsets( + self, + ) -> core::result::Result { + const FIFO_SLOT: u32 = 9; + + let fifo_offset = match self.stride.checked_mul(FIFO_SLOT) { + Some(offset) => offset, + None => return Err(ConstructionError::SksmScratchOffsetOverflow), + }; + let write0 = match self.write0_base.checked_add(fifo_offset) { + Some(offset) => offset, + None => return Err(ConstructionError::SksmScratchOffsetOverflow), + }; + let write1 = match self.write1_base.checked_add(fifo_offset) { + Some(offset) => offset, + None => return Err(ConstructionError::SksmScratchOffsetOverflow), + }; + + Ok(G17SksmScratchOffsets { write0, write1 }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ConstructionError { + WrongChip, + WrongDeclaredIdentity, + WrongHardwareIdentity, + WrongUatGeometry, + UndecodableIdentity, + WrongDecodedIdentity, + WrongSubmissionBackend, + NullUatOwner, + SplitUatOwner, + WrongRoleTopology, + WrongRootPlacement, + WrongSharedClusterSize, + WrongSksmScratchGeometry, + WrongSksmQueueGeometry, + SksmScratchOffsetOverflow, +} + +pub(crate) const T8140_TIMESTAMP_ALIAS_RANGE: Range = + crate::g18::compat::g17p::CLIENT_LOW_VA_START..0x71_0000_0000; + +/// Two GiB boundary inside the existing four GiB runtime reservation. In +/// addition to descriptor arrays, each independently executing CL graph owns +/// a 23-MiB private-memory pool. Keep room for every advertised queue without +/// reducing concurrency; the remaining 2 GiB hold per-VM private aliases. +pub(crate) const CLIENT_LOW_VA_START: u64 = 0x70_8000_0000; + +pub(crate) const QID_COUNT: usize = 128; + +/// Device-wide QID gauges for `g17p_queue_slots`, indexed by [`Engine`]. +/// Kept by every table mutation and undone by `Drop`, so replacing the table +/// at processor stop leaves only the new table's leases counted. +pub(crate) static LEASED: [AtomicU32; 3] = [AtomicU32::new(0), AtomicU32::new(0), AtomicU32::new(0)]; + +pub(crate) static PUBLISHED: [AtomicU32; 3] = [AtomicU32::new(0), AtomicU32::new(0), AtomicU32::new(0)]; + +/// Most QIDs leased at once since load. +pub(crate) static HIGH_WATER: AtomicU32 = AtomicU32::new(0); + +/// `acquire` calls that found no free QID (the ENOSPC of queue creation and +/// of a new render pair). +pub(crate) static EXHAUSTED: AtomicU64 = AtomicU64::new(0); + +/// New compute graphs refused to keep [`COMPUTE_RENDER_RESERVE`] QIDs free. +pub(crate) static RESERVE_REFUSED: AtomicU64 = AtomicU64::new(0); + +/// QIDs a new compute graph may never take: kept for render pairs. +/// +/// A published QID is only returned at processor stop, and a quarantined or +/// exit-killed compute graph is never reused, so compute faults consume the +/// table one QID at a time. Without a reserve the last free QID goes to a +/// compute graph, and from then on every client that needs a new render +/// pair -- every new desktop client -- fails its first render with ENOSPC. +/// With it, exhaustion fails the next compute queue creation instead. 16 is +/// eight render pairs. +pub(crate) const COMPUTE_RENDER_RESERVE: usize = 16; + +/// Total QIDs currently leased, all engines. +pub(crate) fn leased_total() -> u32 { + LEASED.iter().map(|count| count.load(Ordering::Relaxed)).sum() +} + +/// bit0: publish a work-tied CLPC deadline control effort at RegionC+0xe4. +pub(crate) const PARITY_CE: u32 = 1 << 0; + +/// bit1: native submit-in-progress bracket order (+0x7c, then +0x80). +pub(crate) const PARITY_BRACKET: u32 = 1 << 1; + +/// bit2: native-gated 0x87 note inside the bracket (requires bit1). +pub(crate) const PARITY_KICK: u32 = 1 << 2; + +pub(crate) const PARITY_MASK_ALL: u32 = PARITY_CE | PARITY_BRACKET | PARITY_KICK; + +/// 1.0 in the kext's 16.16 effort encoding; larger values are rejected by +/// `updateCLPCDeadlineInterfaceCE(u32)` (`cmp w1, #0x10, lsl #12; b.ls`). +pub(crate) const CE_ONE: u32 = 0x1_0000; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Config { + pub(crate) mask: u32, + pub(crate) hold_ns: u64, + pub(crate) ramp_ns: u64, + pub(crate) ce_max: u32, +} + +pub(crate) const DEFAULT_HOLD_US: u64 = 16_000; + +pub(crate) const MIN_HOLD_US: u64 = 100; + +pub(crate) const MAX_HOLD_US: u64 = 1_000_000; + +pub(crate) const MAX_RAMP_US: u64 = 5_000_000; + +pub(crate) const DEFAULT_CONFIG: Config = Config { + // v84: native CLPC deadline-effort parity (bit0) is the production + // default (v83 hardware-qualified). Bracket/kick (bits 1/2) stay opt-in. + mask: PARITY_CE, + hold_ns: DEFAULT_HOLD_US * 1000, + ramp_ns: 0, + ce_max: CE_ONE, +}; + +fn parse_number(text: &str) -> Option { + match text.strip_prefix("0x") { + Some(hex) => u64::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + } +} + +/// Parse `MASK [HOLD_US [RAMP_US [CE_MAX]]]`. Omitted fields keep defaults. +pub(crate) fn parse(text: &str) -> Option { + let mut fields = text.split_ascii_whitespace(); + let mask = parse_number(fields.next()?)?; + if mask > u64::from(PARITY_MASK_ALL) { + return None; + } + let mut config = DEFAULT_CONFIG; + config.mask = mask as u32; + if config.mask & PARITY_KICK != 0 && config.mask & PARITY_BRACKET == 0 { + // Native only sends the note from inside updateSubmitInProgress(true). + return None; + } + if let Some(hold) = fields.next() { + let hold = parse_number(hold)?; + if !(MIN_HOLD_US..=MAX_HOLD_US).contains(&hold) { + return None; + } + config.hold_ns = hold * 1000; + } + if let Some(ramp) = fields.next() { + let ramp = parse_number(ramp)?; + if ramp > MAX_RAMP_US { + return None; + } + config.ramp_ns = ramp * 1000; + } + if let Some(max) = fields.next() { + let max = parse_number(max)?; + if max == 0 || max > u64::from(CE_ONE) { + return None; + } + config.ce_max = max as u32; + } + if fields.next().is_some() { + return None; + } + Some(config) +} + +#[cfg(CONFIG_DEV_COREDUMP)] +struct PhysPageMapping { + ptr: NonNull, +} + +/// A bounded read-only probe for one qword in live firmware-owned physical +/// memory. +/// +/// Construction maps the containing page once. [`Self::observe_mask`] then +/// invalidates only the sampled AP cacheline before each load, so a short +/// firmware transition can be observed without remapping, writing, or holding +/// any firmware lock. +#[cfg(CONFIG_DEV_COREDUMP)] +pub(crate) struct LiveFirmwareU64Probe { + page: PhysPageMapping, + offset: usize, +} + +/// Result of one bounded live-mask observation. +#[cfg(CONFIG_DEV_COREDUMP)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct LiveFirmwareMaskObservation { + pub(crate) first: u64, + pub(crate) last: u64, + pub(crate) observed_or: u64, + pub(crate) mask_seen: bool, + pub(crate) polls: usize, +} + +#[cfg(CONFIG_DEV_COREDUMP)] +impl LiveFirmwareU64Probe { + const MAX_POLLS: usize = 256; + + pub(crate) fn new(physical: PhysicalAddr) -> Result { + if physical & 7 != 0 { + return Err(EINVAL); + } + let page_mask = UAT_PGMSK as u64; + let page = PhysPageMapping::new(physical & !page_mask)?; + let offset = (physical & page_mask) as usize; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > UAT_PGSZ { + return Err(EINVAL); + } + Ok(Self { page, offset }) + } + + /// Observe one mask for at most `polls` cache-invalidated qword loads. + /// DIAGNOSTIC. Set bits in the probed qword; returns (before, after). + pub(crate) fn set_bits(&self, bits: u64) -> Result<(u64, u64)> { + self.page.set_bits_u64(self.offset, bits) + } + + /// DIAGNOSTIC (v93r). `set_bits` on a freshly invalidated line, then + /// clean it to the point of coherency so the coprocessor sees the bits. + pub(crate) fn set_bits_visible(&self, bits: u64) -> Result<(u64, u64)> { + self.page.invalidate_live_range(self.offset, size_of::())?; + let result = self.page.set_bits_u64(self.offset, bits)?; + self.page.clean_written_range(self.offset, size_of::())?; + Ok(result) + } + + /// DIAGNOSTIC. Read `out.len()` consecutive qwords from the probed qword + /// onwards, within its page, after invalidating their AP cachelines. Not + /// an atomic snapshot of the run. + pub(crate) fn read_run(&self, out: &mut [u64]) -> Result { + let size = out.len().checked_mul(size_of::()).ok_or(EOVERFLOW)?; + self.page.invalidate_live_range(self.offset, size)?; + for (index, value) in out.iter_mut().enumerate() { + *value = self.page.read_u64(self.offset + index * size_of::())?; + } + Ok(()) + } + + pub(crate) fn observe_mask( + &self, + mask: u64, + polls: usize, + ) -> Result { + if mask == 0 || polls == 0 || polls > Self::MAX_POLLS { + return Err(EINVAL); + } + + let mut first = 0; + let mut last = 0; + let mut observed_or = 0; + let mut mask_seen = false; + let mut completed = 0; + for index in 0..polls { + self.page.invalidate_live_range(self.offset, size_of::())?; + let value = self.page.read_u64(self.offset)?; + if index == 0 { + first = value; + } + last = value; + observed_or |= value; + completed = index + 1; + if value & mask != 0 { + mask_seen = true; + break; + } + core::hint::spin_loop(); + } + + Ok(LiveFirmwareMaskObservation { + first, + last, + observed_or, + mask_seen, + polls: completed, + }) + } +} + +#[cfg(CONFIG_DEV_COREDUMP)] +impl PhysPageMapping { + fn new(phys: PhysicalAddr) -> Result { + if phys & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + let ptr = unsafe { + // SAFETY: the caller supplies a page-aligned physical address + // obtained from a live firmware UAT PTE. The mapping is used only + // for bounded reads and is released by Drop. + bindings::memremap( + phys, + UAT_PGSZ, + bindings::MEMREMAP_WB as _, + ) + }; + let ptr = NonNull::new(ptr).ok_or_else(|| { + pr_err!("UAT crash walker: memremap failed for {:#x}\n", phys); + ENOMEM + })?; + Ok(Self { ptr }) + } + + fn invalidate_live_range(&self, offset: usize, size: usize) -> Result { + let end = offset.checked_add(size).ok_or(EOVERFLOW)?; + if size == 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + let ctr_el0: u64; + unsafe { + // SAFETY: CTR_EL0 is readable at EL1 and reports DminLine as + // log2(words). The resulting line size controls only bounded cache + // maintenance on this read-only mapping. + core::arch::asm!( + "mrs {ctr}, ctr_el0", + ctr = out(reg) ctr_el0, + options(nomem, nostack, preserves_flags), + ); + } + let line_size = 4usize + .checked_shl(((ctr_el0 >> 16) & 0xf) as u32) + .ok_or(EOVERFLOW)?; + if !line_size.is_power_of_two() { + return Err(EINVAL); + } + let start_address = self.ptr.as_ptr() as usize + offset; + let end_address = self.ptr.as_ptr() as usize + end; + let mut address = start_address & !(line_size - 1); + unsafe { + // SAFETY: the page remains mapped for the whole loop. `dc ivac` + // invalidates AP cachelines only; this diagnostic never writes the + // firmware-owned page. The barriers order invalidation before the + // atomic loads in `copy_atomic()`. + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + while address < end_address { + core::arch::asm!( + "dc ivac, {address}", + address = in(reg) address, + options(nostack, preserves_flags), + ); + address = address.checked_add(line_size).ok_or(EOVERFLOW)?; + } + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + } + Ok(()) + } + + fn read_u64(&self, offset: usize) -> Result { + let end = offset.checked_add(8).ok_or(EOVERFLOW)?; + if offset & 7 != 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + let pointer = unsafe { + // SAFETY: offset is qword-aligned and bounded to the mapped page. + self.ptr.as_ptr().cast::().add(offset).cast::() + }; + Ok(unsafe { &*pointer }.load(Ordering::Acquire)) + } + + /// DIAGNOSTIC. Set bits in one firmware-DRAM qword. + /// + /// The b1 image gates KTrace emission on class bytes at 0x17780c/0x17780d: + /// bit 0 is the only one this driver has ever had set, which is why the + /// halt-decision records (0x909 with the scan's own inputs, 0x113 at the + /// thread-4 wake, 0x105, 0x110) have never appeared in a capture. Arming + /// them is a read-modify-write of a debug-enable byte the firmware only + /// reads; it steers no scheduling decision. + fn set_bits_u64(&self, offset: usize, bits: u64) -> Result<(u64, u64)> { + let end = offset.checked_add(8).ok_or(EOVERFLOW)?; + if offset & 7 != 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + let pointer = unsafe { + // SAFETY: offset is qword-aligned and bounded to the mapped page, + // exactly as read_u64 checks above. + self.ptr.as_ptr().cast::().add(offset).cast::() + }; + let cell = unsafe { &*pointer }; + let before = cell.load(Ordering::Acquire); + cell.store(before | bits, Ordering::Release); + let after = cell.load(Ordering::Acquire); + Ok((before, after)) + } + + /// Clean AP cachelines covering a range this mapping has just written. + /// + /// The mapping is write-back, so a store can sit in an AP cacheline the + /// coprocessor's fetch path will not see. `invalidate_live_range()` above + /// is the read-side mirror of this; keep the two symmetric. + fn clean_written_range(&self, offset: usize, size: usize) -> Result { + let end = offset.checked_add(size).ok_or(EOVERFLOW)?; + if size == 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + let ctr_el0: u64; + unsafe { + // SAFETY: CTR_EL0 is readable at EL1 and reports DminLine as + // log2(words); the value bounds cache maintenance only. + core::arch::asm!( + "mrs {ctr}, ctr_el0", + ctr = out(reg) ctr_el0, + options(nomem, nostack, preserves_flags), + ); + } + let line_size = 4usize + .checked_shl(((ctr_el0 >> 16) & 0xf) as u32) + .ok_or(EOVERFLOW)?; + if !line_size.is_power_of_two() { + return Err(EINVAL); + } + let start_address = self.ptr.as_ptr() as usize + offset; + let end_address = self.ptr.as_ptr() as usize + end; + let mut address = start_address & !(line_size - 1); + unsafe { + // SAFETY: the page stays mapped for the whole loop. `dc cvac` + // cleans AP cachelines to the point of coherency; the barriers + // order the preceding stores before the clean and the clean before + // anything the coprocessor is subsequently told to do. + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + while address < end_address { + core::arch::asm!( + "dc cvac, {address}", + address = in(reg) address, + options(nostack, preserves_flags), + ); + address = address.checked_add(line_size).ok_or(EOVERFLOW)?; + } + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + } + Ok(()) + } + + fn write_atomic(&self, offset: usize, data: &[u8]) -> Result { + let end = offset.checked_add(data.len()).ok_or(EOVERFLOW)?; + if (offset | data.len()) & 7 != 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + for (index, source) in data.chunks_exact(size_of::()).enumerate() { + let at = offset + index * size_of::(); + let pointer = unsafe { + // SAFETY: `at` is qword-aligned and bounded to the mapped page + // by the check above. + self.ptr.as_ptr().cast::().add(at).cast::() + }; + let mut qword = [0u8; size_of::()]; + qword.copy_from_slice(source); + unsafe { &*pointer }.store(u64::from_le_bytes(qword), Ordering::Release); + } + self.clean_written_range(offset, data.len()) + } + + fn copy_atomic(&self, offset: usize, out: &mut [u8]) -> Result { + let end = offset.checked_add(out.len()).ok_or(EOVERFLOW)?; + if (offset | out.len()) & 7 != 0 || end > UAT_PGSZ { + return Err(EINVAL); + } + for (index, destination) in out.chunks_exact_mut(size_of::()).enumerate() { + destination.copy_from_slice( + &self + .read_u64(offset + index * size_of::())? + .to_le_bytes(), + ); + } + Ok(()) + } +} + +#[cfg(CONFIG_DEV_COREDUMP)] +impl Drop for PhysPageMapping { + fn drop(&mut self) { + unsafe { + // SAFETY: ptr was returned by memremap and has not been unmapped. + bindings::memunmap(self.ptr.as_ptr()); + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/compat/gem_stats.rs b/drivers/gpu/drm/asahi/g18/compat/gem_stats.rs new file mode 100644 index 0000000000000..22f02811b9478 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/gem_stats.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Live GEM object accounting (`gem_stats`, root-only, read-only sysfs). +//! +//! Every shmem object this driver creates is counted from creation until its +//! last reference drops, split by user and kernel ownership. Once a GEM object +//! is mapped into a GPU VM its pages stay pinned (and unevictable) until the +//! object is freed, so `user_bytes + kernel_bytes` is an upper bound on the +//! GPU share of /proc/meminfo `Unevictable`, and a value that keeps growing +//! after clients exit identifies a reference leak in the driver. The +//! `*_created` / `*_freed` totals show allocation churn. +use core::fmt::Write; +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +static USER_OBJECTS: AtomicU64 = AtomicU64::new(0); +static USER_BYTES: AtomicU64 = AtomicU64::new(0); +static USER_CREATED: AtomicU64 = AtomicU64::new(0); +static KERNEL_OBJECTS: AtomicU64 = AtomicU64::new(0); +static KERNEL_BYTES: AtomicU64 = AtomicU64::new(0); +static KERNEL_CREATED: AtomicU64 = AtomicU64::new(0); + +fn counters(kernel: bool) -> (&'static AtomicU64, &'static AtomicU64, &'static AtomicU64) { + if kernel { + (&KERNEL_OBJECTS, &KERNEL_BYTES, &KERNEL_CREATED) + } else { + (&USER_OBJECTS, &USER_BYTES, &USER_CREATED) + } +} + +/// Count one new GEM object. +pub(crate) fn note_new(kernel: bool, size: usize) { + let (objects, bytes, created) = counters(kernel); + objects.fetch_add(1, Ordering::Relaxed); + bytes.fetch_add(size as u64, Ordering::Relaxed); + created.fetch_add(1, Ordering::Relaxed); +} + +/// Count one freed GEM object. +pub(crate) fn note_free(kernel: bool, size: usize) { + let (objects, bytes, _) = counters(kernel); + objects.fetch_sub(1, Ordering::Relaxed); + bytes.fetch_sub(size as u64, Ordering::Relaxed); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn stats_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let r = |c: &AtomicU64| c.load(Ordering::Relaxed); + let uc = r(&USER_CREATED); + let kc = r(&KERNEL_CREATED); + let (uo, ko) = (r(&USER_OBJECTS), r(&KERNEL_OBJECTS)); + let _ = writeln!(out, + "user_objects={} user_bytes={} user_created={} user_freed={} \ + kernel_objects={} kernel_bytes={} kernel_created={} kernel_freed={}", + uo, r(&USER_BYTES), uc, uc.wrapping_sub(uo), + ko, r(&KERNEL_BYTES), kc, kc.wrapping_sub(ko)); + out.used as core::ffi::c_int +} + +unsafe extern "C" fn stats_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + -1 // EPERM: read-only counters +} + +static STATS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(stats_set), get: Some(stats_get), free: None, +}; +#[link_section = "__param"] +#[used] +static STATS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"gem_stats" + } else { c"asahi.gem_stats" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&STATS_OPS), perm: 0o400, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/host_path.rs b/drivers/gpu/drm/asahi/g18/compat/host_path.rs new file mode 100644 index 0000000000000..fb3bf77011806 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/host_path.rs @@ -0,0 +1,780 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! v84 host refill-path controls. +//! +//! `g17p_host_path` (root-only sysfs, runtime-writable bitmask): +//! bit0 EARLY_PUBLISH: a render whose submit ioctl has no unsignalled input, +//! prefix or firmware dependency, whose queue has no job still inside +//! the DRM scheduler, and for which a physical lease and a preparation +//! epoch are available without waiting, is published (doorbell rung) +//! from the submit ioctl. The DRM scheduler job is still created and +//! pushed; its run_job returns the already-published hardware fence, so +//! timeouts, credits, teardown and cancellation keep their existing +//! owners. Anything else takes the unchanged scheduler path. +//! bit1 FEED_PERF: GPU-feed CPU performance request (see below). +//! bit2 EARLY_SHALLOW (v87): bit0's early publication, taken only while the +//! queue has fewer than `g17p_early_depth` (default 2) of its own renders +//! outstanding, counting this one. The count is the queue's existing +//! pruned `render_frontier` (unsignalled hardware fences). A deeper +//! queue takes exactly the v84 scheduler path (bit0 off). +//! bit3 STATE_WORD (v87): publication and event-service recovery gates read +//! only the firmware recovery *state* word (the only field they test) +//! instead of the three-word handshake snapshot. Same predicate. +//! bit4 SETTLE_ONCE (v87): within one Runtime hold of the publish +//! reservation (warm checks + restage take), test the open-recovery +//! gate once instead of once per helper. The final gate immediately +//! before the doorbell (submit_translated_render) is unchanged. +//! bit5 RING_CACHE (v87): firmware-event ring drains skip re-reading the two +//! host-authored main-config ring pointers once they have been checked +//! equal to their derivation; revalidated every RING_REVALIDATE drains. +//! bit6 WORKER_TAIL (v87): event-worker bookkeeping after the drain (DVFS +//! parity, teardown service, back-pressure, recovery/poll/requeue +//! decisions) runs under one Runtime acquisition instead of six. +//! bit7 OBS_SKIP (v87): the render observation loop is skipped when, under +//! the drain's own Runtime hold, no pending slot has both type-1 masks. +//! bit8 VM_STATUS_MIRROR (v87): GET_PARAMS group 3 registration of a +//! userspace word that mirrors the sticky VM status (see g17_status). +//! bit9 VALIDATE_PROOF (v91): the submit ioctl's UAPI address validation +//! (one GPUVM exec lock per range before) records the ranges during +//! translation and proves them with ONE exec lock, or with none when +//! the queue's previous proof covers them at the VM's unchanged +//! mapping-validation epoch. Any failed proof reruns the unchanged +//! per-range translation, so errors are exactly those of v90. +//! bit10 PENDING_EXACT (v91): the ioctl's pending-command vector grows to +//! the commands actually present instead of preallocating 64 x ~1.2 KB +//! (a kmalloc_large page allocation and free on every submission). +//! bit11 WORD_STORES (v91): descriptor/record encoders store naturally +//! aligned 16/32/64-bit fields with one store instead of byte stores +//! (identical bytes; fewer write-combined stores). +//! bit12 PCORE_NUDGE (v92, DIAGNOSTIC, default off): while the feed-perf +//! request is applied, a submit ioctl that runs on a lower-capacity CPU +//! moves its own thread onto the highest-capacity cluster once, by a +//! transient affinity restriction that is restored immediately; later +//! wakeups then prefer that (idle) previous CPU. It transiently rewrites +//! a user-visible attribute, so it exists only to measure the value of +//! big-core placement (what CLPC's P-core recommendation and, on Linux, +//! a UCLAMP_TASK kernel would provide) before any kernel change. +//! bit13 FW_DEPS (v93, default off until lab-qualified): an input syncobj +//! whose fence (after unwrapping timeline chain links) consists only of +//! already-signalled fences and aggregates of this same queue's recent +//! fully enqueued ioctls is not waited on the host. Instead the +//! command's logical render/compute prefixes are raised to cover those +//! ioctls, which the existing prefix machinery turns into firmware +//! barrier tokens (render->render at publish, compute<->render through +//! publication fences plus tokens, compute->compute through the QID's +//! implicit parent), and each matched aggregate is status-checked at +//! publication. Anything else (another queue, another VM or client, a +//! foreign or errored fence, a fence array, an ioctl that did not fully +//! enqueue, a chain deeper than FW_DEP_LINKS_MAX) keeps the unchanged +//! host wait for that whole sync. Same VM by construction, so a faulted +//! predecessor's existing fail-outstanding/quarantine paths release the +//! firmware waiter. +//! bit14 AGG_FIRST (v93, default off until lab-qualified): a completion +//! signals the ioctl's user-visible aggregate fence before its per- +//! command hardware fence (whose inline callbacks are drm_sched job-done +//! bookkeeping). Timestamp writes, sticky status, lease release and +//! VM-job release still precede both: userspace reads timestamps and +//! recycles BOs/VAs as soon as the aggregate signals. +//! bit15 EARLY_DEPS (v93, default off until lab-qualified): early +//! publication also admits a render whose host dependencies are all +//! already signalled and whose remaining dependencies are firmware +//! status checks (FW_DEPS aggregates, prefix hardware fences); it still +//! declines any unsignalled host dependency, a scheduler backlog or an +//! unavailable lease, so the ioctl never waits for a dependency. +//! bit16 WAIT_UTIL_HINT: tag accepted feeder work for 8 ms while the +//! existing starvation detector requests CPU performance. A blocking +//! syncobj wait uses a scheduler capacity hint of 512 only until it +//! returns. User clamps, cgroup limits and CPU affinity remain intact. +//! Default on; clearing this bit is the same-kernel A/B control. +//! +//! v92 per-cluster feed levels: `g17p_feed_perf_pct_p` / `g17p_feed_perf_pct_e` +//! (0 = inherit `g17p_feed_perf_pct`) set the floor of the highest-capacity +//! (P) and the other (E) cpufreq policies separately while the GPU is starved. +//! Native IOGPU reports every command buffer to CLPC (IOPerfControl workSubmit +//! at IOGPUBlockFence::init, workBegin, workEnd with GPU data), and CLPC raises +//! the submitting thread group's CPU performance only while it gates the GPU. +//! The submitting thread and the ioctl run on E cores ~90% of the time and the +//! completion worker on CPU0 (P), so the two clusters bound different parts +//! of the host loop and need not share one level. +//! +//! GPU-feed CPU performance request. Native analogue: IOGPU reports GPU work +//! to CLPC (`IOGPUFenceMachine::notifyCLPCIOPerfControl`) so CPU performance +//! follows a GPU-bound feed loop. On Linux a dma-fence waiter's sleep is not +//! iowait, so schedutil sees ~10% utilisation and keeps both clusters at their +//! lowest OPP while the GPU idles waiting for that same CPU work (measured v81: +//! 91-99.7% lowest OPP, GPU idle 47-52% at depth 1/2). +//! +//! Detector (hot path, atomics only): every retired render reports its GPU pass +//! envelope. Decayed sums track GPU busy time and GPU idle time in short gaps +//! (0 < gap < MAX_REFILL: the GPU waited for the host refill path). A long gap +//! (paced client or idle desktop) resets the loop streak. The request is armed +//! while a closed feed loop keeps the GPU idle >= 20% of the time; it lingers +//! LINGER after the last such pass and is then withdrawn. GPU-bound +//! (overlapped) streams never arm it and no request exists at idle. +//! Actuator (process context, one work item): a cpufreq freq_qos MIN request on +//! each policy at `g17p_feed_perf_pct` percent of that policy's maximum. +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const EARLY_PUBLISH: u32 = 1 << 0; +pub(crate) const FEED_PERF: u32 = 1 << 1; +pub(crate) const EARLY_SHALLOW: u32 = 1 << 2; +pub(crate) const STATE_WORD: u32 = 1 << 3; +pub(crate) const SETTLE_ONCE: u32 = 1 << 4; +pub(crate) const RING_CACHE: u32 = 1 << 5; +pub(crate) const WORKER_TAIL: u32 = 1 << 6; +pub(crate) const OBS_SKIP: u32 = 1 << 7; +pub(crate) const VM_STATUS_MIRROR: u32 = 1 << 8; +pub(crate) const VALIDATE_PROOF: u32 = 1 << 9; +pub(crate) const PENDING_EXACT: u32 = 1 << 10; +pub(crate) const WORD_STORES: u32 = 1 << 11; +pub(crate) const PCORE_NUDGE: u32 = 1 << 12; +pub(crate) const FW_DEPS: u32 = 1 << 13; +pub(crate) const AGG_FIRST: u32 = 1 << 14; +pub(crate) const EARLY_DEPS: u32 = 1 << 15; +pub(crate) const WAIT_UTIL_HINT: u32 = 1 << 16; +/// Read only producer cursors when no completion/retirement decision is made. +pub(crate) const PRODUCER_READS: u32 = 1 << 17; +/// Count the retained graph as a PB owner, preserving its binding token. +pub(crate) const PB_OWNER_REF: u32 = 1 << 18; +/// Native pool sharing. Captured at logical-context creation, never changes a +/// live pool. Clearing this bit is a same-kernel per-pair allocation control. +pub(crate) const SHARED_FLIST: u32 = 1 << 19; +const ALL_BITS: u32 = (1 << 20) - 1; + +/// Production default for this candidate. v84-final had FEED_PERF only; v87 +/// added bits 2-8 and v91 bits 9-11. `g17p_host_path=0x1fe` is exactly the +/// v90 behaviour and `=2` the v84-final behaviour, for same-boot A/B. +/// v93 bits 13-15 stay off. The scoped wait hint adds bit16 to the default; +/// `0xffe` disables only that hint for same-kernel comparison. `0xeffe` +/// enables the older three probes without the scoped hint. +// Producer-only reads extend 0x10ffe to 0x30ffe; clearing bit17 restores +// the original full snapshots for an otherwise identical comparison. +const DEFAULT_MASK: u32 = FEED_PERF | EARLY_SHALLOW | STATE_WORD | SETTLE_ONCE + | RING_CACHE | WORKER_TAIL | OBS_SKIP | VM_STATUS_MIRROR + | VALIDATE_PROOF | PENDING_EXACT | WORD_STORES | WAIT_UTIL_HINT | PRODUCER_READS | PB_OWNER_REF | SHARED_FLIST; +static MASK: AtomicU32 = AtomicU32::new(DEFAULT_MASK); +static FEED_PCT: AtomicU32 = AtomicU32::new(50); +/// v92 per-cluster overrides of FEED_PCT (0 = inherit). +static FEED_PCT_P: AtomicU32 = AtomicU32::new(0); +static FEED_PCT_E: AtomicU32 = AtomicU32::new(0); +/// EARLY_SHALLOW depth: early publication while fewer than this many renders +/// of the queue are outstanding, counting the one being submitted. +static EARLY_DEPTH: AtomicU32 = AtomicU32::new(2); +const EARLY_DEPTH_MAX: u32 = 64; +/// RING_CACHE: re-read and re-check the main-config ring pointers on at +/// least every this many drains of one role. +pub(crate) const RING_REVALIDATE: u32 = 256; + +#[inline(always)] +pub(crate) fn enabled(bit: u32) -> bool { MASK.load(Ordering::Relaxed) & bit != 0 } + +/// v87 counters (relaxed; observability only, printed by `g17p_host_path`). +pub(crate) static EARLY_TAKEN: AtomicU64 = AtomicU64::new(0); +pub(crate) static SETTLE_SKIPPED: AtomicU64 = AtomicU64::new(0); +pub(crate) static RING_CACHE_HITS: AtomicU64 = AtomicU64::new(0); +pub(crate) static TAIL_MERGED: AtomicU64 = AtomicU64::new(0); +pub(crate) static OBS_SKIPPED: AtomicU64 = AtomicU64::new(0); +pub(crate) static MIRRORS: AtomicU64 = AtomicU64::new(0); +pub(crate) static PROOF_HITS: AtomicU64 = AtomicU64::new(0); +pub(crate) static PROOF_BATCHES: AtomicU64 = AtomicU64::new(0); +pub(crate) static PROOF_FALLBACKS: AtomicU64 = AtomicU64::new(0); +pub(crate) static NUDGES: AtomicU64 = AtomicU64::new(0); +pub(crate) static NUDGE_FAILS: AtomicU64 = AtomicU64::new(0); +/// v93 FW_DEPS input syncs: converted to firmware prefixes, kept as host +/// waits, or dropped because already signalled without error. +pub(crate) static FW_DEP_CONVERTED: AtomicU64 = AtomicU64::new(0); +pub(crate) static FW_DEP_HOST: AtomicU64 = AtomicU64::new(0); +pub(crate) static FW_DEP_SATISFIED: AtomicU64 = AtomicU64::new(0); +/// v93 EARLY_DEPS: early publications that carried dependencies. +pub(crate) static EARLY_WITH_DEPS: AtomicU64 = AtomicU64::new(0); +/// v93 AGG_FIRST completions (aggregate member completed before hw fence). +pub(crate) static AGG_FIRST_TAKEN: AtomicU64 = AtomicU64::new(0); + +#[inline(always)] +pub(crate) fn count(counter: &AtomicU64) { counter.fetch_add(1, Ordering::Relaxed); } + +/// EARLY_SHALLOW gate. `outstanding_before` is the queue's unsignalled +/// renders excluding this one; admit while the total stays within `depth`. +#[inline(always)] +pub(crate) const fn early_shallow_admits(outstanding_before: usize, depth: u32) -> bool { + outstanding_before < depth as usize +} + +/// Whether a render submission may attempt early publication: bit0 alone +/// (v84, any depth) or bit2 within the depth. Dependencies always decline. +/// With v93 EARLY_DEPS the caller passes whether any host dependency is +/// still unsignalled instead of whether one exists. +#[inline(always)] +pub(crate) fn early_publish_allowed(outstanding_before: usize, has_dependencies: bool) -> bool { + if has_dependencies { return false; } + let mask = MASK.load(Ordering::Relaxed); + if mask & EARLY_PUBLISH != 0 { return true; } + if mask & EARLY_SHALLOW == 0 { return false; } + // No counter here: a declined (deep) submission must stay the v84 path. + early_shallow_admits(outstanding_before, EARLY_DEPTH.load(Ordering::Relaxed)) +} + +/// v93 FW_DEPS: longest timeline chain walked per input sync; ring size. +pub(crate) const FW_DEP_LINKS_MAX: u32 = 16; +pub(crate) const FW_DEP_RECENT: usize = 8; + +/// One unwrapped input-fence leaf, as seen by `fw_dep_step`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum FwDepLeaf { + /// Signalled without error: satisfies nothing further. + Signalled, + /// Signalled with an error: the host path must consume it. + Failed, + /// Unsignalled; identity is the fence address (compared, never used). + Pending(usize), + /// Nested container, fence array, missing leaf or over-long chain. + Unsupported, +} + +/// A recent fully enqueued ioctl of this queue: aggregate identity and the +/// inclusive logical sequence ranges of its commands (0 = none of that kind). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct FwDepRecent { + pub(crate) identity: usize, + pub(crate) render_last: u64, + pub(crate) compute_last: u64, +} + +/// Accumulated firmware expression of one input sync. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct FwDepResolution { + /// Highest render / compute logical sequence that must retire first. + pub(crate) render_through: u64, + pub(crate) compute_through: u64, + /// Bit i: recent entry i supplied a pending leaf (status-checked later). + pub(crate) used: u32, + /// Leaves seen (Signalled included); 0 means an empty walk. + pub(crate) leaves: u32, +} + +/// Fold one leaf into `resolution`. False means the sync is not provably +/// expressible in firmware and must keep its original host wait. +pub(crate) fn fw_dep_step( + resolution: &mut FwDepResolution, leaf: FwDepLeaf, recent: &[Option], +) -> bool { + resolution.leaves += 1; + let identity = match leaf { + FwDepLeaf::Signalled => return true, + FwDepLeaf::Failed | FwDepLeaf::Unsupported => return false, + FwDepLeaf::Pending(identity) => identity, + }; + if identity == 0 { return false; } + let Some(index) = recent.iter().position(|entry| + entry.is_some_and(|entry| entry.identity == identity)) else { return false; }; + let Some(entry) = recent[index] else { return false; }; + resolution.render_through = resolution.render_through.max(entry.render_last); + resolution.compute_through = resolution.compute_through.max(entry.compute_last); + resolution.used |= 1u32 << index; + true +} + +/// Raise an optional logical prefix (`None` = independent) to cover `through`. +#[inline(always)] +pub(crate) const fn fw_dep_raise(prefix: Option, through: u64) -> Option { + if through == 0 { return prefix; } + match prefix { + Some(limit) if limit >= through => Some(limit), + _ => Some(through), + } +} + +/// RING_CACHE decision for one role. `entry` is (cursor, ring, drains left) +/// of the last successful check; `expected` is this drain's derivation. +/// Returns true when the two pointer reads may be skipped (and consumes one +/// drain of the budget); false means read and check them, then `ring_checked`. +#[inline(always)] +pub(crate) fn ring_cache_hit(entry: &mut (u64, u64, u32), expected: (u64, u64), enabled: bool) -> bool { + if !enabled || entry.2 == 0 || entry.0 != expected.0 || entry.1 != expected.1 { + return false; + } + entry.2 -= 1; + true +} + +/// Record a successful full pointer check for `ring_cache_hit`. +#[inline(always)] +pub(crate) fn ring_checked(entry: &mut (u64, u64, u32), checked: (u64, u64)) { + *entry = (checked.0, checked.1, RING_REVALIDATE); +} + +// --------------------------------------------------------------------------- +// Feed-perf detector state. GPU pass stamps are 24 MHz firmware ticks. +const MAX_REFILL_TICKS: u64 = 24_000 * 2; // 2 ms: longer gaps are paced/idle +const STREAK_ON: u32 = 8; +const STREAK_MAX: u32 = 64; +const LINGER_NS: u64 = 8_000_000; +const POLL_MS: i64 = 2; + +static LAST_END: AtomicU64 = AtomicU64::new(0); +/// Exponentially decayed (1/8 per pass) GPU idle and busy ticks over recent +/// passes. Idle counts only short gaps: the GPU waited for the host. +static IDLE_EWMA: AtomicU64 = AtomicU64::new(0); +static BUSY_EWMA: AtomicU64 = AtomicU64::new(0); +/// Consecutive passes without a long (paced) gap. +static STREAK: AtomicU32 = AtomicU32::new(0); +static UNTIL_NS: AtomicU64 = AtomicU64::new(0); +/// A worker owns the actuator (running or queued). +static ACTIVE: AtomicBool = AtomicBool::new(false); +/// The detector asked for a worker; the event worker queues it. +static PENDING: AtomicBool = AtomicBool::new(false); +static UNBINDING: AtomicBool = AtomicBool::new(false); +static STARVED: AtomicU64 = AtomicU64::new(0); +static FED: AtomicU64 = AtomicU64::new(0); +static BOOSTS: AtomicU64 = AtomicU64::new(0); +static BOOSTED_NS: AtomicU64 = AtomicU64::new(0); + +fn now_ns() -> u64 { + ::ktime_get() as u64 +} + +fn decay_add(sum: &AtomicU64, add: u64) -> u64 { + let mut next = 0; + let _ = sum.fetch_update(Ordering::AcqRel, Ordering::Acquire, |v| { + next = v - v / 8 + add; + Some(next) + }); + next +} + +/// Hot path: one retired render's GPU pass envelope [start, end] (24 MHz). +/// Atomics only. Concurrent callers may interleave; the estimate tolerates it. +pub(crate) fn note_render_pass(start: u64, end: u64) { + if !enabled(FEED_PERF) || end <= start { return; } + let previous = LAST_END.fetch_max(end, Ordering::AcqRel); + let (idle, busy, paced) = if previous == 0 { + (0, end - start, true) + } else if start >= previous { + let gap = start - previous; + if gap < MAX_REFILL_TICKS { (gap, end - start, false) } else { (0, end - start, true) } + } else if end > previous { + (0, end - previous, false) // overlapped: only the new busy tail counts + } else { + (0, 0, false) + }; + let idle_sum = decay_add(&IDLE_EWMA, idle); + let busy_sum = decay_add(&BUSY_EWMA, busy); + let streak = if paced { + STREAK.store(0, Ordering::Release); + 0 + } else { + STREAK.fetch_update(Ordering::AcqRel, Ordering::Acquire, + |s| Some((s + 1).min(STREAK_MAX))).unwrap_or(0) + 1 + }; + // Host-starved feed loop: a closed loop (no paced gaps) in which the GPU + // is idle at least 20% of the time waiting for the host. + if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { + STARVED.fetch_add(1, Ordering::Relaxed); + UNTIL_NS.store(now_ns() + LINGER_NS, Ordering::Release); + if !ACTIVE.swap(true, Ordering::AcqRel) { + PENDING.store(true, Ordering::Release); + } + } else { + FED.fetch_add(1, Ordering::Relaxed); + } +} + +/// Called by the firmware-event worker after each pass (process context). +pub(crate) fn take_pending() -> bool { PENDING.swap(false, Ordering::AcqRel) } + +// --------------------------------------------------------------------------- +// Actuator. Only the single feed-perf work item (and unbind after that item is +// disabled) touches REQS, so accesses are serialised by construction. +const MAX_POLICIES: usize = 8; +struct Reqs { + policy: [AtomicPtr; MAX_POLICIES], + max_khz: [AtomicU32; MAX_POLICIES], + req: core::cell::UnsafeCell<[bindings::freq_qos_request; MAX_POLICIES]>, + count: AtomicU32, + registered: AtomicBool, + applied_pct: AtomicU32, + /// v92: this policy has the highest maximum frequency (the P cluster). + big: [AtomicBool; MAX_POLICIES], + /// v92: value last written to each request (skips redundant updates). + applied_khz: [AtomicU32; MAX_POLICIES], + /// v92: CPUs (bit per CPU id, ids < 64) of the highest-capacity policy. + big_cpus: AtomicU64, +} +// SAFETY: `req` is only accessed by the serialised actuator (see above). +unsafe impl Sync for Reqs {} +static REQS: Reqs = Reqs { + policy: [const { AtomicPtr::new(core::ptr::null_mut()) }; MAX_POLICIES], + max_khz: [const { AtomicU32::new(0) }; MAX_POLICIES], + // SAFETY: freq_qos_request is plain C data; all-zero is its unregistered state. + req: core::cell::UnsafeCell::new(unsafe { core::mem::zeroed() }), + count: AtomicU32::new(0), + registered: AtomicBool::new(false), + applied_pct: AtomicU32::new(0), + big: [const { AtomicBool::new(false) }; MAX_POLICIES], + applied_khz: [const { AtomicU32::new(0) }; MAX_POLICIES], + big_cpus: AtomicU64::new(0), +}; + +/// v92: the floor (kHz, 0 = withdrawn) one policy gets for request level +/// `base` percent. A big policy uses the P override, others the E override; +/// an override of 0 inherits `base`. Level 0 always withdraws. +pub(crate) const fn cluster_floor_khz(max_khz: u32, big: bool, base: u32, pct_p: u32, pct_e: u32) -> u32 { + if base == 0 { return 0; } + let over = if big { pct_p } else { pct_e }; + let pct = if over != 0 { over } else { base }; + let pct = if pct > 100 { 100 } else { pct }; + ((max_khz as u64 * pct as u64) / 100) as u32 +} + +/// v92: index of the policy with the highest maximum frequency (first on a +/// tie), or None when there are fewer than two distinct maxima (no big/little +/// distinction to make). +pub(crate) fn big_policy(max_khz: &[u32]) -> Option { + let top = *max_khz.iter().max()?; + if max_khz.iter().all(|&k| k == top) { return None; } + max_khz.iter().position(|&k| k == top) +} + +fn register() -> bool { + if REQS.registered.load(Ordering::Acquire) { return true; } + // SAFETY: plain read of a kernel global set at boot. + let cpus = unsafe { bindings::nr_cpu_ids }; + let mut count = 0usize; + let mut policy_cpus = [0u64; MAX_POLICIES]; + for cpu in 0..cpus { + // SAFETY: cpufreq_cpu_get tolerates any cpu index and takes a reference. + let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; + if policy.is_null() { continue; } + let known = (0..count).position(|i| REQS.policy[i].load(Ordering::Relaxed) == policy); + if let (Some(index), true) = (known, cpu < 64) { policy_cpus[index] |= 1u64 << cpu; } + if known.is_none() && count < MAX_POLICIES { + // SAFETY: `policy` is referenced; the request slot is unregistered + // and lives in static storage for the module lifetime. + let ret = unsafe { + let slot = (*REQS.req.get()).as_mut_ptr().add(count); + bindings::freq_qos_add_request(&mut (*policy).constraints, slot, + bindings::freq_qos_req_type_FREQ_QOS_MIN, + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32) + }; + if ret >= 0 { + REQS.policy[count].store(policy, Ordering::Relaxed); + // SAFETY: referenced policy; cpuinfo is immutable after init. + REQS.max_khz[count].store(unsafe { (*policy).cpuinfo.max_freq }, Ordering::Relaxed); + REQS.applied_khz[count].store(bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32, Ordering::Relaxed); + if cpu < 64 { policy_cpus[count] |= 1u64 << cpu; } + count += 1; + } + } + // SAFETY: balances cpufreq_cpu_get above. The request keeps its own + // pointer to the policy's constraints, as cpufreq cooling devices do. + unsafe { bindings::cpufreq_cpu_put(policy) }; + } + let mut maxima = [0u32; MAX_POLICIES]; + for (i, slot) in maxima.iter_mut().enumerate().take(count) { + *slot = REQS.max_khz[i].load(Ordering::Relaxed); + } + let big = big_policy(&maxima[..count]); + for i in 0..count { REQS.big[i].store(Some(i) == big, Ordering::Relaxed); } + // CPU ids >= 64 are not represented; the nudge then stays disabled. + REQS.big_cpus.store(match big { Some(i) if cpus <= 64 => policy_cpus[i], _ => 0 }, Ordering::Relaxed); + REQS.count.store(count as u32, Ordering::Release); + REQS.registered.store(count != 0, Ordering::Release); + count != 0 +} + +fn apply(pct: u32) { + if pct != 0 && !register() { return; } + if !REQS.registered.load(Ordering::Acquire) { return; } + let (pct_p, pct_e) = (FEED_PCT_P.load(Ordering::Relaxed), FEED_PCT_E.load(Ordering::Relaxed)); + for i in 0..REQS.count.load(Ordering::Acquire) as usize { + let max = REQS.max_khz[i].load(Ordering::Relaxed); + let floor = cluster_floor_khz(max, REQS.big[i].load(Ordering::Relaxed), pct, pct_p, pct_e); + let value = if floor == 0 { bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 } else { floor }; + // v92: per-cluster levels may change while applied; write on change only. + if REQS.applied_khz[i].swap(value, Ordering::Relaxed) == value { continue; } + // SAFETY: slot `i` was registered by `register` and is only touched here. + unsafe { bindings::freq_qos_update_request((*REQS.req.get()).as_mut_ptr().add(i), value as i32) }; + } + let before = REQS.applied_pct.swap(pct, Ordering::Relaxed); + if before == pct { return; } + let now = now_ns(); + if before == 0 && pct != 0 { + BOOSTS.fetch_add(1, Ordering::Relaxed); + BOOST_SINCE.store(now, Ordering::Relaxed); + } else if before != 0 && pct == 0 { + BOOSTED_NS.fetch_add(now.saturating_sub(BOOST_SINCE.load(Ordering::Relaxed)), Ordering::Relaxed); + } + crate::g18::compat::e2e::record(crate::g18::compat::e2e::K_FEED_PERF, pct, STREAK.load(Ordering::Relaxed) as u64, + [before as u64, STARVED.load(Ordering::Relaxed), FED.load(Ordering::Relaxed), 0, 0]); +} +static BOOST_SINCE: AtomicU64 = AtomicU64::new(0); + +/// v92 PCORE_NUDGE (diagnostic, opt-in): see the module documentation. +/// Called at the start of the submit ioctl (process context, sleepable, no +/// driver lock held). Only while the feed request is applied, only from a CPU +/// outside the big cluster, and only when the thread may run there at all. +pub(crate) fn nudge_feeding_thread() { + if !enabled(PCORE_NUDGE) || REQS.applied_pct.load(Ordering::Relaxed) == 0 { return; } + let big = REQS.big_cpus.load(Ordering::Relaxed); + // SAFETY: raw_smp_processor_id has no preconditions; a stale value only + // makes this diagnostic skip or nudge once more. + let cpu = unsafe { bindings::raw_smp_processor_id() } as u32; + if big == 0 || cpu >= 64 || big & (1u64 << cpu) != 0 { return; } + let task = kernel::task::Task::current_raw(); + // SAFETY: `task` is the current task, alive for this call. Its effective + // mask changes only through set_cpus_allowed_ptr; a concurrent + // sched_setaffinity() from another thread of this process during these two + // calls could be overwritten by the restore (documented diagnostic limit). + let saved = unsafe { core::ptr::read(core::ptr::addr_of!((*task).cpus_mask)) }; + let mut target = saved; + target.bits[0] &= big as usize; + for word in target.bits.iter_mut().skip(1) { *word = 0; } + if target.bits[0] == 0 { return; } + // SAFETY: valid current task and masks; both calls may sleep (ioctl context). + let moved = unsafe { bindings::set_cpus_allowed_ptr(task, &target) }; + if moved != 0 { count(&NUDGE_FAILS); return; } + // SAFETY: as above; restores the exact effective mask read before. + let restored = unsafe { bindings::set_cpus_allowed_ptr(task, &saved) }; + if restored != 0 { count(&NUDGE_FAILS); } else { count(&NUDGES); } +} + +/// Capacity hint for a client actively replenishing a CPU-starved GPU. +/// A per-file expiry and a scoped syncobj wait keep this out of idle/non-GPU +/// work. No task attribute, affinity or saved userspace clamp is rewritten. +pub(crate) fn feeder_wait_util_min() -> u32 { + if !cfg!(CONFIG_UCLAMP_TASK) { return 0; } + if enabled(WAIT_UTIL_HINT) && wanted() + && REQS.applied_pct.load(Ordering::Relaxed) != 0 { 512 } else { 0 } +} + +fn wanted() -> bool { + enabled(FEED_PERF) && !UNBINDING.load(Ordering::Acquire) + && now_ns() < UNTIL_NS.load(Ordering::Acquire) +} + +/// Body of the feed-perf work item. Holds the request while wanted, polling +/// the linger deadline; withdraws it and exits otherwise. +pub(crate) fn run_actuator() { + loop { + if wanted() { + apply(FEED_PCT.load(Ordering::Relaxed).clamp(1, 100)); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(POLL_MS)); + continue; + } + apply(0); + ACTIVE.store(false, Ordering::SeqCst); + // Close the race with a detector that saw ACTIVE still set. + if wanted() && !ACTIVE.swap(true, Ordering::AcqRel) { continue; } + return; + } +} + +/// Unbind, before the work item is disabled: makes a running actuator exit. +pub(crate) fn begin_shutdown() { UNBINDING.store(true, Ordering::Release); } + +/// Unbind: after the work item is disabled, withdraw and remove requests. +pub(crate) fn shutdown() { + UNBINDING.store(true, Ordering::Release); + if !REQS.registered.load(Ordering::Acquire) { return; } + for i in 0..REQS.count.load(Ordering::Acquire) as usize { + // SAFETY: registered slot; the actuator work item is disabled. + unsafe { bindings::freq_qos_remove_request((*REQS.req.get()).as_mut_ptr().add(i)) }; + REQS.policy[i].store(core::ptr::null_mut(), Ordering::Relaxed); + } + REQS.registered.store(false, Ordering::Release); +} + +// --------------------------------------------------------------------------- +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} + +fn parse_u32(value: *const core::ffi::c_char) -> Option { + if value.is_null() { return None; } + let mut command = [0u8; 16]; + let mut len = 0; + while len < command.len() { + // SAFETY: NUL-terminated parameter string; at most 16 bytes examined. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return None; } + let text = core::str::from_utf8(&command[..len]).ok()?.trim(); + match text.strip_prefix("0x") { + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + } +} + +unsafe extern "C" fn mask_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(mask) = parse_u32(value) else { return -22; }; + if mask & !ALL_BITS != 0 { return -22; } + MASK.store(mask, Ordering::Release); + if mask & FEED_PERF == 0 { UNTIL_NS.store(0, Ordering::Release); STREAK.store(0, Ordering::Release); } + 0 +} +unsafe extern "C" fn mask_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1000) }; + let mut out = Output { bytes, used: 0 }; + let _ = writeln!(out, "{:#x} pct={} applied={} streak={} idle={} busy={} starved={} fed={} boosts={} boosted_ns={} active={} policies={} depth={} early={} settle_skip={} ring_hits={} tail={} obs_skip={} mirrors={} proof_hits={} proof_batches={} proof_fallbacks={} pct_p={} pct_e={} big_cpus={:#x} applied_khz={}/{} nudges={} nudge_fails={} fwdeps={}/{}/{} early_deps={} agg_first={}", + MASK.load(Ordering::Relaxed), FEED_PCT.load(Ordering::Relaxed), REQS.applied_pct.load(Ordering::Relaxed), + STREAK.load(Ordering::Relaxed), IDLE_EWMA.load(Ordering::Relaxed), BUSY_EWMA.load(Ordering::Relaxed), + STARVED.load(Ordering::Relaxed), FED.load(Ordering::Relaxed), + BOOSTS.load(Ordering::Relaxed), BOOSTED_NS.load(Ordering::Relaxed), ACTIVE.load(Ordering::Relaxed), + REQS.count.load(Ordering::Relaxed), EARLY_DEPTH.load(Ordering::Relaxed), + EARLY_TAKEN.load(Ordering::Relaxed), + SETTLE_SKIPPED.load(Ordering::Relaxed), RING_CACHE_HITS.load(Ordering::Relaxed), + TAIL_MERGED.load(Ordering::Relaxed), OBS_SKIPPED.load(Ordering::Relaxed), + MIRRORS.load(Ordering::Relaxed), PROOF_HITS.load(Ordering::Relaxed), + PROOF_BATCHES.load(Ordering::Relaxed), PROOF_FALLBACKS.load(Ordering::Relaxed), + FEED_PCT_P.load(Ordering::Relaxed), FEED_PCT_E.load(Ordering::Relaxed), + REQS.big_cpus.load(Ordering::Relaxed), REQS.applied_khz[0].load(Ordering::Relaxed), + REQS.applied_khz[1].load(Ordering::Relaxed), NUDGES.load(Ordering::Relaxed), + NUDGE_FAILS.load(Ordering::Relaxed), FW_DEP_CONVERTED.load(Ordering::Relaxed), + FW_DEP_HOST.load(Ordering::Relaxed), FW_DEP_SATISFIED.load(Ordering::Relaxed), + EARLY_WITH_DEPS.load(Ordering::Relaxed), AGG_FIRST_TAKEN.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} +unsafe extern "C" fn pct_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(pct) = parse_u32(value) else { return -22; }; + if pct == 0 || pct > 100 { return -22; } + FEED_PCT.store(pct, Ordering::Release); + 0 +} +unsafe extern "C" fn pct_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 64) }; + let mut out = Output { bytes, used: 0 }; + let _ = writeln!(out, "{}", FEED_PCT.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} +static MASK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(mask_set), get: Some(mask_get), free: None, +}; +static PCT_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(pct_set), get: Some(pct_get), free: None, +}; +unsafe extern "C" fn depth_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(depth) = parse_u32(value) else { return -22; }; + if depth == 0 || depth > EARLY_DEPTH_MAX { return -22; } + EARLY_DEPTH.store(depth, Ordering::Release); + 0 +} +unsafe extern "C" fn depth_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 64) }; + let mut out = Output { bytes, used: 0 }; + let _ = writeln!(out, "{}", EARLY_DEPTH.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} +static DEPTH_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(depth_set), get: Some(depth_get), free: None, +}; +unsafe extern "C" fn pct_p_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(pct) = parse_u32(value) else { return -22; }; + if pct > 100 { return -22; } + FEED_PCT_P.store(pct, Ordering::Release); + 0 +} +unsafe extern "C" fn pct_e_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(pct) = parse_u32(value) else { return -22; }; + if pct > 100 { return -22; } + FEED_PCT_E.store(pct, Ordering::Release); + 0 +} +fn pct_get_from(counter: &AtomicU32, buf: *mut core::ffi::c_char) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 64) }; + let mut out = Output { bytes, used: 0 }; + let _ = writeln!(out, "{}", counter.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} +unsafe extern "C" fn pct_p_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + pct_get_from(&FEED_PCT_P, buf) +} +unsafe extern "C" fn pct_e_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + pct_get_from(&FEED_PCT_E, buf) +} +static PCT_P_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(pct_p_set), get: Some(pct_p_get), free: None, +}; +static PCT_E_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(pct_e_set), get: Some(pct_e_get), free: None, +}; +#[link_section = "__param"] +#[used] +static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_host_path" + } else { c"asahi.g17p_host_path" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&MASK_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +#[link_section = "__param"] +#[used] +static PCT_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_feed_perf_pct" + } else { c"asahi.g17p_feed_perf_pct" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PCT_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +#[link_section = "__param"] +#[used] +static DEPTH_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_early_depth" + } else { c"asahi.g17p_early_depth" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&DEPTH_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +#[link_section = "__param"] +#[used] +static PCT_P_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_feed_perf_pct_p" + } else { c"asahi.g17p_feed_perf_pct_p" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PCT_P_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +#[link_section = "__param"] +#[used] +static PCT_E_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_feed_perf_pct_e" + } else { c"asahi.g17p_feed_perf_pct_e" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PCT_E_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/mem_path.rs b/drivers/gpu/drm/asahi/g18/compat/mem_path.rs new file mode 100644 index 0000000000000..535ab69caeadc --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/mem_path.rs @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! v94 memory-path controls and stall counters (`g17p_mem_path`). +//! +//! Root-only sysfs, runtime-writable bitmask; default 0 is exactly v93f. +//! bit0 ASYNC_CLOSED_DRAIN: GEM objects whose handles closed while the VM +//! had accepted jobs are parked (their mappings and backing retained) +//! until the VM's job count reaches zero. v93f then unmapped and freed +//! all of them inline in the last job's `T8140VmJobGuard` drop, which +//! runs inside that job's completion before its output fence signals, +//! and it held the VM's `draining` gate for the whole drain, stalling +//! every new submission (`retain_t8140_job`) and every new map +//! (`wait_for_user_map_admission`). With this bit, when the parked work +//! is only closed objects on a healthy VM (not quarantined or closed, no +//! deferred unmaps or close ranges), the list captured at that idle +//! instant is released on the unbound workqueue without the gate. Jobs +//! accepted later cannot name a closed handle, and a reused VA has +//! already been remapped by GPUVM, so no accepted job can reach what the +//! worker unmaps. Quarantine and VM close keep the gated path. +//! bit1 MEM_STATS: relaxed counters and maxima for the paths above, GEM +//! creation, VM_BIND phases and admission waits (printed on read). +//! bit2 COMPUTE_TS_POOL: compute completion-timestamp pages are recycled +//! per queue instead of allocating, zeroing, GPU-mapping and later +//! unmapping a fresh GEM page for every compute command. +//! bit3 REACH_LOCKLESS: the retained-graph reachability revalidation that +//! every render publication runs (`first_unreachable_cached`) walks +//! the app and kernel VM change histories without taking either +//! GPUVM exec lock (a drm_exec transaction with a 16 KiB kvmalloc and +//! the VM reservation ww_mutex). Writers record the history lock-free +//! while the bit is set; a refused lock-free walk (writer active, +//! history lost, entry not yet recorded) takes the v93f locked walk, +//! which, with this bit, also serves a publication that found a writer +//! in progress instead of a full page-table walk. +//! MEM_STATS additionally counts VM mapping changes per class and the +//! reachability paths taken (exact / lock-free walk / locked walk / full). +//! Boot-only `g17p_gem_huge` (asahi.rs) is reported here too. +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const ASYNC_CLOSED_DRAIN: u32 = 1 << 0; +pub(crate) const MEM_STATS: u32 = 1 << 1; +pub(crate) const COMPUTE_TS_POOL: u32 = 1 << 2; +pub(crate) const REACH_LOCKLESS: u32 = 1 << 3; +const ALL_BITS: u32 = (1 << 4) - 1; + +static MASK: AtomicU32 = AtomicU32::new(0); +/// Set at probe when the huge GEM mount was created. +pub(crate) static HUGE_MOUNT: AtomicU32 = AtomicU32::new(0); + +#[inline(always)] +pub(crate) fn enabled(bit: u32) -> bool { MASK.load(Ordering::Relaxed) & bit != 0 } + +/// CNTPCT (1 GHz on Neo) when MEM_STATS is set, else 0. +#[inline(always)] +pub(crate) fn stamp() -> u64 { + if enabled(MEM_STATS) { crate::g18::compat::timing_buffer::arch_counter() } else { 0 } +} + +/// One counted event with a duration: count, total and maximum. +pub(crate) struct Stat { count: AtomicU64, total: AtomicU64, max: AtomicU64 } +impl Stat { + pub(crate) const fn new() -> Self { + Self { count: AtomicU64::new(0), total: AtomicU64::new(0), max: AtomicU64::new(0) } + } + /// Record `now - start` when `start` came from `stamp()` (0 = disabled). + #[inline(always)] + pub(crate) fn since(&self, start: u64) { + if start == 0 { return; } + self.add(crate::g18::compat::timing_buffer::arch_counter().saturating_sub(start)); + } + pub(crate) fn add(&self, ticks: u64) { + self.count.fetch_add(1, Ordering::Relaxed); + self.total.fetch_add(ticks, Ordering::Relaxed); + self.max.fetch_max(ticks, Ordering::Relaxed); + } + fn reset(&self) { + self.count.store(0, Ordering::Relaxed); + self.total.store(0, Ordering::Relaxed); + self.max.store(0, Ordering::Relaxed); + } + fn print(&self, out: &mut Output<'_>, name: &str) { + let _ = write!(out, " {}={}/{}/{}", name, self.count.load(Ordering::Relaxed), + self.total.load(Ordering::Relaxed), self.max.load(Ordering::Relaxed)); + } +} + +/// Inline (v93f) and asynchronous closed-object drains, per drain. +pub(crate) static DRAIN_INLINE: Stat = Stat::new(); +pub(crate) static DRAIN_ASYNC: Stat = Stat::new(); +/// Objects released by either drain. +pub(crate) static DRAIN_OBJECTS: AtomicU64 = AtomicU64::new(0); +/// Objects parked at close because jobs were active. +pub(crate) static PARKED: AtomicU64 = AtomicU64::new(0); +/// Waits for the VM `draining` gate: new submissions and new maps. +pub(crate) static RETAIN_WAIT: Stat = Stat::new(); +pub(crate) static ADMISSION_WAIT: Stat = Stat::new(); +/// VM_BIND ioctl: whole call, validation+backing (pages) phase, commit phase. +pub(crate) static BIND_CALL: Stat = Stat::new(); +pub(crate) static BIND_PREPARE: Stat = Stat::new(); +pub(crate) static BIND_COMMIT: Stat = Stat::new(); +pub(crate) static BIND_OPS: AtomicU64 = AtomicU64::new(0); +pub(crate) static BIND_BYTES: AtomicU64 = AtomicU64::new(0); +/// GEM_CREATE ioctl. +pub(crate) static GEM_CREATE: Stat = Stat::new(); +pub(crate) static GEM_BYTES: AtomicU64 = AtomicU64::new(0); +/// COMPUTE_TS_POOL: pages reused, and freshly allocated. +pub(crate) static TS_REUSED: AtomicU64 = AtomicU64::new(0); +pub(crate) static TS_ALLOCATED: AtomicU64 = AtomicU64::new(0); + +/// Retained-graph reachability (per render publication): exact epoch match, +/// lock-free history walk, locked history walk, full page-table walk. +pub(crate) static REACH_EXACT: Stat = Stat::new(); +pub(crate) static REACH_LOCKLESS_WALK: Stat = Stat::new(); +pub(crate) static REACH_LOCKED_WALK: Stat = Stat::new(); +pub(crate) static REACH_FULL: Stat = Stat::new(); +/// History entries examined by either walk. +pub(crate) static REACH_WALKED: AtomicU64 = AtomicU64::new(0); +/// Why a full walk ran: no cached proof for the generation, a writer was in +/// progress (no key), the history walk refused, a change touched the graph. +pub(crate) static REACH_FULL_NO_PROOF: AtomicU64 = AtomicU64::new(0); +pub(crate) static REACH_FULL_NO_KEY: AtomicU64 = AtomicU64::new(0); +pub(crate) static REACH_FULL_REFUSED: AtomicU64 = AtomicU64::new(0); +pub(crate) static REACH_FULL_TOUCHED: AtomicU64 = AtomicU64::new(0); +/// Mapping-validation epoch bumps: kernel VM; user VM inside the kernel's +/// timestamp-alias window (compute completion pages); user VM elsewhere. +pub(crate) static VM_CHG_KERNEL: AtomicU64 = AtomicU64::new(0); +pub(crate) static VM_CHG_TS_ALIAS: AtomicU64 = AtomicU64::new(0); +pub(crate) static VM_CHG_USER: AtomicU64 = AtomicU64::new(0); + +/// Called from `VmInner::note_mapping_validation_change` when MEM_STATS is set. +pub(crate) fn note_vm_change(kernel: bool, start: u64) { + let counter = if kernel { + &VM_CHG_KERNEL + } else if crate::g18::compat::g17p::T8140_TIMESTAMP_ALIAS_RANGE.contains(&start) { + &VM_CHG_TS_ALIAS + } else { + &VM_CHG_USER + }; + counter.fetch_add(1, Ordering::Relaxed); +} + +#[inline(always)] +pub(crate) fn count(counter: &AtomicU64, value: u64) { + if enabled(MEM_STATS) { counter.fetch_add(value, Ordering::Relaxed); } +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} + +fn parse<'a>(value: *const core::ffi::c_char) -> Option<&'a str> { + if value.is_null() { return None; } + // SAFETY: the parameter core supplies a NUL-terminated string. + let text = unsafe { core::ffi::CStr::from_ptr(value) }.to_str().ok()?; + Some(text.trim()) +} + +unsafe extern "C" fn mask_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + let Some(text) = parse(value) else { return -22; }; + if text == "reset" { + for stat in [&DRAIN_INLINE, &DRAIN_ASYNC, &RETAIN_WAIT, &ADMISSION_WAIT, &BIND_CALL, + &BIND_PREPARE, &BIND_COMMIT, &GEM_CREATE, &REACH_EXACT, &REACH_LOCKLESS_WALK, + &REACH_LOCKED_WALK, &REACH_FULL] { + stat.reset(); + } + for counter in [&DRAIN_OBJECTS, &PARKED, &BIND_OPS, &BIND_BYTES, &GEM_BYTES, &TS_REUSED, + &TS_ALLOCATED, &REACH_WALKED, &REACH_FULL_NO_PROOF, &REACH_FULL_NO_KEY, + &REACH_FULL_REFUSED, &REACH_FULL_TOUCHED, &VM_CHG_KERNEL, &VM_CHG_TS_ALIAS, + &VM_CHG_USER] { + counter.store(0, Ordering::Relaxed); + } + return 0; + } + let parsed = match text.strip_prefix("0x") { + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + }; + let Some(mask) = parsed else { return -22; }; + if mask & !ALL_BITS != 0 { return -22; } + MASK.store(mask, Ordering::Release); + 0 +} + +unsafe extern "C" fn mask_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes, used: 0 }; + let _ = write!(out, "{:#x} huge_mount={} parked={} drained_objects={} bind_ops={} bind_bytes={} gem_bytes={} ts_reused={} ts_allocated={} (stats: count/total_ns/max_ns)", + MASK.load(Ordering::Relaxed), HUGE_MOUNT.load(Ordering::Relaxed), + PARKED.load(Ordering::Relaxed), DRAIN_OBJECTS.load(Ordering::Relaxed), + BIND_OPS.load(Ordering::Relaxed), BIND_BYTES.load(Ordering::Relaxed), + GEM_BYTES.load(Ordering::Relaxed), TS_REUSED.load(Ordering::Relaxed), + TS_ALLOCATED.load(Ordering::Relaxed)); + DRAIN_INLINE.print(&mut out, "drain_inline"); + DRAIN_ASYNC.print(&mut out, "drain_async"); + RETAIN_WAIT.print(&mut out, "retain_wait"); + ADMISSION_WAIT.print(&mut out, "admission_wait"); + BIND_CALL.print(&mut out, "bind_call"); + BIND_PREPARE.print(&mut out, "bind_prepare"); + BIND_COMMIT.print(&mut out, "bind_commit"); + GEM_CREATE.print(&mut out, "gem_create"); + let _ = write!(out, " vm_chg_kernel={} vm_chg_ts_alias={} vm_chg_user={} reach_walked={} full_no_proof={} full_no_key={} full_refused={} full_touched={}", + VM_CHG_KERNEL.load(Ordering::Relaxed), VM_CHG_TS_ALIAS.load(Ordering::Relaxed), + VM_CHG_USER.load(Ordering::Relaxed), REACH_WALKED.load(Ordering::Relaxed), + REACH_FULL_NO_PROOF.load(Ordering::Relaxed), REACH_FULL_NO_KEY.load(Ordering::Relaxed), + REACH_FULL_REFUSED.load(Ordering::Relaxed), REACH_FULL_TOUCHED.load(Ordering::Relaxed)); + REACH_EXACT.print(&mut out, "reach_exact"); + REACH_LOCKLESS_WALK.print(&mut out, "reach_lockless"); + REACH_LOCKED_WALK.print(&mut out, "reach_locked"); + REACH_FULL.print(&mut out, "reach_full"); + let _ = writeln!(out); + out.used as core::ffi::c_int +} + +static MASK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(mask_set), get: Some(mask_get), free: None, +}; +#[link_section = "__param"] +#[used] +static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_mem_path" + } else { c"asahi.g17p_mem_path" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&MASK_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/mod.rs b/drivers/gpu/drm/asahi/g18/compat/mod.rs new file mode 100644 index 0000000000000..e8f8c8e6b009e --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/mod.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Module 1134 diagnostic controls kept for its module-parameter interface. + +#[allow(dead_code)] +pub(crate) mod dep_stats; +#[allow(dead_code)] +pub(crate) mod dvfs_parity; +#[allow(dead_code)] +pub(crate) mod e2e; +#[allow(dead_code)] +pub(crate) mod gem_stats; +#[allow(dead_code)] +pub(crate) mod host_path; +#[allow(dead_code)] +pub(crate) mod mem_path; +#[allow(dead_code)] +pub(crate) mod queue_slots; +#[allow(dead_code)] +pub(crate) mod recovery_isolation; +#[allow(dead_code)] +pub(crate) mod recovery_path; +#[allow(dead_code)] +pub(crate) mod stackwatch; +#[allow(dead_code)] +pub(crate) mod timing_buffer; +#[allow(dead_code)] +pub(crate) mod g17p; diff --git a/drivers/gpu/drm/asahi/g18/compat/queue_slots.rs b/drivers/gpu/drm/asahi/g18/compat/queue_slots.rs new file mode 100644 index 0000000000000..62f57e101dcbb --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/queue_slots.rs @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Firmware queue-slot accounting (`g17p_queue_slots`, root-only, read-only +//! sysfs). Counter-only: relaxed atomics, no printk. +//! +//! Every render pair and compute queue leases QIDs from one 128-entry table +//! (`g17_channel`). A published QID is kept until processor stop, and a +//! quarantined or exit-killed compute graph is never handed out again, so a +//! boot can run out of QIDs while memory is fine. Queue creation then fails +//! with ENOSPC, and so does a new render pair (`render-warm-pairs`). The +//! `qids` fields are exact (kept by the table itself); the `compute`, +//! `teardowns` and `render_quarantined` fields are a snapshot taken on every +//! teardown turn and on every queue creation. +//! +//! Every ENOSPC from one of these pools is counted by pool and named in +//! dmesg (the first one, then each power of two), so a failed QUEUE_CREATE +//! or submit says which pool ran out. +use core::fmt::Write; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam, prelude::*}; + +use crate::g18::compat::g17p::{self as g17_channel, QID_COUNT}; + +static COMPUTE_GRAPHS: AtomicU32 = AtomicU32::new(0); +static COMPUTE_OWNED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_RELEASED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_QUARANTINED: AtomicU32 = AtomicU32::new(0); +static COMPUTE_EXIT_KILLED: AtomicU32 = AtomicU32::new(0); +static RETIRED_GRAPHS: AtomicU32 = AtomicU32::new(0); +static TEARDOWNS: AtomicU32 = AtomicU32::new(0); +static RENDER_QUARANTINED: AtomicU32 = AtomicU32::new(0); + +/// Pools whose exhaustion returns ENOSPC to a client. +#[derive(Clone, Copy)] +pub(crate) enum Pool { + /// A render slot's first use leases two QIDs (`render-warm-pairs`). + QidRenderPair, + /// A new compute graph leases one QID (queue creation / first compute). + QidCompute, + /// A new logical queue context (QUEUE_CREATE): its only 128-wide pool + /// is the QoS hardware-buffer IDs, held by logical queues and installed + /// graph bindings. + QueueContextQos, + /// Any other submit-path ENOSPC (TSQ render slots, retained scenes, + /// compute item ring), named by its submit stage in the log line. + SubmitOther, +} + +const POOL_COUNT: usize = 4; +const POOL_NAMES: [&str; POOL_COUNT] = ["qid-render-pair", "qid-compute", "queue-context-qos", "submit-other"]; +static POOL_ENOSPC: [AtomicU64; POOL_COUNT] = + [AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0), AtomicU64::new(0)]; +/// Submits that failed with ENOSPC, whichever pool. +static SUBMIT_ENOSPC: AtomicU64 = AtomicU64::new(0); +static SUBMIT_FAILURES: AtomicU64 = AtomicU64::new(0); +static CONTEXT_UNBOUND_REFUSALS: AtomicU64 = AtomicU64::new(0); + +pub(crate) fn note_submit_failure() { + SUBMIT_FAILURES.fetch_add(1, Ordering::Relaxed); +} + +pub(crate) fn note_context_unbound_refusal() { + CONTEXT_UNBOUND_REFUSALS.fetch_add(1, Ordering::Relaxed); +} + +fn log_enospc(pool: &str, stage: &str, count: u64) { + // First one, then each power of two: a stuck pool cannot flood dmesg. + if count.is_power_of_two() { + pr_warn!( + "G17P ENOSPC pool={} stage={} (#{}): qids leased={}/{} compute={} compute-quarantined={} \ + reserve-refused={}\n", + pool, stage, count, g17_channel::leased_total(), QID_COUNT, + g17_channel::LEASED[2].load(Ordering::Relaxed), + COMPUTE_QUARANTINED.load(Ordering::Relaxed), + g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), + ); + } +} + +/// Count and name one ENOSPC from `pool`, and return it. +#[cold] +pub(crate) fn enospc(pool: Pool) -> Error { + let count = POOL_ENOSPC[pool as usize].fetch_add(1, Ordering::Relaxed) + 1; + log_enospc(POOL_NAMES[pool as usize], "-", count); + ENOSPC +} + +/// A submit failed with ENOSPC at `stage`. Pools tagged at their source +/// were already counted and named; this names the rest by stage. +#[cold] +pub(crate) fn note_submit_enospc(stage: &str) { + SUBMIT_ENOSPC.fetch_add(1, Ordering::Relaxed); + if stage != "render-warm-pairs" { + let other = POOL_ENOSPC[Pool::SubmitOther as usize].fetch_add(1, Ordering::Relaxed) + 1; + log_enospc(POOL_NAMES[Pool::SubmitOther as usize], stage, other); + } +} + +/// [`enospc`] for a call that may fail for other reasons too. +pub(crate) fn note_error(pool: Pool, error: Error) { + if error == ENOSPC { + enospc(pool); + } +} + +/// Snapshot of the runtime's queue-slot holders. +pub(crate) struct Snapshot { + pub(crate) compute_graphs: usize, + pub(crate) compute_owned: usize, + pub(crate) compute_released: usize, + pub(crate) compute_quarantined: usize, + pub(crate) compute_exit_killed: usize, + pub(crate) retired_graphs: usize, + pub(crate) teardowns: usize, + pub(crate) render_quarantined: usize, +} + +pub(crate) fn note(snapshot: Snapshot) { + let set = |gauge: &AtomicU32, value: usize| { + gauge.store(u32::try_from(value).unwrap_or(u32::MAX), Ordering::Relaxed) + }; + set(&COMPUTE_GRAPHS, snapshot.compute_graphs); + set(&COMPUTE_OWNED, snapshot.compute_owned); + set(&COMPUTE_RELEASED, snapshot.compute_released); + set(&COMPUTE_QUARANTINED, snapshot.compute_quarantined); + set(&COMPUTE_EXIT_KILLED, snapshot.compute_exit_killed); + set(&RETIRED_GRAPHS, snapshot.retired_graphs); + set(&TEARDOWNS, snapshot.teardowns); + set(&RENDER_QUARANTINED, snapshot.render_quarantined); +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +unsafe extern "C" fn slots_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let r = |c: &AtomicU32| c.load(Ordering::Relaxed); + let e = |pool: usize| POOL_ENOSPC[pool].load(Ordering::Relaxed); + let leased = g17_channel::leased_total(); + let published: u32 = g17_channel::PUBLISHED.iter().map(r).sum(); + let _ = writeln!(out, + "qids leased={} free={} published={} tiling={} fragment={} compute={} high_water={} exhausted={} \ + reserve={} reserve_refused={} \ + compute graphs={} owned={} released={} quarantined={} exit_killed={} retired_graphs={} \ + teardowns={} render_quarantined={} \ + enospc qid-render-pair={} qid-compute={} queue-context-qos={} submit-other={} submits={}", + leased, (QID_COUNT as u32).saturating_sub(leased), published, + r(&g17_channel::LEASED[0]), r(&g17_channel::LEASED[1]), r(&g17_channel::LEASED[2]), + r(&g17_channel::HIGH_WATER), g17_channel::EXHAUSTED.load(Ordering::Relaxed), + g17_channel::COMPUTE_RENDER_RESERVE, g17_channel::RESERVE_REFUSED.load(Ordering::Relaxed), + r(&COMPUTE_GRAPHS), r(&COMPUTE_OWNED), r(&COMPUTE_RELEASED), r(&COMPUTE_QUARANTINED), + r(&COMPUTE_EXIT_KILLED), r(&RETIRED_GRAPHS), r(&TEARDOWNS), r(&RENDER_QUARANTINED), + e(0), e(1), e(2), e(3), SUBMIT_ENOSPC.load(Ordering::Relaxed)); + let _ = writeln!(out, "submit failures={} context-unbound-refusals={}", + SUBMIT_FAILURES.load(Ordering::Relaxed), + CONTEXT_UNBOUND_REFUSALS.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} + +unsafe extern "C" fn slots_set(_: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + -1 // EPERM: read-only counters +} + +static SLOTS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(slots_set), get: Some(slots_get), free: None, +}; +#[link_section = "__param"] +#[used] +static SLOTS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_queue_slots" + } else { c"asahi.g17p_queue_slots" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&SLOTS_OPS), perm: 0o400, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/recovery_isolation.rs b/drivers/gpu/drm/asahi/g18/compat/recovery_isolation.rs new file mode 100644 index 0000000000000..933ad025526f7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/recovery_isolation.rs @@ -0,0 +1,644 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Recovery isolation (`g17p_recovery_isolation`): after a firmware +//! stamp-blamed recovery only the guilty client loses. +//! +//! Root-only sysfs, runtime-writable bitmask; also settable at load. Writing +//! 0 restores the accepted Y behaviour exactly (no completion is ever marked, +//! no admission wait is taken). Writing "reset" clears the counters. +//! +//! Hardware evidence (saved kernel logs, 2026-09-30/10-01): a stamp-blamed +//! recovery cancels in-flight work of EVERY client, and Y then poisons each +//! affected VM's sticky status, which Mesa turns into VK_ERROR_DEVICE_LOST: +//! * mmuf2 01:10:21 -- blamed qid 10 (Hyprland, VM 15). Xwayland's compute +//! qid 7 (VM 17) retired zero-duration -> `fail_while_owned(ENODATA)` +//! -> VM 17 status -61 -> every map refused -> Xwayland died. +//! * hypr-crash-0224 02:24:31 -- blamed qid 5 (another client). Hyprland's +//! parked 3D (qid 14) never ran after the recovery; its 2 s timeout +//! retired it unexecuted -> `complete(Err(EIO))` -> VM 43 status -5 -> +//! Hyprland aborted. +//! +//! Recovery-info entry layout. Native decode (static RE of the J700 kext, +//! `AGXArmFirmware::processFirmwareInitiatedRecovery` in +//! neo-stability-20260930/native/kext25.dis @fffffe00088a8398): it walks all +//! 256 8-byte entries, skips those without bit0, takes qid = bits 4..10 +//! (`ubfx w24, w22, #4, #7`), looks up the command by (qid, word1) and stores +//! a per-command error code: 5 when bit1 is clear, otherwise 3 (or 0xb when +//! the restart reason is 3). In public macOS logs 5 is +//! kIOGPUCommandBufferCallbackErrorInnocentVictim, 3 Hang, 0xb PageFault +//! (names from macOS log strings, NOT from this binary). So bit1 = GUILTY, +//! and native fails every listed non-guilty command with InnocentVictim -- +//! per command buffer, no replay seen in this function. Linux captures agree: +//! bit1 is set exactly on the queue the host's blamed stamp names in 4/4 +//! recoveries. bit2 is set on queues the firmware cancelled, bit3 on TA +//! queues (captures; native only forwards bit2 into its restart report). +//! +//! bit0 INNOCENT: classify every in-flight completion at each recovery. +//! A VM owns none of the guilty queues (blamed stamp + entries with +//! bit1) -> its cancelled work fails with an error fence ONLY: no +//! sticky VM status, no VM-wide fan-out. Its compute queue, once the +//! existing firmware-retirement witness proves the cancelled kicks +//! retired, returns to service instead of staying quarantined for the +//! boot. Unresolvable blame (a guilty qid whose owner cannot be found) +//! disables isolation for that recovery: today's behaviour. +//! bit1 ADMISSION_WAIT: a VM whose status is clean but whose mappings are +//! transiently blocked by retained (quarantined) work waits up to +//! ADMISSION_WAIT_MS for the release instead of refusing VM_BIND, and +//! a compute submit waits up to SUBMIT_WAIT_MS for the retirement +//! witness of its own spared queue. Poisoned VMs still refuse at once. +//! (iso3) A render submit whose owner has every pair quarantined +//! waits up to ADMISSION_WAIT_MS for the reclaim that returns the +//! pair, instead of Y's immediate EIO, which poisons the VM through +//! `fail_unpublished`. Field case: boot d324fcdd 05:38:44, spared VM +//! 2559's pair (slot 11) was quarantined at .370767, its next render +//! failed at .371211 and poisoned the VM, and the reclaim returned the +//! pair at .373928. An owner with a healthy pair never waits here. +//! bit2 SETTLE_CANCELLED: spared renders the firmware listed as CANCELLED +//! (bit2, not bit1) are settled at the end of the recovery through the +//! same exact-once settle the blamed slot uses, instead of waiting for +//! the 2 s scheduler watchdog. A group firmware has not provably +//! retired keeps its pair quarantined (mappings retained) until the +//! reclaim witness. Listed-only entries (bit2 clear) are left alone. +//! bit3 GUILTY_GRACE: the guilty VM is also spared, at most once per +//! GRACE_WINDOW_NS per VM; a second guilty recovery inside the window +//! poisons it as today. Opt-in: this drops the guilty submission +//! silently instead of reporting device loss. +//! bit4 RESUME_DOORBELL (opt-in, A/B): when spared render work the firmware +//! did not cancel is still pending after the recovery, send ONE paired +//! render doorbell (the message every render submission ends with; +//! it names no queue) so firmware rescans its rings. 02:24 evidence: +//! Hyprland's parked 3D (entry flags 0x1, not cancelled) never ran in +//! the 2 s after the recovery. Whether a doorbell activates it is the +//! question this bit answers; it is not established. +//! +//! bit6 REPLAY (iso4, default on): Vulkan has no per-command-buffer error +//! (native Metal reports InnocentVictim per command buffer; a Vulkan +//! client can only be told "device lost" or be handed wrong results), +//! so a spared command that provably never started is published again +//! instead of failing. Compute: a spared zero-duration queue's kicks +//! are kept across the retirement witness; only if EVERY kept kick's +//! own completion words still show start == end (firmware retired it +//! without executing, or never reached it) after the witness, and the +//! queue was returned to service, are they published again, oldest +//! first, before any newer submission of that queue (the compute +//! backend waits up to SUBMIT_WAIT_MS for the witness). Fences stay +//! unsignalled and CLASS_SPARED throughout, so nothing on this path +//! can poison the VM; a refused or failed replay fails that kick and +//! all its successors exactly as before (ENODATA, spared). A command +//! is replayed at most MAX_REPLAYS times, and each replay grants its +//! scheduler job one fresh timeout period (the recovery consumed the +//! old one). A replayed front kick is never treated as an earlier +//! recovery's collateral: if a later recovery blames its QID, it ran on +//! its own after the replay and its owner is guilty (no replay loop). +//! A kick that a render already named as its native parent is not +//! replayed (see `ComputeReplayQueue::native_consumer`). +//! Renders are NOT replayed yet; spared renders failed after +//! firmware retirement are counted by whether their pass started +//! (`spared-render-started/-unstarted`) to size that work. +//! Evidence level: "start == end means never executed" is the driver's +//! existing zero-duration interpretation (`g17p_zero_duration`), and +//! kicks of one QID run in publication order, so a zero-duration front +//! kick implies none behind it ran. It is not independently decoded +//! from native firmware. Every in-flight kick snapshotted at recovery +//! in the saved logs (199 snapshots) had page words [0, 0]. +//! bit0 also covers inherited errors (iso4): a consumer whose dependency +//! failed fails with ECANCELED instead of the dependency's EIO / +//! ETIMEDOUT / ENODEV, so it is not charged as its own VM's device +//! loss. Since c209c3dc5, consumers of a producer with host-copied +//! timestamps wait for its completion fence, so they always see a +//! failed producer's error; a spared render retired by firmware +//! completes with EIO, which used to poison every consumer's VM. +//! +//! v2 (iso2): a recovery that blames a queue whose OLDEST in-flight work was +//! already spared by an earlier recovery treats it as that recovery's +//! collateral and spares its owner again (counter `collateral`); and the +//! first job guard of a submission takes exactly Y's path unless Y's path +//! refuses it (no extra locking on the hot path). +//! +//! v3 (iso3) bit5 ATTRIBUTION (default on): the firmware's blame is the +//! first set bit of the hardware-slot fault masks (B1 `FUN_5450`, see +//! `PRIMARY_STATUS_B_RECOVERY_SLOT_MASK_OFFSET`), scanned TA first (inferred +//! from captures), not the slot that stalled. +//! With two or more sources, if the blamed slot's pass provably never +//! started (its tiler never stamped the pass start, and no DM1 progress +//! record exists for it), the culprit is re-attributed to the other sources +//! whose pass provably started (stamped pass start). If none did, no client +//! is guilty. A blamed source +//! with any execution evidence, a single-source fault and every page fault +//! (reason 3) keep the firmware's blame. Field case: 06:02:19 boot d324fcdd, +//! mask 0x11: slot 0 = quickshell TA qid 9 (pass start 0, no record) was +//! blamed; slot 4 = Minecraft 3D qid 27 (pass started, DM1 record state 2) +//! had stalled. +//! +//! bit7 VICTIM_DEFER (t-1179, default on): a HOST scheduler timeout is +//! not a firmware recovery, so nothing above classifies it. Field case +//! (boot of 2026-10-06 06:47:56, census J700-003): a vkd3d-proton test +//! (owner 18911, qid 72, vm 25834) stopped retiring compute; its 2 s +//! watchdog quarantined it. gnome-shell's compute kick (owner 3, qid 4, +//! vm 5) had been published right behind it (submissions 3131223 and +//! 3131225) and never ran: global GPU progress moved by one in those +//! 6 s. Its own watchdog fired at 06:48:02, poisoned VM 5 with -110 and +//! failed its render slots; firmware retired BOTH queues at 06:48:05, +//! three seconds later, but VM 5 refused every later map and the +//! compositor aborted. So a compute timeout whose queue's oldest kick +//! was published after an unfinished kick of ANOTHER VM's queue is +//! treated as head-of-line blocked: its scheduler timeout is renewed, +//! at most VICTIM_MAX_RENEWALS times, and nothing else changes (a +//! firmware recovery meanwhile classifies its kicks as usual). If the +//! blocker is still unretired when the budget runs out, its in-flight +//! kicks are classified CLASS_SPARED before the quarantine: their +//! fences fail, the VM status is untouched and the queue returns after +//! the retirement witness. Without a blocker, or once the blocker +//! retired, a timeout is the queue's own and keeps today's behaviour +//! (VM poisoned). A queue quarantined without retirement tickets +//! never counts as a blocker: its kicks stay listed until processor +//! stop, whatever firmware did. Kicks of different QIDs share the +//! compute pipe in publication order -- the field case shows that +//! blocking on G17P and the same client pair shows it on G13 (a 3 s +//! kick delayed another process's tiny kick by 2.998 s), but it is not +//! decoded from firmware -- so the renewal is +//! bounded, and the oldest unretired kick never has a blocker: a stall +//! always charges at least that queue. +//! +//! Memory safety is unchanged in every mode: packets, VM-job guards and +//! quarantined pairs are retained and released only by the existing +//! witnesses (exact retirement, firmware drain/reclaim, processor stop). +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +#[cfg(not(test))] +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const INNOCENT: u32 = 1 << 0; +pub(crate) const ADMISSION_WAIT: u32 = 1 << 1; +pub(crate) const SETTLE_CANCELLED: u32 = 1 << 2; +pub(crate) const GUILTY_GRACE: u32 = 1 << 3; +pub(crate) const RESUME_DOORBELL: u32 = 1 << 4; +pub(crate) const ATTRIBUTION: u32 = 1 << 5; +pub(crate) const REPLAY: u32 = 1 << 6; +pub(crate) const VICTIM_DEFER: u32 = 1 << 7; +const ALL_BITS: u32 = INNOCENT | ADMISSION_WAIT | SETTLE_CANCELLED | GUILTY_GRACE | RESUME_DOORBELL + | ATTRIBUTION | REPLAY | VICTIM_DEFER; +/// Correctness/containment fixes with no hot-path cost are on; the guilty +/// grace changes the guilty client's semantics and stays opt-in. +pub(crate) const DEFAULT_MASK: u32 = INNOCENT | ADMISSION_WAIT | SETTLE_CANCELLED | ATTRIBUTION | REPLAY + | VICTIM_DEFER; + +pub(crate) const ADMISSION_WAIT_MS: u32 = 1000; +/// iso4 REPLAY: publications of one command beyond its first. +pub(crate) const MAX_REPLAYS: u8 = 3; +pub(crate) const SUBMIT_WAIT_MS: u32 = 200; +pub(crate) const GRACE_WINDOW_NS: u64 = 60_000_000_000; +/// bit7: scheduler periods (2 s each) a head-of-line blocked compute job may +/// wait for its blocker beyond its own. The field blocker retired 9 s after +/// its own timeout; 8 extra periods cover that with margin. +pub(crate) const VICTIM_MAX_RENEWALS: u8 = 8; + +static MASK: AtomicU32 = AtomicU32::new(DEFAULT_MASK); + +#[inline(always)] +pub(crate) fn enabled(bit: u32) -> bool { MASK.load(Ordering::Relaxed) & bit != 0 } + +#[inline(always)] +pub(crate) fn mask() -> u32 { MASK.load(Ordering::Relaxed) } + +// ---- REPLAY decision (bit6) --------------------------------------------------- + +/// Host state of one quarantined compute queue at its retirement witness. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct ComputeReplayQueue { + /// Every lost kick was spared and its fence is still unsignalled. + pub(crate) spared_deferred: bool, + /// The queue will be returned to service by this witness. + pub(crate) spared_quarantine: bool, + /// The owner has released (closed) the queue. + pub(crate) released: bool, + /// The queue still belongs to the witness ticket's owner. + pub(crate) owner_matches: bool, + /// A render named one of these kicks as its native parent. Firmware + /// retired the cancelled kick, which satisfies that barrier, so the + /// render may already have run without the kick's output; a replay would + /// then write after its consumer read. + pub(crate) native_consumer: bool, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum ComputeReplay { + /// Not a spared deferred queue, or the bits are off: unchanged path. + NotCandidate, + /// A candidate whose kicks cannot all be proven unexecuted: they fail + /// exactly as before (ENODATA, spared). + Refused, + /// Every kick provably never executed: keep them and publish again, + /// oldest first, once the queue is back in service. + Replay, +} + +/// A kick's own completion words, read after the retirement witness, prove +/// it never executed when start == end: firmware's zero-duration retirement +/// shape, or both words still at the zeroed allocation value. +pub(crate) const fn never_executed(words: [u64; 2]) -> bool { words[0] == words[1] } + +/// `kicks` yields, oldest first, each kick's completion words, or None when +/// its fence is already signalled or the words are unreadable. +pub(crate) fn compute_replay_decision(mask: u32, queue: ComputeReplayQueue, kicks: I) -> ComputeReplay +where I: IntoIterator> { + if mask & (REPLAY | INNOCENT) != (REPLAY | INNOCENT) + || !queue.spared_deferred || !queue.spared_quarantine + || queue.released || !queue.owner_matches + { + return ComputeReplay::NotCandidate; + } + if queue.native_consumer { + return ComputeReplay::Refused; + } + let mut any = false; + for words in kicks { + any = true; + match words { + Some(words) if never_executed(words) => {} + _ => return ComputeReplay::Refused, + } + } + if any { ComputeReplay::Replay } else { ComputeReplay::NotCandidate } +} + +/// An error a consumer inherits from a failed dependency (an input fence or +/// a firmware-ordered parent) is not the consumer's own device loss. With +/// INNOCENT set, the device-loss codes EIO, ETIMEDOUT and ENODEV become +/// ECANCELED for the consumer: its job fails, its VM keeps running. Errnos +/// are negative, as `Error::to_errno` returns them. +pub(crate) const fn inherited_errno(mask: u32, errno: i32) -> i32 { + const EIO: i32 = -5; + const ENODEV: i32 = -19; + const ETIMEDOUT: i32 = -110; + const ECANCELED: i32 = -125; + if mask & INNOCENT != 0 && (errno == EIO || errno == ETIMEDOUT || errno == ENODEV) { + ECANCELED + } else { + errno + } +} + +/// One more publication of a command that has been replayed `replays` times. +pub(crate) const fn replay_budget_left(replays: u8) -> bool { replays < MAX_REPLAYS } + +/// Per-completion classification, written only while servicing a recovery. +pub(crate) const CLASS_NONE: u8 = 0; +/// Cancelled by another client's recovery (or graced): fail the fence only. +pub(crate) const CLASS_SPARED: u8 = 1; +/// Owned by the guilty VM: today's behaviour. +pub(crate) const CLASS_GUILTY: u8 = 2; + +pub(crate) const ENTRY_VALID: u8 = 1 << 0; +pub(crate) const ENTRY_GUILTY: u8 = 1 << 1; +pub(crate) const ENTRY_CANCELLED: u8 = 1 << 2; + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct RecoveryEntry { + pub(crate) qid: u8, + pub(crate) flags: u8, + pub(crate) stamp: u32, +} + +/// Decode one recovery-info table entry (see the module comment). +pub(crate) const fn decode_entry(word0: u32, word1: u32) -> Option { + if word0 & 1 == 0 { return None; } + // Native: ubfx #4, #7. + let qid = (word0 >> 4) & 0x7f; + Some(RecoveryEntry { qid: qid as u8, flags: (word0 & 0xf) as u8, stamp: word1 >> 8 }) +} + +/// QID masks of a decoded table plus the blamed stamp: +/// (guilty, listed, cancelled). The blamed stamp's QID counts as guilty. +pub(crate) fn qid_masks(entries: &[RecoveryEntry], blamed: Option) -> (u128, u128, u128) { + let mut guilty = 0u128; + let mut listed = 0u128; + let mut cancelled = 0u128; + for entry in entries { + let bit = 1u128 << (entry.qid & 0x7f); + listed |= bit; + if entry.flags & ENTRY_GUILTY != 0 { guilty |= bit; } + if entry.flags & ENTRY_CANCELLED != 0 { cancelled |= bit; } + } + if let Some(qid) = blamed.filter(|qid| *qid < 128) { guilty |= 1u128 << qid; } + (guilty, listed, cancelled) +} + +/// Class of one VM for one recovery. Returns (class, grace consumed). +pub(crate) const fn classify(guilty: bool, grace: bool, now_ns: u64, last_grace_ns: u64) -> (u8, bool) { + if !guilty { return (CLASS_SPARED, false); } + if grace && (last_grace_ns == 0 || now_ns.saturating_sub(last_grace_ns) >= GRACE_WINDOW_NS) { + return (CLASS_SPARED, true); + } + (CLASS_GUILTY, false) +} + +// ---- head-of-line victim deferral (bit7) ------------------------------------ + +/// The oldest unretired kick of one OTHER compute queue at a host timeout. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct ComputeHead { + /// VM of that queue's owner. + pub(crate) vm: u64, + /// Global publication order of the kick (`COMPUTE_PUBLICATION`). + pub(crate) published: u64, + /// Its completion record was already latched (finished, not retired yet). + pub(crate) finished: bool, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum VictimTimeout { + /// No older unfinished kick of another VM: the timeout is this queue's own. + Own, + /// Head-of-line blocked: renew the scheduler timeout once more. + Defer, + /// Still blocked after the renewal budget: quarantine spared. + Spare, +} + +/// `own_vm`/`own_published`: the timed-out queue's VM and its oldest +/// unretired kick's publication order; `renewals`: periods already granted. +/// Returns the decision and the publication order of the oldest blocker. +pub(crate) fn victim_timeout(mask: u32, own_vm: u64, own_published: u64, renewals: u8, others: I) + -> (VictimTimeout, Option) +where I: IntoIterator { + if mask & VICTIM_DEFER == 0 { return (VictimTimeout::Own, None); } + let blocker = others.into_iter() + .filter(|head| head.vm != own_vm && !head.finished && head.published < own_published) + .map(|head| head.published) + .min(); + match blocker { + None => (VictimTimeout::Own, None), + Some(_) if renewals < VICTIM_MAX_RENEWALS => (VictimTimeout::Defer, blocker), + // Sparing needs the INNOCENT machinery (no-poison fail, witness return). + Some(_) if mask & INNOCENT != 0 => (VictimTimeout::Spare, blocker), + Some(_) => (VictimTimeout::Own, blocker), + } +} + +/// Global publication order of async compute kicks. Written only under the +/// runtime lock at publication; a plain counter, never reset. +pub(crate) static COMPUTE_PUBLICATION: AtomicU64 = AtomicU64::new(0); + +// ---- fault-source attribution (bit5) --------------------------------------- + +/// One hardware slot of the recovery's fault mask, sampled while firmware is +/// halted: the slot's cached key (qid << 40 | stamp), its DM1 progress-monitor +/// state (0 = never written), and whether its pass had started (None = not a +/// render pass the host can resolve). +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct FaultSource { + pub(crate) slot: u8, + pub(crate) qid: Option, + pub(crate) stamp: u64, + pub(crate) monitor_state: u32, + pub(crate) started: Option, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Attribution { + /// Keep the firmware's guilty set (blamed stamp + table bit1). + Firmware, + /// The blamed slot never ran; these QIDs' owners are guilty instead. + Reattributed(u128), + /// The blamed slot never ran and no other source shows execution. + NoCulprit, +} + +pub(crate) const fn decode_slot_key(key: u64) -> (Option, u64) { + if key == 0 { return (None, 0); } + let qid = (key >> 40) & 0xff; + (if qid < 128 { Some(qid as u8) } else { None }, key & 0xff_ffff_ffff) +} + +/// `blamed_qid` is the QID of the firmware's stamp-blamed request. Across 13 +/// saved recoveries the blamed slot is the lowest faulting slot of the first +/// data master, in TA, 3D, CDM order, that has one (e.g. 06:02:19 mask 0x11 +/// blamed TA slot 0 over 3D slot 4; 03:18:16 mask 0x3 blamed 3D slot 1 over +/// compute slot 0) -- an inference from captures, so the blamed source is +/// matched by QID, never by bit position. +pub(crate) fn attribute(reason: u32, blamed_qid: Option, mask: u32, sources: &[FaultSource]) -> Attribution { + // Page faults name the faulting VM directly; one source leaves no choice. + if reason == 3 || mask.count_ones() < 2 { return Attribution::Firmware; } + let Some(blamed_qid) = blamed_qid else { return Attribution::Firmware; }; + let Some(blamed) = sources.iter().find(|source| source.qid == Some(blamed_qid)) else { + return Attribution::Firmware; + }; + let blamed_slot = blamed.slot; + if blamed.started != Some(false) || blamed.monitor_state != 0 { + return Attribution::Firmware; + } + let mut qids = 0u128; + // A culprit must have provably run: its pass start is stamped. A DM1 + // progress record alone is not used for a culprit (records are historical + // DRAM and slots are shared across data masters, so a record may be stale). + for source in sources.iter().filter(|source| source.slot != blamed_slot) { + if source.started != Some(true) { continue; } + if let Some(qid) = source.qid { qids |= 1u128 << (qid & 0x7f); } + } + if qids != 0 { Attribution::Reattributed(qids) } else { Attribution::NoCulprit } +} + +pub(crate) const MAX_SOURCES: usize = 16; +static SRC_MASK: AtomicU32 = AtomicU32::new(0); +static SRC_REASON: AtomicU32 = AtomicU32::new(0); +static SRC_KEY: [AtomicU64; MAX_SOURCES] = [const { AtomicU64::new(0) }; MAX_SOURCES]; +static SRC_MONITOR: [AtomicU32; MAX_SOURCES] = [const { AtomicU32::new(0) }; MAX_SOURCES]; + +/// Store the halted-firmware sample (`keys`/`monitors` indexed by slot). +pub(crate) fn store_sources(reason: u32, mask: u32, keys: &[u64; MAX_SOURCES], monitors: &[u32; MAX_SOURCES]) { + for slot in 0..MAX_SOURCES { + SRC_KEY[slot].store(keys[slot], Ordering::Relaxed); + SRC_MONITOR[slot].store(monitors[slot], Ordering::Relaxed); + } + SRC_REASON.store(reason, Ordering::Relaxed); + SRC_MASK.store(mask, Ordering::Release); +} + +pub(crate) fn clear_sources() { SRC_MASK.store(0, Ordering::Release); } + +/// (reason, mask, sources without `started`) of the last stored sample. +pub(crate) fn sources(out: &mut [FaultSource; MAX_SOURCES]) -> (u32, u32, usize) { + let mask = SRC_MASK.load(Ordering::Acquire); + let reason = SRC_REASON.load(Ordering::Relaxed); + let mut used = 0; + for slot in 0..MAX_SOURCES { + if mask & (1u32 << slot) == 0 { continue; } + let (qid, stamp) = decode_slot_key(SRC_KEY[slot].load(Ordering::Relaxed)); + out[used] = FaultSource { + slot: slot as u8, qid, stamp, + monitor_state: SRC_MONITOR[slot].load(Ordering::Relaxed), started: None, + }; + used += 1; + } + (reason, mask, used) +} + +// ---- recovery-info snapshot ------------------------------------------------ + +pub(crate) const MAX_ENTRIES: usize = 64; +static ENTRY_WORDS: [AtomicU64; MAX_ENTRIES] = [const { AtomicU64::new(0) }; MAX_ENTRIES]; +static ENTRY_COUNT: AtomicU32 = AtomicU32::new(0); + +/// Forget the previous recovery's table before reading the new one. +pub(crate) fn clear_entries() { ENTRY_COUNT.store(0, Ordering::Release); } + +/// Store the raw (word0, word1) of every VALID entry, in table order. +pub(crate) fn store_entries(raw: &[(u32, u32)]) { + let count = raw.len().min(MAX_ENTRIES); + for (slot, &(word0, word1)) in ENTRY_WORDS.iter().zip(raw.iter()) { + slot.store(u64::from(word0) | u64::from(word1) << 32, Ordering::Relaxed); + } + ENTRY_COUNT.store(count as u32, Ordering::Release); +} + +/// Decoded entries of the last stored table. +pub(crate) fn entries(out: &mut [RecoveryEntry; MAX_ENTRIES]) -> usize { + let count = (ENTRY_COUNT.load(Ordering::Acquire) as usize).min(MAX_ENTRIES); + let mut used = 0; + for slot in ENTRY_WORDS.iter().take(count) { + let raw = slot.load(Ordering::Relaxed); + if let Some(entry) = decode_entry(raw as u32, (raw >> 32) as u32) { + out[used] = entry; + used += 1; + } + } + used +} + +// ---- counters ---------------------------------------------------------------- + +pub(crate) static RECOVERIES: AtomicU64 = AtomicU64::new(0); +pub(crate) static UNRESOLVED: AtomicU64 = AtomicU64::new(0); +pub(crate) static SPARED_RENDERS: AtomicU64 = AtomicU64::new(0); +pub(crate) static SPARED_COMPUTE: AtomicU64 = AtomicU64::new(0); +pub(crate) static GUILTY_MARKED: AtomicU64 = AtomicU64::new(0); +pub(crate) static GRACED: AtomicU64 = AtomicU64::new(0); +pub(crate) static POISON_SKIPPED: AtomicU64 = AtomicU64::new(0); +pub(crate) static FANOUT_SKIPPED: AtomicU64 = AtomicU64::new(0); +pub(crate) static SETTLED_CANCELLED: AtomicU64 = AtomicU64::new(0); +pub(crate) static QUEUES_RETURNED: AtomicU64 = AtomicU64::new(0); +pub(crate) static QUEUE_RETURN_FAILED: AtomicU64 = AtomicU64::new(0); +pub(crate) static FIRST_KICKS_OK: AtomicU64 = AtomicU64::new(0); +pub(crate) static FIRST_KICKS_FAILED: AtomicU64 = AtomicU64::new(0); +pub(crate) static ADMISSION_WAITS: AtomicU64 = AtomicU64::new(0); +pub(crate) static ADMISSION_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +pub(crate) static SUBMIT_WAITS: AtomicU64 = AtomicU64::new(0); +pub(crate) static SUBMIT_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +pub(crate) static RESUME_DOORBELLS: AtomicU64 = AtomicU64::new(0); +pub(crate) static COLLATERAL: AtomicU64 = AtomicU64::new(0); +pub(crate) static REATTRIBUTED: AtomicU64 = AtomicU64::new(0); +pub(crate) static NO_CULPRIT: AtomicU64 = AtomicU64::new(0); +/// iso3.1: unclassified renders queued behind a spared head, failed with it. +pub(crate) static SPARED_SUCCESSORS: AtomicU64 = AtomicU64::new(0); +/// iso4 REPLAY: spared never-started compute kicks published again, and +/// spared kicks that had to fail instead (ran, unprovable, or republish failed). +pub(crate) static REPLAYED_COMPUTE: AtomicU64 = AtomicU64::new(0); +pub(crate) static REPLAY_REFUSED_COMPUTE: AtomicU64 = AtomicU64::new(0); +/// iso4: spared renders failed after firmware retirement, split by whether +/// the pass ever started (pass-start stamp) -- the render replay candidates. +pub(crate) static SPARED_RENDER_UNSTARTED: AtomicU64 = AtomicU64::new(0); +pub(crate) static SPARED_RENDER_STARTED: AtomicU64 = AtomicU64::new(0); +/// iso4: replayed compute jobs given a fresh scheduler timeout, and compute +/// publications that waited for their own queue's witness/replay first. +pub(crate) static REPLAY_TIMEOUT_RENEWED: AtomicU64 = AtomicU64::new(0); +pub(crate) static REPLAY_ORDER_WAITS: AtomicU64 = AtomicU64::new(0); +pub(crate) static REPLAY_ORDER_TIMEOUTS: AtomicU64 = AtomicU64::new(0); +/// iso4: replays refused because a render had already been released +/// against a cancelled kick; consumer failures not charged to the VM. +pub(crate) static REPLAY_REFUSED_CONSUMER: AtomicU64 = AtomicU64::new(0); +pub(crate) static INHERITED_NOT_POISONED: AtomicU64 = AtomicU64::new(0); +/// bit7: compute timeouts renewed because another VM's older kick was still +/// unfinished, and queues quarantined spared after the renewal budget. +pub(crate) static VICTIM_DEFERRED: AtomicU64 = AtomicU64::new(0); +pub(crate) static VICTIM_SPARED: AtomicU64 = AtomicU64::new(0); + +#[inline(always)] +pub(crate) fn count(counter: &AtomicU64) { counter.fetch_add(1, Ordering::Relaxed); } + +static COUNTERS: [(&str, &AtomicU64); 33] = [ + ("recoveries", &RECOVERIES), ("unresolved", &UNRESOLVED), + ("spared-renders", &SPARED_RENDERS), ("spared-compute", &SPARED_COMPUTE), + ("guilty-marked", &GUILTY_MARKED), ("graced", &GRACED), + ("poison-skipped", &POISON_SKIPPED), ("fanout-skipped", &FANOUT_SKIPPED), + ("settled-cancelled", &SETTLED_CANCELLED), ("queues-returned", &QUEUES_RETURNED), + ("queue-return-failed", &QUEUE_RETURN_FAILED), ("first-kicks-ok", &FIRST_KICKS_OK), + ("first-kicks-failed", &FIRST_KICKS_FAILED), ("admission-waits", &ADMISSION_WAITS), + ("admission-timeouts", &ADMISSION_TIMEOUTS), ("submit-waits", &SUBMIT_WAITS), + ("submit-timeouts", &SUBMIT_TIMEOUTS), ("resume-doorbells", &RESUME_DOORBELLS), + ("collateral", &COLLATERAL), ("reattributed", &REATTRIBUTED), ("no-culprit", &NO_CULPRIT), + ("spared-successors", &SPARED_SUCCESSORS), + ("replayed-compute", &REPLAYED_COMPUTE), ("replay-refused-compute", &REPLAY_REFUSED_COMPUTE), + ("spared-render-unstarted", &SPARED_RENDER_UNSTARTED), ("spared-render-started", &SPARED_RENDER_STARTED), + ("replay-timeout-renewed", &REPLAY_TIMEOUT_RENEWED), ("replay-order-waits", &REPLAY_ORDER_WAITS), + ("replay-order-timeouts", &REPLAY_ORDER_TIMEOUTS), + ("replay-refused-consumer", &REPLAY_REFUSED_CONSUMER), + ("inherited-not-poisoned", &INHERITED_NOT_POISONED), + ("victim-deferred", &VICTIM_DEFERRED), ("victim-spared", &VICTIM_SPARED), +]; + +// ---- parameter ----------------------------------------------------------------- + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} + +pub(crate) fn parse_mask(text: &str) -> Option { + let parsed = match text.strip_prefix("0x") { + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + }?; + (parsed & !ALL_BITS == 0).then_some(parsed) +} + +#[cfg(not(test))] +unsafe extern "C" fn mask_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + // SAFETY: the parameter core supplies a NUL-terminated string. + let Ok(text) = unsafe { core::ffi::CStr::from_ptr(value) }.to_str() else { return -22; }; + let text = text.trim(); + if text == "reset" { + for (_, counter) in COUNTERS.iter() { counter.store(0, Ordering::Relaxed); } + return 0; + } + let Some(mask) = parse_mask(text) else { return -22; }; + MASK.store(mask, Ordering::Release); + 0 +} + +#[cfg(not(test))] +unsafe extern "C" fn mask_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes, used: 0 }; + let _ = write!(out, "{:#x}", MASK.load(Ordering::Relaxed)); + for (name, counter) in COUNTERS.iter() { + let _ = write!(out, " {}={}", name, counter.load(Ordering::Relaxed)); + } + let _ = writeln!(out); + out.used as core::ffi::c_int +} + +#[cfg(not(test))] +static MASK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(mask_set), get: Some(mask_get), free: None, +}; +#[cfg(not(test))] +#[link_section = "__param"] +#[used] +static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_recovery_isolation" + } else { c"asahi.g17p_recovery_isolation" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&MASK_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/recovery_path.rs b/drivers/gpu/drm/asahi/g18/compat/recovery_path.rs new file mode 100644 index 0000000000000..5ae0b7c7ac662 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/recovery_path.rs @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! v93r post-recovery controls and control-ring diagnostics (`g17p_recovery_path`). +//! +//! Root-only sysfs, runtime-writable bitmask, also settable at load. Default +//! is RECLAIM_DRAINED; 0 is exactly v93f. Without bit0 a client whose render +//! was failed by a stamp-blamed recovery keeps its whole VM -- every BO it +//! ever mapped -- pinned until both firmware processors stop, which in +//! practice is the next reboot. +//! bit0 RECLAIM_DRAINED: after a firmware stamp-blamed recovery the render +//! whose group could not be proven retired, and every render queued +//! behind it on the same physical pair, are failed with their guards +//! retained (`fail_while_owned`) and the pair is quarantined until both +//! firmware processors stop. With this bit, once the recovery is +//! closed (host and firmware handshake state 0) and the firmware has +//! demonstrably drained that pair -- both engines' post-maintenance +//! completion stamps cover the pair's last published ordinal and the +//! pair FList's completed-kick counter equals its submitted counter -- +//! the event worker retires the pair's trackers through the normal +//! paired-completion transaction (or, for a group the recovery +//! skipped, the existing firmware-retired classifier), releases the +//! retained VM-job and lease guards, and returns the pair. The client's +//! VM stays poisoned; only the physical pair, its FList/HWPB state and +//! the closed queue's teardown are recovered. +//! bit1 CONTROL_TRACE: remember opcode/argument/publish time of every +//! primary device-control record and the time of the last EP 0x21 +//! announcement; a control re-announce then names the oldest pending +//! record and whether it had been announced after its publication. +//! bit2 GROW_REKICK (v96): firmware event 6 (TVB grow) pauses the pool's +//! allocator until the opcode-8 reply. A request whose owning pair is +//! borrowed (`preparing`: warm-up, restage, grow, completion +//! observation) is parked, and the grow worker exits. Without this bit +//! only the next firmware event re-queues the worker; if the stalled +//! render was the only producer of events, nothing does. With this +//! bit, every end of a borrow (and every clearing of control-ring +//! backpressure) re-queues the worker while a request is parked. +//! bit3 reserved (v97 INNOCENT_VICTIM, not in this build; rejected). +//! bit4 STALL_WATCH (v97d, DIAGNOSTIC): while any render pair has +//! outstanding work, a 4 ms delayed work reads the firmware per-QID +//! dependency records (firmware DATA, read-only) of every busy pair. +//! A 3D queue is a stall candidate when its TA's completed stamp has +//! passed the 3D's next stamp (its TA parent is complete) while that +//! queue has not advanced for >= STALL_MS. Other clients' progress +//! must not hide starvation of this queue. The first sample of +//! a stall logs the dependency records, the B1 scheduler masks +//! (requested/sampled/deferred/next), the runnable mask and the park +//! record. Log and count only: no firmware write, no message. +//! bit5 STALL_REDOORBELL (v97d, DIAGNOSTIC, needs bit4): on a detected +//! stall re-send the exact paired render doorbell a submission ends +//! with (the only host action that activates a published render), +//! once per stall. Nothing is sent while a firmware recovery is +//! pending or the session is not accepting work. A doorbell names no +//! queue: firmware rescans its own rings, as after any submission. +//! bit6 STALL_FRAGMENT (v97d, DIAGNOSTIC, needs bits 4+5): if a stall is +//! still present STALL_MS after the paired doorbell, also send the +//! Fragment (3D) doorbell once. Native sends only the paired Vertex +//! doorbell per render, so this arm is a discriminator, never a fix. +//! Grow ingress, deferral, re-kick and recovery-time grow state are always +//! logged (rate-limited) and counted, independent of the mask. +use core::fmt::{self, Write}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const RECLAIM_DRAINED: u32 = 1 << 0; +pub(crate) const CONTROL_TRACE: u32 = 1 << 1; +pub(crate) const GROW_REKICK: u32 = 1 << 2; +pub(crate) const STALL_WATCH: u32 = 1 << 4; +pub(crate) const STALL_REDOORBELL: u32 = 1 << 5; +pub(crate) const STALL_FRAGMENT: u32 = 1 << 6; +const ALL_BITS: u32 = RECLAIM_DRAINED | CONTROL_TRACE | GROW_REKICK | STALL_WATCH | STALL_REDOORBELL + | STALL_FRAGMENT; + +const DEFAULT_MASK: u32 = RECLAIM_DRAINED; +static MASK: AtomicU32 = AtomicU32::new(DEFAULT_MASK); + +#[inline(always)] +pub(crate) fn enabled(bit: u32) -> bool { MASK.load(Ordering::Relaxed) & bit != 0 } + +/// Why a reclaim attempt for one quarantined pair stopped (counters and log). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ReclaimBlock { + /// A firmware recovery is open or the handshake state is not 0. + Recovery, + /// The pair is being prepared by a publisher. + Preparing, + /// A completion stamp does not yet cover the last published ordinal. + Stamps, + /// The pair FList still has submitted kicks not completed. + Flist, + /// A tracked group is not yet provably retired. + Tracker, + /// A step returned an error; the pair stays quarantined. + Error, +} + +impl ReclaimBlock { + const fn index(self) -> usize { + match self { + Self::Recovery => 0, Self::Preparing => 1, Self::Stamps => 2, + Self::Flist => 3, Self::Tracker => 4, Self::Error => 5, + } + } + const NAMES: [&'static str; 6] = ["recovery", "preparing", "stamps", "flist", "tracker", "error"]; +} + +/// Attempts, reclaimed pairs, retired trackers, released completions. +pub(crate) static RECLAIM_ATTEMPTS: AtomicU64 = AtomicU64::new(0); +pub(crate) static RECLAIM_PAIRS: AtomicU64 = AtomicU64::new(0); +pub(crate) static RECLAIM_TRACKERS: AtomicU64 = AtomicU64::new(0); +pub(crate) static RECLAIM_RELEASED: AtomicU64 = AtomicU64::new(0); +static RECLAIM_BLOCKED: [AtomicU64; 6] = [const { AtomicU64::new(0) }; 6]; + +/// v96 TVB grow accounting: firmware requests received, requests left parked +/// because their pair was borrowed when the worker looked, borrow ends that +/// found a parked request, and worker re-queues issued for them (bit2). +pub(crate) static GROW_INGRESS: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_DEFERRED: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_PARKED_AT_RETURN: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_REKICKS: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_REPLIED: AtomicU64 = AtomicU64::new(0); + +/// Handshake steps 5-6: the last opcode-8 grow reply published to the +/// primary control ring (bit63 valid | producer_after << 32 | buffer_slot) +/// and its CNTPCT, until the firmware consumer is observed past it. +pub(crate) static GROW_REPLY_WATCH: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_REPLY_AT: AtomicU64 = AtomicU64::new(0); +pub(crate) static GROW_REPLY_CONSUMED: AtomicU64 = AtomicU64::new(0); + +/// v97d 3D stall watch (bits 4/5). Polls, detections, detections whose 3D +/// QID was in the B1 deferred mask, doorbells sent (paired / Fragment), +/// stalls that cleared after a doorbell, stalls that cleared unaided, and +/// stalls still present when a firmware recovery arrived. +pub(crate) static STALL_POLLS: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_DETECTED: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_DEFERRED: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_DOORBELLS: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_FRAGMENT_DOORBELLS: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_CLEARED_AFTER_DOORBELL: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_CLEARED_UNAIDED: AtomicU64 = AtomicU64::new(0); +pub(crate) static STALL_AT_RECOVERY: AtomicU64 = AtomicU64::new(0); +/// Longest detected stall (ns from candidate start to clearance). +pub(crate) static STALL_MAX_NS: AtomicU64 = AtomicU64::new(0); + +/// Poll period and detection threshold. The native DM monitor escalates a +/// no-progress DM1 slot after two samples ~36 ms apart (T2 hang: +34/+70 ms +/// after the TA completed); detection must precede it. +pub(crate) const STALL_POLL_MS: u64 = 4; +pub(crate) const STALL_MS: u64 = 20; + +/// Per-pair watch state, owned by the Runtime lock (atomics only for +/// static storage). `since` = ns when the pair became a candidate with this +/// 3D completed stamp (0 = not a candidate); `stage` 0 none, 1 detected, +/// 2 paired doorbell sent, 3 Fragment doorbell sent, 4 send failed. +pub(crate) struct StallSlot { + pub(crate) fragment_completed: AtomicU64, + pub(crate) since: AtomicU64, + pub(crate) stage: AtomicU32, + pub(crate) acted_at: AtomicU64, +} +pub(crate) const STALL_SLOTS: usize = 16; +pub(crate) static STALL_SLOT: [StallSlot; STALL_SLOTS] = [const { + StallSlot { fragment_completed: AtomicU64::new(u64::MAX), since: AtomicU64::new(0), + stage: AtomicU32::new(0), acted_at: AtomicU64::new(0) } +}; STALL_SLOTS]; +/// A 3D queue is a stall candidate when it has published work beyond its +/// completed stamp and the paired TA queue (same stamp numbering) has +/// completed at least the 3D's next stamp: its TA parent is complete. +/// Stamps compare as signed 40-bit values, like the firmware's barriers. +pub(crate) fn stall_candidate(ta_completed: u64, fragment_completed: u64, latest: u64, + next: Option) -> bool { + fragment_completed != latest && next.is_some_and(|next| { + let delta = ta_completed.wrapping_sub(next) & 0xff_ffff_ffff; + delta < (1 << 39) + }) +} + +/// Forget an interrupted observation without claiming the work completed. +/// A borrowed/absent graph or an unreadable dependency record is not a +/// retirement witness. A later sample must start a new contiguous interval. +pub(crate) fn discard_stall_observation(slot: &StallSlot) { + slot.since.store(0, Ordering::Relaxed); + slot.stage.store(0, Ordering::Relaxed); + slot.acted_at.store(0, Ordering::Relaxed); + slot.fragment_completed.store(u64::MAX, Ordering::Relaxed); +} + +/// Observe one queue independently of every other queue's progress. Only a +/// changed completion stamp counts as clearance. Losing the candidate +/// predicate or its graph cannot establish that a doorbell rescued it. +pub(crate) fn sample_stall_slot(slot: &StallSlot, now: u64, + fragment_completed: u64, candidate: bool) -> bool { + if slot.fragment_completed.swap(fragment_completed, Ordering::Relaxed) != fragment_completed { + note_stall_cleared(slot, now); + } + if !candidate { + discard_stall_observation(slot); + return false; + } + let since = slot.since.load(Ordering::Relaxed); + if since == 0 { + slot.since.store(now, Ordering::Relaxed); + return false; + } + now.saturating_sub(since) >= STALL_MS * 1_000_000 +} + +pub(crate) fn note_stall_cleared(slot: &StallSlot, now: u64) { + let since = slot.since.swap(0, Ordering::Relaxed); + let stage = slot.stage.swap(0, Ordering::Relaxed); + if stage == 0 || since == 0 { return; } + let lasted = now.saturating_sub(since); + STALL_MAX_NS.fetch_max(lasted, Ordering::Relaxed); + // "After" is temporal association, never proof of a causal rescue. + // Failed sends are deliberately not credited as sent doorbells. + if matches!(stage, 2 | 3) { + STALL_CLEARED_AFTER_DOORBELL.fetch_add(1, Ordering::Relaxed); + } else { + STALL_CLEARED_UNAIDED.fetch_add(1, Ordering::Relaxed); + } +} + +/// A firmware recovery arrived: count and describe every stall still open. +pub(crate) fn note_recovery_for_stalls() -> OpenStalls { + let mut open = OpenStalls::default(); + for (index, slot) in STALL_SLOT.iter().enumerate() { + let stage = slot.stage.load(Ordering::Relaxed); + if stage != 0 { + STALL_AT_RECOVERY.fetch_add(1, Ordering::Relaxed); + if open.count < open.pairs.len() { + open.pairs[open.count] = (index as u8, stage as u8, + slot.fragment_completed.load(Ordering::Relaxed)); + open.count += 1; + } + } + } + open +} + +/// Fixed-capacity summary of open stalls (no allocation on the event path). +#[derive(Default)] +pub(crate) struct OpenStalls { + pub(crate) pairs: [(u8, u8, u64); 4], + pub(crate) count: usize, +} + +pub(crate) fn watch_grow_reply(producer_after: u32, buffer_slot: u32) { + GROW_REPLY_AT.store(crate::g18::compat::timing_buffer::arch_counter(), Ordering::Relaxed); + GROW_REPLY_WATCH.store(1 << 63 | u64::from(producer_after) << 32 | u64::from(buffer_slot), + Ordering::Release); +} + +/// A bounded log budget: the first `first` events, then one in `every`. +pub(crate) fn log_budget(counter: u64, first: u64, every: u64) -> bool { + counter <= first || counter % every == 0 +} + +pub(crate) fn note_blocked(reason: ReclaimBlock) { + RECLAIM_BLOCKED[reason.index()].fetch_add(1, Ordering::Relaxed); +} + +// ---- control-ring shadow (bit1) ----------------------------------------- + +const CONTROL_SLOTS: usize = crate::g18::fw::initdata::CONTROL_RING_ENTRIES; +/// opcode | arg << 32, and CNTPCT at publication, per ring slot. +static CONTROL_RECORD: [AtomicU64; CONTROL_SLOTS] = [const { AtomicU64::new(0) }; CONTROL_SLOTS]; +static CONTROL_PUBLISHED_AT: [AtomicU64; CONTROL_SLOTS] = [const { AtomicU64::new(0) }; CONTROL_SLOTS]; +static LAST_ANNOUNCE: AtomicU64 = AtomicU64::new(0); +/// Re-announces whose oldest pending record was (not) announced after it +/// was published. +static REANNOUNCE_ANNOUNCED: AtomicU64 = AtomicU64::new(0); +static REANNOUNCE_UNANNOUNCED: AtomicU64 = AtomicU64::new(0); +/// Re-announces by the pending record's opcode (0..0x3f, larger folded to 0x3f). +static REANNOUNCE_BY_OPCODE: [AtomicU32; 64] = [const { AtomicU32::new(0) }; 64]; + +/// Called for every primary device-control publication (ring slot index). +#[inline(always)] +pub(crate) fn note_control_publish(slot: u32, opcode: u32, arg: u32) { + if !enabled(CONTROL_TRACE) { return; } + let index = slot as usize % CONTROL_SLOTS; + CONTROL_RECORD[index].store(u64::from(opcode) | u64::from(arg) << 32, Ordering::Relaxed); + CONTROL_PUBLISHED_AT[index].store(crate::g18::compat::timing_buffer::arch_counter(), Ordering::Relaxed); +} + +/// Called for every EP 0x21 device-control announcement actually sent. +#[inline(always)] +pub(crate) fn note_control_announce() { + if !enabled(CONTROL_TRACE) { return; } + LAST_ANNOUNCE.store(crate::g18::compat::timing_buffer::arch_counter(), Ordering::Relaxed); +} + +/// The oldest pending record at `consumer`, if traced: (opcode, arg, age +/// in counter ticks, announced after publication). +pub(crate) fn describe_pending(consumer: u32) -> Option<(u32, u32, u64, bool)> { + if !enabled(CONTROL_TRACE) { return None; } + let index = consumer as usize % CONTROL_SLOTS; + let published = CONTROL_PUBLISHED_AT[index].load(Ordering::Relaxed); + if published == 0 { return None; } + let record = CONTROL_RECORD[index].load(Ordering::Relaxed); + let now = crate::g18::compat::timing_buffer::arch_counter(); + let announced = LAST_ANNOUNCE.load(Ordering::Relaxed) >= published; + let opcode = record as u32; + let counter = if announced { &REANNOUNCE_ANNOUNCED } else { &REANNOUNCE_UNANNOUNCED }; + counter.fetch_add(1, Ordering::Relaxed); + REANNOUNCE_BY_OPCODE[(opcode as usize).min(63)].fetch_add(1, Ordering::Relaxed); + Some((opcode, (record >> 32) as u32, now.saturating_sub(published), announced)) +} + +// ---- parameter ------------------------------------------------------------ + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} + +/// Accepts "reset" or a decimal/0x-hex mask of known bits. +fn parse_mask(text: &str) -> Option { + let parsed = match text.strip_prefix("0x") { + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + }?; + (parsed & !ALL_BITS == 0).then_some(parsed) +} + +unsafe extern "C" fn mask_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + // SAFETY: the parameter core supplies a NUL-terminated string. + let Ok(text) = unsafe { core::ffi::CStr::from_ptr(value) }.to_str() else { return -22; }; + let text = text.trim(); + if text == "reset" { + for counter in [&RECLAIM_ATTEMPTS, &RECLAIM_PAIRS, &RECLAIM_TRACKERS, &RECLAIM_RELEASED, + &REANNOUNCE_ANNOUNCED, &REANNOUNCE_UNANNOUNCED, &GROW_INGRESS, + &GROW_DEFERRED, &GROW_PARKED_AT_RETURN, &GROW_REKICKS, &GROW_REPLIED, + &GROW_REPLY_CONSUMED, &STALL_POLLS, &STALL_DETECTED, &STALL_DEFERRED, + &STALL_DOORBELLS, &STALL_FRAGMENT_DOORBELLS, &STALL_CLEARED_AFTER_DOORBELL, + &STALL_CLEARED_UNAIDED, &STALL_AT_RECOVERY, &STALL_MAX_NS] { + counter.store(0, Ordering::Relaxed); + } + for counter in RECLAIM_BLOCKED.iter() { counter.store(0, Ordering::Relaxed); } + for counter in REANNOUNCE_BY_OPCODE.iter() { counter.store(0, Ordering::Relaxed); } + return 0; + } + let Some(mask) = parse_mask(text) else { return -22; }; + MASK.store(mask, Ordering::Release); + 0 +} + +unsafe extern "C" fn mask_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes, used: 0 }; + let _ = write!(out, "{:#x} reclaim attempts={} pairs={} trackers={} released={} blocked:", + MASK.load(Ordering::Relaxed), RECLAIM_ATTEMPTS.load(Ordering::Relaxed), + RECLAIM_PAIRS.load(Ordering::Relaxed), RECLAIM_TRACKERS.load(Ordering::Relaxed), + RECLAIM_RELEASED.load(Ordering::Relaxed)); + for (name, counter) in ReclaimBlock::NAMES.iter().zip(RECLAIM_BLOCKED.iter()) { + let _ = write!(out, " {}={}", name, counter.load(Ordering::Relaxed)); + } + let _ = write!(out, " tvb-grow ingress={} replied={} reply-consumed={} deferred={} parked-at-return={} rekicks={}", + GROW_INGRESS.load(Ordering::Relaxed), GROW_REPLIED.load(Ordering::Relaxed), + GROW_REPLY_CONSUMED.load(Ordering::Relaxed), + GROW_DEFERRED.load(Ordering::Relaxed), GROW_PARKED_AT_RETURN.load(Ordering::Relaxed), + GROW_REKICKS.load(Ordering::Relaxed)); + let _ = write!(out, " stall polls={} detected={} deferred={} doorbells={} fragment-doorbells={} cleared-after-doorbell={} cleared-unaided={} at-recovery={} max-ms={}", + STALL_POLLS.load(Ordering::Relaxed), STALL_DETECTED.load(Ordering::Relaxed), + STALL_DEFERRED.load(Ordering::Relaxed), STALL_DOORBELLS.load(Ordering::Relaxed), + STALL_FRAGMENT_DOORBELLS.load(Ordering::Relaxed), + STALL_CLEARED_AFTER_DOORBELL.load(Ordering::Relaxed), + STALL_CLEARED_UNAIDED.load(Ordering::Relaxed), STALL_AT_RECOVERY.load(Ordering::Relaxed), + STALL_MAX_NS.load(Ordering::Relaxed) / 1_000_000); + let _ = write!(out, " reannounce announced={} unannounced={} by-opcode:", + REANNOUNCE_ANNOUNCED.load(Ordering::Relaxed), REANNOUNCE_UNANNOUNCED.load(Ordering::Relaxed)); + for (opcode, counter) in REANNOUNCE_BY_OPCODE.iter().enumerate() { + let count = counter.load(Ordering::Relaxed); + if count != 0 { let _ = write!(out, " {:#x}={}", opcode, count); } + } + let _ = writeln!(out); + out.used as core::ffi::c_int +} + +static MASK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(mask_set), get: Some(mask_get), free: None, +}; +#[link_section = "__param"] +#[used] +static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_recovery_path" + } else { c"asahi.g17p_recovery_path" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&MASK_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/stackwatch.rs b/drivers/gpu/drm/asahi/g18/compat/stackwatch.rs new file mode 100644 index 0000000000000..23d8977ed8d2f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/stackwatch.rs @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! DIAGNOSTIC (default off, read through sysfs only). +//! +//! `g17p_stack_watch` (root-only, runtime or boot "1"/"0"/"reset"): kernel +//! stack high-water mark of render publication. With it set, publish_with +//! paints the unused part of the task stack below the current SP and, after +//! publication returns, scans for the deepest overwritten word. The result +//! includes every callee (kernel printk, scheduler, exception entry). Kept +//! separately for the submit-ioctl early path and the scheduler worker path, +//! and for publications during which a first-kick KSMFE configure or a +//! registration power bracket ran. THREAD_SIZE is 32 KiB (MIN_THREAD_SHIFT 15, +//! 16K pages, VMAP_STACK). Never a default. +use core::fmt::Write; +use core::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use kernel::{bindings, module_param::KernelParam}; + +const THREAD_SIZE: usize = 1 << 15; +const GUARD: usize = 256; +const PATTERN: u64 = 0xa5a5_5a5a_c3c3_3c3c; + +// ---- stack watch ---- +static SW_ENABLED: AtomicU32 = AtomicU32::new(0); +/// [early, scheduler, first-install, bracket]: samples, max used, max SP depth at entry. +static SW_SAMPLES: [AtomicU64; 4] = [const { AtomicU64::new(0) }; 4]; +static SW_MAX_USED: [AtomicUsize; 4] = [const { AtomicUsize::new(0) }; 4]; +static SW_MAX_ENTRY: [AtomicUsize; 4] = [const { AtomicUsize::new(0) }; 4]; +static SW_SKIPPED: AtomicU64 = AtomicU64::new(0); +pub(crate) static FIRST_INSTALL_SEQ: AtomicU64 = AtomicU64::new(0); +pub(crate) static BRACKET_SEQ: AtomicU64 = AtomicU64::new(0); + +pub(crate) struct StackWatch { base: usize, top: usize, entry_depth: usize, seq: [u64; 2] } + +#[inline(always)] +fn current_sp() -> usize { + let sp: usize; + // SAFETY: reads the stack pointer; no memory access. + unsafe { core::arch::asm!("mov {}, sp", out(reg) sp, options(nomem, nostack, preserves_flags)) }; + sp +} + +/// Paint the unused stack below this frame (only while enabled). +#[inline(never)] +pub(crate) fn begin() -> Option { + if SW_ENABLED.load(Ordering::Relaxed) == 0 { return None; } + // SAFETY: `current` is always valid in process/kthread context. + let base = unsafe { (*bindings::get_current()).stack } as usize; + let sp = current_sp(); + let top = base.checked_add(THREAD_SIZE)?; + if base == 0 || sp <= base + GUARD + 64 || sp >= top { + SW_SKIPPED.fetch_add(1, Ordering::Relaxed); + return None; + } + let mut at = (base + GUARD + 7) & !7; + let end = (sp - 64) & !7; + while at < end { + // SAFETY: [base+GUARD, sp-64) is this task's own unused stack; the + // arm64 kernel ABI has no red zone and nothing live sits below SP. + unsafe { core::ptr::write_volatile(at as *mut u64, PATTERN) }; + at += 8; + } + Some(StackWatch { base, top, entry_depth: top - sp, + seq: [FIRST_INSTALL_SEQ.load(Ordering::Relaxed), BRACKET_SEQ.load(Ordering::Relaxed)] }) +} + +#[inline(never)] +pub(crate) fn end(watch: Option, early: bool) { + let Some(w) = watch else { return }; + let mut at = (w.base + GUARD + 7) & !7; + // SAFETY: scanning this task's own stack between the guard and the top. + while at < w.top && unsafe { core::ptr::read_volatile(at as *const u64) } == PATTERN { at += 8; } + let used = w.top - at; + let mut classes = [false; 4]; + classes[if early { 0 } else { 1 }] = true; + classes[2] = FIRST_INSTALL_SEQ.load(Ordering::Relaxed) != w.seq[0]; + classes[3] = BRACKET_SEQ.load(Ordering::Relaxed) != w.seq[1]; + for (index, set) in classes.into_iter().enumerate() { + if !set { continue; } + SW_SAMPLES[index].fetch_add(1, Ordering::Relaxed); + SW_MAX_USED[index].fetch_max(used, Ordering::Relaxed); + SW_MAX_ENTRY[index].fetch_max(w.entry_depth, Ordering::Relaxed); + } +} + +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> core::fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(core::fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); + Ok(()) + } +} + +fn param_text<'a>(value: *const core::ffi::c_char) -> Option<&'a str> { + if value.is_null() { return None; } + // SAFETY: the parameter core supplies a NUL-terminated string. + unsafe { core::ffi::CStr::from_ptr(value) }.to_str().ok().map(str::trim) +} + +unsafe extern "C" fn sw_set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + match param_text(value) { + Some("reset") => { + for i in 0..4 { + SW_SAMPLES[i].store(0, Ordering::Relaxed); + SW_MAX_USED[i].store(0, Ordering::Relaxed); + SW_MAX_ENTRY[i].store(0, Ordering::Relaxed); + } + 0 + } + Some("1") => { SW_ENABLED.store(1, Ordering::Relaxed); 0 } + Some("0") => { SW_ENABLED.store(0, Ordering::Relaxed); 0 } + _ => -22, + } +} + +unsafe extern "C" fn sw_get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1024) }; + let mut out = Output { bytes, used: 0 }; + let _ = write!(out, "enabled={} thread_size={} skipped={}", SW_ENABLED.load(Ordering::Relaxed), + THREAD_SIZE, SW_SKIPPED.load(Ordering::Relaxed)); + for (index, name) in ["early", "sched", "first_install", "bracket"].iter().enumerate() { + let used = SW_MAX_USED[index].load(Ordering::Relaxed); + let _ = write!(out, " {}:samples={}/max_used={}/margin={}/max_entry_depth={}", name, + SW_SAMPLES[index].load(Ordering::Relaxed), used, + if used == 0 { 0 } else { THREAD_SIZE as i64 - used as i64 }, + SW_MAX_ENTRY[index].load(Ordering::Relaxed)); + } + let _ = writeln!(out, " first_install_seq={} bracket_seq={}", + FIRST_INSTALL_SEQ.load(Ordering::Relaxed), BRACKET_SEQ.load(Ordering::Relaxed)); + out.used as core::ffi::c_int +} + +static SW_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(sw_set), get: Some(sw_get), free: None, +}; +#[link_section = "__param"] +#[used] +static SW_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_stack_watch" + } else { c"asahi.g17p_stack_watch" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&SW_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/compat/timing_buffer.rs b/drivers/gpu/drm/asahi/g18/compat/timing_buffer.rs new file mode 100644 index 0000000000000..5e4c614f196f9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/compat/timing_buffer.rs @@ -0,0 +1,929 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! One-shot diagnostic buffer. No allocation, printk, waiting or overwrite. +//! Atomic words avoid unsafe shared payload access; ready publishes each slot. +use core::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use core::fmt::{self, Write}; +use kernel::{bindings, module_param::KernelParam}; + +pub(crate) const WORDS: usize = 24; +const CAPACITY: usize = 16384; +struct Slot { ready: AtomicU32, words: [AtomicU64; WORDS] } +static SLOTS: [Slot; CAPACITY] = [const { Slot { + ready: AtomicU32::new(0), words: [const { AtomicU64::new(0) }; WORDS], +} }; CAPACITY]; +// 0 unarmed, 1 capturing, 2 permanently frozen until module unload. +static STATE: AtomicU32 = AtomicU32::new(0); +// Freeze and reservation share one atomic: stop cannot miss a late reservation. +const FROZEN: usize = 1usize << (usize::BITS - 1); +static NEXT: AtomicUsize = AtomicUsize::new(FROZEN); +static FROZEN_BOUND: AtomicUsize = AtomicUsize::new(0); +static CURSOR: AtomicUsize = AtomicUsize::new(0); +static DROPPED: AtomicU64 = AtomicU64::new(0); + +pub(crate) fn enabled() -> bool { STATE.load(Ordering::Relaxed) == 1 } +pub(crate) fn now() -> u64 { + ::ktime_get() as u64 +} + +/// Record kinds. 1/2: render admission/completion. 3/4: compute publication +/// return and exact retirement. 5: render publication sub-stages. Records +/// carry the architectural counter. Host and GPU rates must be converted +/// separately before testing whether their epochs align. +pub(crate) const KIND_RENDER_ADMISSION: u64 = 1; +pub(crate) const KIND_RENDER_COMPLETION: u64 = 2; +pub(crate) const KIND_COMPUTE_PUBLISH: u64 = 3; +pub(crate) const KIND_COMPUTE_RETIRE: u64 = 4; +pub(crate) const KIND_RENDER_PUBLISH_STAGES: u64 = 5; +/// Kind6: exact post-maintenance descriptor observations, only while armed. +/// Words1..7: pair slot, ordinal, TA/FRG descriptor VAs, physical TA/FRG kick +/// timestamps, packed TA/FRG QIDs. Words8..9: external pass envelope;10..13: +/// latest qualifying adjusted TA/FRG pair-B segments (24MHz).14: packed observed +/// retirement stamps;15: host counter;16: read-success;17: host frequency. +/// A zero/reversed pair is unavailable. Qualifying segments may be overwritten +/// on rearm; these are not aggregate execution or shader-only measurements. +pub(crate) const KIND_RENDER_DEVICE_SEGMENTS: u64 = 6; +/// Layout of the render completion record's words 10..19: TA start, TA end, +/// 3D start, 3D end (absolute GPU ticks), the architectural counter when the +/// fence is signalled (14), this version (15), then the service pass that +/// proved retirement on the same counter: last notification ingress on the +/// primary and secondary RTKit roles (16, 17), service-pass entry (18) and +/// the observation under the runtime mutex (19). Version 2 had words 10..14 +/// only, with word 14 sampled at observation. +pub(crate) const RENDER_COMPLETION_LAYOUT: u64 = 3; + +/// Host CNTPCT_EL0 ticks; `arch_frequency` gives their rate. Only read while armed. +/// Neo Linux exposes a 1 GHz physical timer while GPU command timestamps use +/// 24 MHz. Consumers must convert each clock separately before comparing them. +pub(crate) fn arch_counter() -> u64 { + let counter: u64; + // SAFETY: CNTPCT_EL0 is readable at EL1 on this platform (the driver's + // QoS publication already reads it) and has no memory side effects. + unsafe { + core::arch::asm!("mrs {c}, CNTPCT_EL0", c = out(reg) counter, + options(nomem, nostack, preserves_flags)); + } + counter +} + +pub(crate) fn arch_frequency() -> u64 { + let frequency: u64; + // SAFETY: CNTFRQ_EL0 is a read-only system register. + unsafe { + core::arch::asm!("mrs {f}, CNTFRQ_EL0", f = out(reg) frequency, + options(nomem, nostack, preserves_flags)); + } + frequency +} +pub(crate) fn record(words: [u64; WORDS]) { + if !enabled() { return; } + let next = NEXT.load(Ordering::Relaxed); + if next & FROZEN != 0 { return; } + if next >= CAPACITY { DROPPED.fetch_add(1, Ordering::Relaxed); return; } + // A freeze racing this reservation either includes this ticket in its + // finite drain bound, or this ticket observes FROZEN and writes nothing. + let index = NEXT.fetch_add(1, Ordering::Relaxed); + if index & FROZEN != 0 { return; } + if index >= CAPACITY { DROPPED.fetch_add(1, Ordering::Relaxed); return; } + let slot = &SLOTS[index]; + for (dst, src) in slot.words.iter().zip(words) { dst.store(src, Ordering::Relaxed); } + slot.ready.store(1, Ordering::Release); +} + +fn reserved() -> usize { + if STATE.load(Ordering::Acquire) == 2 { FROZEN_BOUND.load(Ordering::Relaxed) } + else { (NEXT.load(Ordering::Relaxed) & !FROZEN).min(CAPACITY) } +} + +// Limit each sysfs response to 4095 bytes, below PAGE_SIZE even on 4K kernels. +struct Output<'a> { bytes: &'a mut [u8], used: usize } +impl Write for Output<'_> { + fn write_str(&mut self, value: &str) -> fmt::Result { + if value.len() > self.bytes.len() - self.used { return Err(fmt::Error); } + self.bytes[self.used..self.used + value.len()].copy_from_slice(value.as_bytes()); + self.used += value.len(); Ok(()) + } +} +unsafe extern "C" fn get(buf: *mut core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + // SAFETY: sysfs module parameter get supplies a PAGE_SIZE output buffer. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4095) }; + let mut out = Output { bytes, used: 0 }; + let mut cursor = CURSOR.load(Ordering::Relaxed); + // The kernel parameter mutex serializes readers/setters, never writers. + // Four records fit even when every u64 takes 20 decimal digits. + for _ in 0..4 { + if cursor >= reserved() { break; } + let slot = &SLOTS[cursor]; + if slot.ready.load(Ordering::Acquire) == 0 { break; } + let _ = write!(out, "record {}", cursor); + for word in &slot.words { let _ = write!(out, " {}", word.load(Ordering::Relaxed)); } + let _ = writeln!(out); + cursor += 1; + } + CURSOR.store(cursor, Ordering::Relaxed); + let reserved = reserved(); + let _ = writeln!(out, "status state={} cursor={} reserved={} dropped={} capacity={}", + STATE.load(Ordering::Relaxed), cursor, reserved, DROPPED.load(Ordering::Relaxed), CAPACITY); + out.used as core::ffi::c_int +} +unsafe extern "C" fn set(value: *const core::ffi::c_char, _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + // Bounded parser avoids strlen and rejects commands longer than 7 bytes. + let mut command = [0u8; 8]; + let mut len = 0; + while len < command.len() { + // SAFETY: parameter core supplies a NUL-terminated value; never read + // beyond the first NUL, and at most eight bytes are examined. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let arg = &command[..len]; + match arg { + b"1" | b"1\n" => { + if *crate::module_parameters::g17p_completion_trace.value() != 0 + || *crate::module_parameters::g17p_trace_depth.value() != 0 { return -22; } + if STATE.load(Ordering::Relaxed) != 0 { return -16; } + NEXT.store(0, Ordering::Relaxed); + STATE.store(1, Ordering::Relaxed); + } + b"0" | b"0\n" => { + if STATE.load(Ordering::Relaxed) != 2 { + FROZEN_BOUND.store((NEXT.fetch_or(FROZEN, Ordering::Relaxed) & !FROZEN).min(CAPACITY), Ordering::Relaxed); + STATE.store(2, Ordering::Release); + } + } + // A nonempty frozen capture proves that this module has published G17 + // work. Only the explicit root-only command reads fixed firmware DATA. + b"clock" | b"clock\n" => { + if STATE.load(Ordering::Acquire) != 2 || reserved() == 0 { return -22; } + let before = arch_counter(); + let snapshot = crate::g18::compat::g17p::diagnostic_accounting_clocks(); + let after = arch_counter(); + kernel::pr_info!("G17P accounting clocks: host-before={} host-after={} host-hz={} fields(TA,FRG,CL,all,freq-word,offset,sign-word,abs-offset)={:x?}\n", + before, after, arch_frequency(), snapshot); + if let Err(error) = snapshot { return error.to_errno(); } + } + // Rewind readout only. Slots remain immutable, including while capturing. + b"rewind" | b"rewind\n" => { CURSOR.store(0, Ordering::Relaxed); } + _ => return -22, + } + 0 +} +static OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(set), get: Some(get), free: None, +}; +#[link_section = "__param"] +#[used] +static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_timing_buffer" + } else { c"asahi.g17p_timing_buffer" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); + +// One explicit post-window snapshot. No live ring reads occur in record(). +// 0 unused, 1 requested, 2 copying, 3 complete, 4 failed; no reset until unload. +const RING_SLOTS: usize = 2048; +struct CompletionSlot { words: [AtomicU64; 8], changed: AtomicU32 } +static COMPLETION_SLOTS: [CompletionSlot; 2 * RING_SLOTS] = [const { CompletionSlot { + words: [const { AtomicU64::new(0) }; 8], changed: AtomicU32::new(0), +} }; 2 * RING_SLOTS]; +static COMPLETION_STATE: AtomicU32 = AtomicU32::new(0); +static COMPLETION_CURSOR: AtomicUsize = AtomicUsize::new(0); +static COMPLETION_COUNT: AtomicUsize = AtomicUsize::new(0); +static COMPLETION_ERROR: AtomicU32 = AtomicU32::new(0); +// request, copy-start/end CNTPCT, CNTFRQ, bias-before/after, ring0/2 DVA, +// accepted firmware recovery generation; then sixteen u32 table fields before +// and sixteen after the copy (eight per ordinal0/2 row). Non-atomic snapshots. +static COMPLETION_META: [AtomicU64; 49] = [const { AtomicU64::new(0) }; 49]; + +pub(crate) fn claim_completion_snapshot() -> bool { + COMPLETION_STATE.compare_exchange(1, 2, Ordering::AcqRel, Ordering::Acquire).is_ok() +} + +pub(crate) fn completion_snapshot_meta(index: usize, value: u64) { + if let Some(word) = COMPLETION_META.get(index) { word.store(value, Ordering::Relaxed); } +} + +pub(crate) fn store_completion_snapshot(index: usize, words: [u64; 8], changed: bool) { + let Some(slot) = COMPLETION_SLOTS.get(index) else { return; }; + for (dst, value) in slot.words.iter().zip(words) { dst.store(value, Ordering::Relaxed); } + slot.changed.store(u32::from(changed), Ordering::Relaxed); + COMPLETION_COUNT.store(index + 1, Ordering::Relaxed); +} + +pub(crate) fn finish_completion_snapshot(result: kernel::error::Result) { + completion_snapshot_meta(2, arch_counter()); + let error = result.err().map_or(0, |error| error.to_errno()); + COMPLETION_ERROR.store(error as u32, Ordering::Relaxed); + COMPLETION_STATE.store(if error == 0 { 3 } else { 4 }, Ordering::Release); +} + +unsafe extern "C" fn completion_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 8]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + match &command[..len] { + b"1" | b"1\n" => { + if !matches!(STATE.load(Ordering::Acquire), 1 | 2) || reserved() == 0 { return -22; } + // A claimed request is never overwritten. Reads remain unavailable + // until COMPLETE/FAILED acquire, including concurrent sysfs reads. + if COMPLETION_STATE.compare_exchange(0, 5, Ordering::AcqRel, Ordering::Acquire).is_err() { + return -16; + } + completion_snapshot_meta(0, arch_counter()); + COMPLETION_STATE.store(1, Ordering::Release); + } + b"rewind" | b"rewind\n" => { + if !matches!(COMPLETION_STATE.load(Ordering::Acquire), 3 | 4) { return -16; } + COMPLETION_CURSOR.store(0, Ordering::Relaxed); + } + _ => return -22, + } + 0 +} + +unsafe extern "C" fn completion_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4095) }; + let mut out = Output { bytes, used: 0 }; + let state = COMPLETION_STATE.load(Ordering::Acquire); + let count = if state == 3 || state == 4 { COMPLETION_COUNT.load(Ordering::Relaxed) } else { 0 }; + let cursor = COMPLETION_CURSOR.load(Ordering::Relaxed).min(count); + let end = (cursor + 12).min(count); + for index in cursor..end { + let slot = &COMPLETION_SLOTS[index]; + let ordinal = if index < RING_SLOTS { 0 } else { 2 }; + let _ = write!(out, "completion {} {} {}", ordinal, index % RING_SLOTS, + slot.changed.load(Ordering::Relaxed)); + for word in &slot.words { let _ = write!(out, " {:016x}", word.load(Ordering::Relaxed)); } + let _ = writeln!(out); + } + if end != cursor { COMPLETION_CURSOR.store(end, Ordering::Relaxed); } + let _ = write!(out, "completion-status state={} cursor={} count={} error={} meta=", + state, end, count, COMPLETION_ERROR.load(Ordering::Relaxed) as i32); + for word in &COMPLETION_META { let _ = write!(out, " {}", word.load(Ordering::Relaxed)); } + let _ = writeln!(out); + out.used as core::ffi::c_int +} +static COMPLETION_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(completion_set), get: Some(completion_get), free: None, +}; +#[link_section = "__param"] +#[used] +static COMPLETION_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_completion_buffer" + } else { c"asahi.g17p_completion_buffer" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&COMPLETION_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); + +/// Armed-only host observation gate; no additional device reads. Kind7 words: +/// slot, front ordinal (MAX if unavailable), reason (1borrowed,2restaged,3UMA, +/// 4stamp pending,5stamp ready,6notification mask pending), expected stamp, +/// observed TA/FRG stamps, host counter, host frequency, then four optional masks. +pub(crate) fn record_render_gate(slot: u8, ordinal: u64, reason: u64, + expected: u32, stamps: [u32; 2], masks: [u64; 4]) { + if !enabled() { return; } + let mut words = [0; WORDS]; + words[0] = 7; + words[1] = u64::from(slot); + words[2] = ordinal; + words[3] = reason; + words[4] = u64::from(expected); + words[5] = u64::from(stamps[0]); + words[6] = u64::from(stamps[1]); + words[7] = arch_counter(); + words[8] = arch_frequency(); + words[9..13].copy_from_slice(&masks); + record(words); +} + +static TSQ_GATE_SAMPLE: AtomicU32 = AtomicU32::new(0); +/// One in eight armed stamp-gate observations. Ordinary work never maps TSQ +/// diagnostic data. Kind8 is a non-atomic observation, never a completion gate. +pub(crate) fn sample_tsq_gate() -> bool { + enabled() && TSQ_GATE_SAMPLE.fetch_add(1, Ordering::Relaxed) & 7 == 0 +} + +/// Root-only runtime diagnostic knobs, `g17p_diag_knobs` (u32 bitmask, +/// decimal or 0x hex). Read when each render descriptor/kick is built, so a +/// change applies to the next submission and permits same-boot A/B. Cached +/// family bits affect fresh resource owners only. bit0: FRG +0x222d = 0 (native light +/// captures); bit1: omit the per-pass KSM MCache aperture (native final +/// commands carry pointer/count zero). +/// v84 production defaults (native-matching, hardware-qualified since v64/v67 +/// under 0x9000 in every throughput and stress run): bit12 native TA +/// force-drain value (+0x8c2 = 0) and bit15 aligned word record snapshots. +/// Bits27..30 cache native command-object families with reserved return storage +/// and final-stop/UAT lifetime checks. Writing the knob still permits A/B. +pub(crate) const DIAG_PRODUCTION_DEFAULT: u32 = (1 << 12) | (1 << 15) | (0xf << 27); +static DIAG_KNOBS: AtomicU32 = AtomicU32::new(DIAG_PRODUCTION_DEFAULT); +pub(crate) const DIAG_FRG_222D_CLEAR: u32 = 1 << 0; +pub(crate) const DIAG_NO_KSM_MCACHE: u32 = 1 << 1; +/// bit2: TA 0x0a5a1 PTC high word 0xfe (captured Metal value) instead of 0x60. +pub(crate) const DIAG_TA_PTC_NATIVE: u32 = 1 << 2; +/// bit3: FRG 0x0a5a9 PTC high word 0xd5 (captured Metal value) instead of 0x60. +pub(crate) const DIAG_FRG_PTC_NATIVE: u32 = 1 << 3; +/// bit4: hold firmware idle descent off (IdlePowerOff(0)) while set; the next +/// submission after clearing it restores the ordinary policy (arg 1). +pub(crate) const DIAG_IDLE_HOLD: u32 = 1 << 4; +/// bit5: copy the first 64 KiB of the host-owned primary state grid (Status-A +/// and Status-B) into `GRID` at the next submission, then clear this bit. +pub(crate) const DIAG_GRID_SNAPSHOT: u32 = 1 << 5; +/// bit6: native `updateSubmitInProgress` accounting. Maintain RegionC+0x7c +/// (submissions in progress) and +0x80 (monotonic submit count) around each +/// render/compute submission instead of leaving the static init values. +pub(crate) const DIAG_SUBMIT_ACCOUNTING: u32 = 1 << 6; +/// bit7: native kickFirmware note (EP 0x21 `0x0087_0000_0000_0010`) right after +/// the submit-in-progress bracket opens, as native `notifyFirmware(4, 1, 0)`. +pub(crate) const DIAG_SUBMIT_KICK: u32 = 1 << 7; +/// bit8: copy the live hardware-data block (shared cluster, 0x4000 bytes) +/// into `GRID` at the next submission, then clear this bit. +pub(crate) const DIAG_HWDATA_SNAPSHOT: u32 = 1 << 8; +/// bit9: CLPC deadline control effort. RegionC+0xe4 = 0x10000 (1.0 in 16.16) +/// with +0xe8 = 0 while set, as native CLPC writes during active streams; +/// clearing restores 0. GFX1 reads it live as a pstate floor (unit 9). +pub(crate) const DIAG_CLPC_EFFORT: u32 = 1 << 9; +pub(crate) fn clear_diag_knob(bit: u32) { DIAG_KNOBS.fetch_and(!bit, Ordering::Relaxed); } +pub(crate) const GRID_QWORDS: usize = 0x10000 / 8; +pub(crate) static GRID: [AtomicU64; GRID_QWORDS] = [const { AtomicU64::new(0) }; GRID_QWORDS]; +pub(crate) fn diag_knob(bit: u32) -> bool { DIAG_KNOBS.load(Ordering::Relaxed) & bit != 0 } +unsafe extern "C" fn diag_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 16]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + let parsed = match text.strip_prefix("0x") { + Some(hex) => u32::from_str_radix(hex, 16), + None => text.parse::(), + }; + let Ok(value) = parsed else { return -22; }; + DIAG_KNOBS.store(value, Ordering::Relaxed); + kernel::pr_info!("G17P diagnostic knobs = {:#x}\n", value); + 0 +} +unsafe extern "C" fn diag_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 512) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let _ = writeln!(out, "{:#x}", DIAG_KNOBS.load(Ordering::Relaxed)); + // Firmware DATA power bookkeeping (RAM, not MMIO; bounded single loads): + // 124ba0 power-up count, 1771f8 power state (2 on, 4 stage-1 idle), + // 177838 idle-off policy (RegionC+0x30 copy), 128a68 keep-on bits. + for (name, link) in [("powerups", 0xffff_fc00_0012_4ba0u64), ("state", 0xffff_fc00_0017_71f8), + ("idleoff", 0xffff_fc00_0017_7838), ("keepon", 0xffff_fc00_0012_8a68)] { + let value = crate::g18::compat::g17p::LiveFirmwareU64Probe::new( + crate::g18::compat::g17p::g17p_gfx_link_to_physical(link)) + .and_then(|probe| probe.observe_mask(u64::MAX, 1)) + .map(|sample| sample.first); + let _ = writeln!(out, "{} {:x?}", name, value); + } + out.used as core::ffi::c_int +} +/// Root-only firmware DATA peek, `g17p_fw_peek`: write a GFX LINK address in +/// the DATA segment (hex, 8-aligned); reading returns up to 64 qwords from it, +/// stopping at the page end. Bounded host reads of firmware RAM, no writes. +static PEEK_ADDR: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +const PEEK_DATA_START: u64 = 0xffff_fc00_0004_c000; +const PEEK_DATA_END: u64 = 0xffff_fc00_0018_0000; +unsafe extern "C" fn peek_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 24]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + let Ok(address) = u64::from_str_radix(text.strip_prefix("0x").unwrap_or(text), 16) else { return -22; }; + if address & 7 != 0 || !(PEEK_DATA_START..PEEK_DATA_END).contains(&address) { return -22; } + PEEK_ADDR.store(address, Ordering::Relaxed); + 0 +} +unsafe extern "C" fn peek_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let address = PEEK_ADDR.load(Ordering::Relaxed); + if address < PEEK_DATA_START { return 0; } + let page = crate::pgtable::UAT_PGSZ as u64; + let physical = crate::g18::compat::g17p::g17p_gfx_link_to_physical(address); + let room = ((page - (physical & (page - 1))) / 8).min(64) as usize; + let room = room.min(((PEEK_DATA_END - address) / 8) as usize); + let mut values = [0u64; 64]; + match crate::g18::compat::g17p::LiveFirmwareU64Probe::new(physical) + .and_then(|probe| probe.read_run(&mut values[..room])) { + Ok(()) => for (row, chunk) in values[..room].chunks(4).enumerate() { + let _ = write!(out, "{:x}:", address + row as u64 * 32); + for value in chunk { let _ = write!(out, " {:016x}", value); } + let _ = writeln!(out); + }, + Err(error) => { let _ = writeln!(out, "error {:?}", error); } + } + out.used as core::ffi::c_int +} +static PEEK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(peek_set), get: Some(peek_get), free: None, +}; +#[link_section = "__param"] +#[used] +static PEEK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_fw_peek" + } else { c"asahi.g17p_fw_peek" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PEEK_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +/// Root-only `g17p_grid_peek`: write a byte offset (hex, 8-aligned) into the +/// last `DIAG_GRID_SNAPSHOT` copy; reading returns up to 64 qwords from it. +static GRID_OFFSET: AtomicUsize = AtomicUsize::new(0); +unsafe extern "C" fn grid_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 24]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + let Ok(offset) = usize::from_str_radix(text.strip_prefix("0x").unwrap_or(text), 16) else { return -22; }; + if offset & 7 != 0 || offset >= GRID_QWORDS * 8 { return -22; } + GRID_OFFSET.store(offset, Ordering::Relaxed); + 0 +} +unsafe extern "C" fn grid_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let first = GRID_OFFSET.load(Ordering::Relaxed) / 8; + let last = (first + 64).min(GRID_QWORDS); + for row in (first..last).step_by(4) { + let _ = write!(out, "{:x}:", row * 8); + for slot in &GRID[row..(row + 4).min(last)] { + let _ = write!(out, " {:016x}", slot.load(Ordering::Relaxed)); + } + let _ = writeln!(out); + } + out.used as core::ffi::c_int +} +static GRID_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(grid_set), get: Some(grid_get), free: None, +}; +#[link_section = "__param"] +#[used] +static GRID_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_grid_peek" + } else { c"asahi.g17p_grid_peek" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&GRID_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +/// Root-only `g17p_gpc_sample`: writing N (1..=400000) samples the always-on +/// GPC performance state (sgx+0xe01000 & 0xf, J700 physical 0x4_80e0_1000) +/// N times, sleeping ~20 us between reads; reading prints the histogram of +/// states 0..15. Read-only MMIO of an always-on power register. +static GPC_HIST: [AtomicU64; 16] = [const { AtomicU64::new(0) }; 16]; +static GPC_SAMPLED_NS: AtomicU64 = AtomicU64::new(0); +const GPC_STATE_PHYS: u64 = 0x4_80e0_1000; +unsafe extern "C" fn gpc_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 16]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + let Ok(samples) = text.parse::() else { return -22; }; + if samples == 0 || samples > 400_000 { return -22; } + // SAFETY: maps one always-on 4-byte device register; unmapped below. + let base = unsafe { bindings::ioremap(GPC_STATE_PHYS as bindings::phys_addr_t, 4) }; + if base.is_null() { return -12; } + for slot in &GPC_HIST { slot.store(0, Ordering::Relaxed); } + let start = now(); + for _ in 0..samples { + // SAFETY: `base` is a live 4-byte MMIO mapping. + let state = unsafe { bindings::readl(base) } & 0xf; + GPC_HIST[state as usize].fetch_add(1, Ordering::Relaxed); + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(20)); + } + GPC_SAMPLED_NS.store(now().wrapping_sub(start), Ordering::Relaxed); + // SAFETY: `base` came from ioremap above and is not used afterwards. + unsafe { bindings::iounmap(base) }; + 0 +} +unsafe extern "C" fn gpc_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 1000) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let _ = write!(out, "ns={}", GPC_SAMPLED_NS.load(Ordering::Relaxed)); + for (state, slot) in GPC_HIST.iter().enumerate() { + let count = slot.load(Ordering::Relaxed); + if count != 0 { let _ = write!(out, " s{}={}", state, count); } + } + let _ = writeln!(out); + out.used as core::ffi::c_int +} +static GPC_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(gpc_set), get: Some(gpc_get), free: None, +}; +#[link_section = "__param"] +#[used] +static GPC_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_gpc_sample" + } else { c"asahi.g17p_gpc_sample" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&GPC_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +static DIAG_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(diag_set), get: Some(diag_get), free: None, +}; +#[link_section = "__param"] +#[used] +static DIAG_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_diag_knobs" + } else { c"asahi.g17p_diag_knobs" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&DIAG_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); +/// Opt-in `g17p_diag_knobs` bit10: one-shot log of host<->firmware traffic. +/// Each entry is [counter | kind << 56, a, b, c]; the log stops at capacity +/// until `g17p_msglog` is written "reset". Logging only; changes no behaviour. +pub(crate) const DIAG_MSGLOG: u32 = 1 << 10; +/// a=endpoint, b=message, c=RTKit instance address (role tag). +pub(crate) const MSG_SEND: u64 = 1; +/// a=endpoint | processor << 8, b=message. Hard-IRQ callback. +pub(crate) const MSG_RECV_EARLY: u64 = 2; +/// a=endpoint | processor << 8, b=message. Workqueue callback. +pub(crate) const MSG_RECV: u64 = 3; +/// a=opcode, b=arg, c=producer before | consumer before << 32. +pub(crate) const MSG_DEVICE_CONTROL: u64 = 4; +/// a=role | consumer << 8 | producer << 32, b/c=record qwords 0/1. +pub(crate) const MSG_EVENT: u64 = 5; +/// a/b/c=record qwords 2/3/4 of the preceding MSG_EVENT. +pub(crate) const MSG_EVENT_TAIL: u64 = 6; +/// a=producer before, b=command VA, c=producer after (compute channel). +pub(crate) const MSG_CHANNEL: u64 = 7; +/// a=render slot | 1 << 32 for compute, b=phase (0 begin, 1 end). +pub(crate) const MSG_SUBMIT: u64 = 8; +/// a=role | consumer << 8 | producer << 32, b=consumed (stats ring). +pub(crate) const MSG_STATS: u64 = 9; +/// a/b/c=first three qwords of the statistics record at the consumer. +pub(crate) const MSG_STATS_RECORD: u64 = 10; +/// Opt-in bit11: consume both roles' statistics rings on every event-worker +/// pass, as native does on every interrupt. Diagnostic; default off. +pub(crate) const DIAG_STATS_DRAIN: u32 = 1 << 11; +/// Opt-in bit12: use the existing native-TA-drain probe for ordinary shared +/// TSQ render jobs. Keep the bootstrap and split-launch paths unchanged. +/// This permits same-boot A/B of combined TA/FRG visibility publication; +/// mandatory cache maintenance and both final-stamp retirement checks stay. +pub(crate) const DIAG_NATIVE_TA_DRAIN: u32 = 1 << 12; +/// Opt-in bit13: announce an eligible render submission before descriptor +/// preparation, to test overlap of native wake protocol with CPU preparation. +/// No clock, idle-policy, dependency or queue-capacity changes. +pub(crate) const DIAG_EARLY_RENDER_KICK: u32 = 1 << 13; +/// a=firmware timestamp (24 MHz), b=key | thread << 16 | secondary << 24 +/// | flag << 32, c=arg0. Logged instead of printed while bit10 is set. +pub(crate) const MSG_KTRACE: u64 = 11; +/// a/b/c=args 1..3 of the preceding MSG_KTRACE. +pub(crate) const MSG_KTRACE_ARGS: u64 = 12; +/// v93: a=record qword 3 (+0x18; GPU pstate index in type-0x10 records), +/// b=role, c=consumer, following the MSG_STATS_RECORD of the same record. +pub(crate) const MSG_STATS_RECORD_Q3: u64 = 13; +const MSGLOG_ENTRIES: usize = 8192; +static MSGLOG: [[AtomicU64; 4]; MSGLOG_ENTRIES] = + [const { [const { AtomicU64::new(0) }; 4] }; MSGLOG_ENTRIES]; +static MSGLOG_NEXT: AtomicUsize = AtomicUsize::new(0); +static MSGLOG_READ: AtomicUsize = AtomicUsize::new(0); +pub(crate) fn msglog(kind: u64, a: u64, b: u64, c: u64) { + if !diag_knob(DIAG_MSGLOG) { return; } + let index = MSGLOG_NEXT.fetch_add(1, Ordering::Relaxed); + let Some(slot) = MSGLOG.get(index) else { return; }; + slot[1].store(a, Ordering::Relaxed); + slot[2].store(b, Ordering::Relaxed); + slot[3].store(c, Ordering::Relaxed); + slot[0].store((arch_counter() & 0x00ff_ffff_ffff_ffff) | (kind << 56), Ordering::Release); +} +/// Root-only `g17p_msglog`: write "reset" to clear, or a hex entry index to +/// choose where reading starts; reading prints up to 40 entries from there. +unsafe extern "C" fn msglog_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 24]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + if text == "reset" { + for slot in &MSGLOG { slot[0].store(0, Ordering::Relaxed); } + MSGLOG_READ.store(0, Ordering::Relaxed); + MSGLOG_NEXT.store(0, Ordering::Release); + return 0; + } + let Ok(index) = usize::from_str_radix(text.strip_prefix("0x").unwrap_or(text), 16) else { return -22; }; + if index >= MSGLOG_ENTRIES { return -22; } + MSGLOG_READ.store(index, Ordering::Relaxed); + 0 +} +unsafe extern "C" fn msglog_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let next = MSGLOG_NEXT.load(Ordering::Acquire).min(MSGLOG_ENTRIES); + let first = MSGLOG_READ.load(Ordering::Relaxed); + let _ = writeln!(out, "n={:x}", next); + for index in first..(first + 40).min(next) { + let slot = &MSGLOG[index]; + let head = slot[0].load(Ordering::Acquire); + let _ = writeln!(out, "{:x} {:x} {} {:x} {:x} {:x}", index, + head & 0x00ff_ffff_ffff_ffff, head >> 56, slot[1].load(Ordering::Relaxed), + slot[2].load(Ordering::Relaxed), slot[3].load(Ordering::Relaxed)); + } + out.used as core::ffi::c_int +} +static MSGLOG_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(msglog_set), get: Some(msglog_get), free: None, +}; +#[link_section = "__param"] +#[used] +static MSGLOG_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_msglog" + } else { c"asahi.g17p_msglog" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&MSGLOG_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); + +/// Opt-in bit14: request a work-ring scan before eligible render preparation. +pub(crate) const DIAG_EARLY_RENDER_DOORBELL: u32 = 1 << 14; + +/// Retired bit15: published firmware records are always snapshotted with +/// aligned word reads (g17p_snapshot_fw_record); the bit is ignored. + +/// Opt-in bit16: early work scan only when host tracks no submitted GPU work. +pub(crate) const DIAG_IDLE_RENDER_DOORBELL: u32 = 1 << 16; + +/// v92c bits20..22: DISABLE switches for the default-on fault guards (all +/// clear = guarded; the lab's 0x9000/0x4009000 writes keep every guard on). +/// bit20: the 1 ms teardown poll no longer settles consumed FList releases. +pub(crate) const DIAG_TEARDOWN_SETTLE_OFF: u32 = 1 << 20; +/// bit21: a new owner context may take over a render pair whose FList still +/// carries an unobserved 0x2e release witness (the pre-v92c behaviour). +pub(crate) const DIAG_FLIST_TAKEOVER_GUARD_OFF: u32 = 1 << 21; +/// bit22: no device-control re-announcement while control records stay +/// unconsumed (teardown poll and takeover wait). +pub(crate) const DIAG_CONTROL_REANNOUNCE_OFF: u32 = 1 << 22; + +/// v92d DIAGNOSTIC bit23 (default off): write MCache selector 1 (native's +/// captured value) into every fragment MCache aperture entry instead of the +/// render's own application-GART context ID. Native's value is its process's +/// GART context ID (AGXContextIDManager::alloc -> TA descriptor +0x1274 -> +/// halMCacheApertureSetup bits 41..46); on Linux context 1 is a different +/// address space, so this arm tests whether the aperture match matters at +/// all. It is not native parity. Each patched table is read back and must +/// differ from the default only in those six bits. +pub(crate) const DIAG_MCACHE_SELECTOR_NATIVE: u32 = 1 << 23; + +/// v89 A/B-only bit26: suspend the default native both-role trace/statistics +/// ring consumers at runtime (the pre-v89 diagnostic-gated behaviour). +pub(crate) const DIAG_NATIVE_RING_DRAIN_OFF: u32 = 1 << 26; +/// bit25: A/B only. Publish the compute tag-16 EntrySignal on every repeat CL +/// kick, as before the native per-command policy. Default clear: only a kick +/// whose barriers name another hardware queue publishes one. +pub(crate) const DIAG_CL_ENTRY_SIGNAL_EVERY_KICK: u32 = 1 << 25; + +/// v90 bits27..30: allocate render descriptors / TSQ node page / ChannelState +/// prefix / ConfigUpdate as pooled firmware-cached objects (native bit41). +/// Sampled for fresh allocations only; existing objects never change. +pub(crate) fn fw_cached_family_mask() -> u32 { + (DIAG_KNOBS.load(Ordering::Relaxed) >> 27) & 0xf +} + +// --------------------------------------------------------------------------- +/// v93 statistics pstate sampler: root-only `g17p_stats_pstate`, default 0. +/// +/// Writing 1 makes the (default, native-parity) statistics-ring drain also +/// read four u32 words of every drained record -- +0x04/+0x08 (timestamp, as +/// one u64) and +0x18/+0x1c (qword 3; the GPU pstate index of type-0x10 +/// records) -- before it publishes the consumer, and record them here. The +/// drain's legality check, consumer publication and order are unchanged; a +/// failed sample read is counted and skipped, never propagated. Written 0 +/// (default) the drain pays one relaxed load per pass. "reset" clears. +/// +/// Reading prints cumulative counters (difference two reads for a window; +/// the lab samples at 4 Hz), the latest type-0x10 sample, per-pstate record +/// counts and timestamp ticks (each type-0x10 delta is attributed to the +/// previous record's pstate, per role), then the newest ring entries. +/// Buckets are the low byte of qword 3, 16 meaning >= 16. The pstate +/// meaning of qword 3 is taken from native RE, not verified on Linux. +pub(crate) const STATS_PSTATE_TYPE: u32 = 0x10; +const PSTATE_BUCKETS: usize = 17; +const PSTATE_RING: usize = 64; +static PSTATE_ON: AtomicU32 = AtomicU32::new(0); +static PSTATE_RECORDS: AtomicU64 = AtomicU64::new(0); +static PSTATE_TYPE10: AtomicU64 = AtomicU64::new(0); +static PSTATE_READ_FAILS: AtomicU64 = AtomicU64::new(0); +/// Latest type-0x10 sample: timestamp, qword 3, role. +static PSTATE_LAST: [AtomicU64; 3] = [const { AtomicU64::new(0) }; 3]; +static PSTATE_COUNT: [AtomicU64; PSTATE_BUCKETS] = [const { AtomicU64::new(0) }; PSTATE_BUCKETS]; +static PSTATE_TICKS: [AtomicU64; PSTATE_BUCKETS] = [const { AtomicU64::new(0) }; PSTATE_BUCKETS]; +/// Per role: previous type-0x10 timestamp and bucket + 1 (0 = none yet). +static PSTATE_PREV: [[AtomicU64; 2]; 2] = [const { [const { AtomicU64::new(0) }; 2] }; 2]; +/// (timestamp, qword 3 with role in bit 63). +static PSTATE_SAMPLES: [[AtomicU64; 2]; PSTATE_RING] = [const { [const { AtomicU64::new(0) }; 2] }; PSTATE_RING]; +static PSTATE_NEXT: AtomicUsize = AtomicUsize::new(0); + +#[inline(always)] +pub(crate) fn stats_pstate_enabled() -> bool { PSTATE_ON.load(Ordering::Relaxed) != 0 } + +pub(crate) fn stats_pstate_read_failed() { PSTATE_READ_FAILS.fetch_add(1, Ordering::Relaxed); } + +/// Pure bucket choice (low byte of qword 3; 16 = out of range). +pub(crate) const fn stats_pstate_bucket(qword3: u64) -> usize { + let index = (qword3 & 0xff) as usize; + if index < PSTATE_BUCKETS - 1 { index } else { PSTATE_BUCKETS - 1 } +} + +/// Record one drained statistics record (single writer: the drain runs +/// under the runtime mutex; readers only load). +pub(crate) fn stats_pstate_note(role: usize, entry_type: u32, timestamp: u64, qword3: u64) { + PSTATE_RECORDS.fetch_add(1, Ordering::Relaxed); + if entry_type != STATS_PSTATE_TYPE { return; } + let role = role.min(1); + PSTATE_TYPE10.fetch_add(1, Ordering::Relaxed); + let bucket = stats_pstate_bucket(qword3); + PSTATE_COUNT[bucket].fetch_add(1, Ordering::Relaxed); + let previous = PSTATE_PREV[role][1].load(Ordering::Relaxed); + let previous_ts = PSTATE_PREV[role][0].load(Ordering::Relaxed); + if previous != 0 && timestamp > previous_ts { + PSTATE_TICKS[(previous - 1) as usize % PSTATE_BUCKETS] + .fetch_add(timestamp - previous_ts, Ordering::Relaxed); + } + PSTATE_PREV[role][0].store(timestamp, Ordering::Relaxed); + PSTATE_PREV[role][1].store(bucket as u64 + 1, Ordering::Relaxed); + PSTATE_LAST[0].store(timestamp, Ordering::Relaxed); + PSTATE_LAST[1].store(qword3, Ordering::Relaxed); + PSTATE_LAST[2].store(role as u64, Ordering::Relaxed); + let slot = PSTATE_NEXT.fetch_add(1, Ordering::Relaxed) % PSTATE_RING; + PSTATE_SAMPLES[slot][0].store(timestamp, Ordering::Relaxed); + PSTATE_SAMPLES[slot][1].store((qword3 & !(1 << 63)) | (role as u64) << 63, Ordering::Relaxed); +} + +unsafe extern "C" fn pstate_set(value: *const core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if value.is_null() { return -22; } + let mut command = [0u8; 16]; + let mut len = 0; + while len < command.len() { + // SAFETY: the parameter core supplies a NUL-terminated value. + let byte = unsafe { *value.add(len) } as u8; + if byte == 0 { break; } + command[len] = byte; + len += 1; + } + if len == command.len() { return -22; } + let text = match core::str::from_utf8(&command[..len]) { Ok(text) => text.trim(), Err(_) => return -22 }; + match text { + "0" => PSTATE_ON.store(0, Ordering::Release), + "1" => PSTATE_ON.store(1, Ordering::Release), + "reset" => { + for counter in [&PSTATE_RECORDS, &PSTATE_TYPE10, &PSTATE_READ_FAILS] { + counter.store(0, Ordering::Relaxed); + } + for counter in PSTATE_LAST.iter().chain(PSTATE_COUNT.iter()).chain(PSTATE_TICKS.iter()) { + counter.store(0, Ordering::Relaxed); + } + for role in &PSTATE_PREV { role[0].store(0, Ordering::Relaxed); role[1].store(0, Ordering::Relaxed); } + for slot in &PSTATE_SAMPLES { slot[0].store(0, Ordering::Relaxed); slot[1].store(0, Ordering::Relaxed); } + PSTATE_NEXT.store(0, Ordering::Release); + } + _ => return -22, + } + 0 +} +unsafe extern "C" fn pstate_get(buf: *mut core::ffi::c_char, + _: *const bindings::kernel_param) -> core::ffi::c_int { + if buf.is_null() { return -22; } + // SAFETY: sysfs parameter output is PAGE_SIZE, at least 4096 bytes. + let bytes = unsafe { core::slice::from_raw_parts_mut(buf.cast::(), 4000) }; + let mut out = Output { bytes: &mut bytes[..], used: 0 }; + let _ = writeln!(out, "on={} records={} type10={} read_fails={} last_ts={:#x} last_q3={:#x} last_role={} now={:#x}", + PSTATE_ON.load(Ordering::Relaxed), PSTATE_RECORDS.load(Ordering::Relaxed), + PSTATE_TYPE10.load(Ordering::Relaxed), PSTATE_READ_FAILS.load(Ordering::Relaxed), + PSTATE_LAST[0].load(Ordering::Relaxed), PSTATE_LAST[1].load(Ordering::Relaxed), + PSTATE_LAST[2].load(Ordering::Relaxed), arch_counter()); + let _ = write!(out, "count"); + for counter in &PSTATE_COUNT { let _ = write!(out, " {}", counter.load(Ordering::Relaxed)); } + let _ = write!(out, "\nticks"); + for counter in &PSTATE_TICKS { let _ = write!(out, " {}", counter.load(Ordering::Relaxed)); } + let _ = writeln!(out); + let next = PSTATE_NEXT.load(Ordering::Acquire); + let first = next.saturating_sub(PSTATE_RING); + for index in first..next { + let slot = &PSTATE_SAMPLES[index % PSTATE_RING]; + let _ = writeln!(out, "{:x} {:x}", slot[0].load(Ordering::Relaxed), slot[1].load(Ordering::Relaxed)); + } + out.used as core::ffi::c_int +} +static PSTATE_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { + flags: 0, set: Some(pstate_set), get: Some(pstate_get), free: None, +}; +#[link_section = "__param"] +#[used] +static PSTATE_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g17p_stats_pstate" + } else { c"asahi.g17p_stats_pstate" }), + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PSTATE_OPS), perm: 0o600, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: core::ptr::null_mut() }, +}); diff --git a/drivers/gpu/drm/asahi/g18/debug.rs b/drivers/gpu/drm/asahi/g18/debug.rs new file mode 100644 index 0000000000000..8336c9f67df60 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/debug.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 kick, frame-pass and resource logging policy (`g18_kick_log`). +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. +use core::sync::atomic::{AtomicBool, Ordering}; +use crate::module_parameters; + +/// Set once the G18 application DRM node is live (`G18_DRM_READY`). +static G18_DRM_LIVE: AtomicBool = AtomicBool::new(false); + +/// Mark the G18 application DRM node live. Output before this is unaffected. +pub(crate) fn g18_mark_drm_live() { + G18_DRM_LIVE.store(true, Ordering::Relaxed); +} + +/// Whether a per-kick, frame-pass or per-job resource line is printed: always +/// before `G18_DRM_READY`, afterwards only with `g18_kick_log=1`. Fault, +/// recovery, refusal, wait, stall and admission lines never consult this. +#[inline(always)] +pub(crate) fn g18_kick_log() -> bool { + *module_parameters::g18_kick_log.value() != 0 || !G18_DRM_LIVE.load(Ordering::Relaxed) +} + +/// dev_info!() for a G18 per-kick, frame-pass or per-job resource line +#[macro_export] +macro_rules! g18_kick_info ( + ($dev:expr, $($arg:tt)*) => ( + if $crate::g18::debug::g18_kick_log() { + ::kernel::dev_info! ( $dev, $($arg)* ); + } + ) +); + +/// pr_info!() for a G18 per-kick, frame-pass or per-job resource line +#[macro_export] +macro_rules! g18_kick_pr_info ( + ($($arg:tt)*) => ( + if $crate::g18::debug::g18_kick_log() { + ::kernel::pr_info! ( $($arg)* ); + } + ) +); + +/// dev_info!() for a firmware event record: only a `routine` record follows +/// `g18_kick_log`; every other record is always printed +#[macro_export] +macro_rules! g18_event_info ( + ($routine:expr, $dev:expr, $($arg:tt)*) => ( + if !($routine) || $crate::g18::debug::g18_kick_log() { + ::kernel::dev_info! ( $dev, $($arg)* ); + } + ) +); diff --git a/drivers/gpu/drm/asahi/g18/drm.rs b/drivers/gpu/drm/asahi/g18/drm.rs new file mode 100644 index 0000000000000..9e826e69d4d8a --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/drm.rs @@ -0,0 +1,709 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G18 (J873, T8152/G18G) DRM render-node adapter. +//! +//! A structural copy of the M5 adapter (`m5_drm.rs`): synchronous +//! submission, one hardware command per submit, out-fences signalled by a +//! publication guard. It differs where G18 differs: +//! - render commands go through `g18_application` to a job sink the G18 +//! runtime implements, not through a captured reference profile; +//! - `map_timestamp_buffer` is implemented (contract K8), so Mesa's G18G +//! screen exposes timer queries and with them GL 3.3-4.6; +//! - compute is refused unless the adapter is built with `admit_compute`. +//! +//! The adapter holds no firmware state. Everything it needs from the live +//! G18 session goes through [`G18DrmRuntime`]. +//! +//! [`asahi_gem_prime_import`] is the driver's `drm_driver.gem_prime_import` +//! hook. On the G18 device it stops foreign dma-bufs from becoming bare +//! shmem objects which asahi ioctls then misread (module 1134); every other +//! device keeps the DRM core's default import, as on t6050-gpu. + +use core::ops::Range; +use core::sync::atomic::{fence, AtomicBool, AtomicPtr, Ordering}; +use kernel::{bindings, c_str, macros::vtable, new_mutex, prelude::*, static_lock_class, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + sync::{Arc, Mutex}, uapi, xarray}; +use crate::{alloc, file, g18::uapi::command as g17_uapi, g18::hw, + g18::address_map as va, + g18::uapi::application::{self as g18_application, G18ApplicationRefusal, G18RenderJobSink, G18RenderRetirement}, + gem, gpu, g18::fw::vm_layout as m5_vm, mmu, queue}; + +const USER_END: u64 = (1 << 42) - 2 * mmu::UAT_PGSZ as u64; +/// Largest compute control stream admitted, in bytes. The kernel does not read +/// the stream (the CDM does); this only bounds what the firmware is pointed at. +/// 64 KiB is a sizing choice, not a hardware limit: no decode names one. It +/// admits a stream of many launches, which cost 2.10 ms per op on the M6 against +/// 10.45 ms when each op is its own stream. Only reachable with `admit_compute`. +pub(crate) const G18_COMPUTE_STREAM_BYTES: u64 = 64 * 1024; +/// `DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE`: the dispatch binds texture or image +/// state, so the command takes the native texture compute mode (d62a, +/// `g18_application_compute_resources.rs`). The only compute flag G18 admits. +/// Spelled as its value, `(1 << 1)` in include/uapi/drm/asahi_drm.h, because +/// the incremental module build reuses pinned bindgen output that predates it. +const G18_COMPUTE_TEXTURE_STATE: u32 = 1 << 1; + +// Identity Mesa's G18G bring-up gate accepts (`agx_chip.c:471-477`) and the +// drm-shim `g18g` profile publishes (`asahi_noop.c:129-147`), measured on +// J873: two clusters of six cores (ioreg core_mask_list=(63,63)). +const G18_CHIP_ID: u32 = 0x8152; +const G18_CLUSTERS: u32 = 2; +const G18_CORES_PER_CLUSTER: u32 = 6; +const G18_CORE_MASK: u32 = 0x3f; +const G18_USC_GENERATION: u32 = 3; +/// Revision B1, what the Linux SGX ID probe decodes on J873. Used only when +/// the runtime supplies no observed identity. +const G18_REVISION_B1: u32 = hw::GpuRevision::B1 as u32; +/// `DRM_ASAHI_GPU_HAL_300`: the UAPI has no HAL400 value yet and Mesa's +/// G18G gate requires 300. Apple's kext is PI_400. +const G18_UAPI_HAL_GENERATION: u32 = 300; + +/// Kernel-VM window for K8 timestamp buffers. It must stay outside every +/// fixed G18 reservation; the assertion below re-checks that whenever the +/// address map changes. +pub(crate) const G18_TIMESTAMP_KERNEL_RANGE: Range = + (va::KERNEL_BASE + 0xc600_0000)..(va::KERNEL_BASE + 0xc700_0000); + +const fn timestamp_range_is_free() -> bool { + let (start, end) = (G18_TIMESTAMP_KERNEL_RANGE.start, G18_TIMESTAMP_KERNEL_RANGE.end); + let mut i = 0; + while i < va::REGIONS.len() { + let region = va::REGIONS[i]; + if start < region.base + region.bytes && region.base < end { + return false; + } + i += 1; + } + true +} +const _: () = assert!(timestamp_range_is_free()); + +static QUEUE_NAME: &CStr = c_str!("g18_queue"); +static QUEUE_CLASS_KEY: Pin<&'static kernel::sync::LockClassKey> = static_lock_class!(); + +/// What the adapter needs from the live G18 session. The conductor +/// implements it over the G18 runtime once firmware and the graphics +/// queues are up (see the G18 DRM adapter notes). +pub(crate) trait G18DrmRuntime: G18RenderJobSink { + /// Any crash, fault or quarantine that ends submission for this boot. + fn is_crashed(&self) -> bool; + /// VM_CREATE: a user VM in the session UAT. The adapter has already + /// checked `kernel_range` against the M5/Mesa compact contract. + fn new_user_vm(&self, id: u64, kernel_range: Range) -> Result; + /// K8: map a timestamp BO range. [`map_timestamp_buffer_in`] is the + /// intended implementation. + fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) + -> Result; + /// Compute (K1). Only called when the adapter admits compute. + fn submit_compute(&self, _vm: &mmu::Vm, _command: &g17_uapi::TranslatedComputeCommand) + -> Result { + Err(ENOTSUPP) + } +} + +/// K8 mapping for [`G18DrmRuntime::map_timestamp_buffer`]: the G17P manager +/// body (`g17_manager.rs:3949-3962`) in the G18 kernel VM, inside +/// [`G18_TIMESTAMP_KERNEL_RANGE`]. The adapter writes timestamps through the +/// mapping's CPU view; the firmware-shared GPU view is what the file layer +/// retains and what firmware-written timestamps will use later. +pub(crate) fn map_timestamp_buffer_in(uat: &mmu::Uat, mut bo: gem::ObjectRef, + range: Range) -> Result { + bo.map_range_into_range(uat.kernel_vm(), range, G18_TIMESTAMP_KERNEL_RANGE, + mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false) +} + +/// Adapter options fixed at registration. +#[derive(Debug, Copy, Clone, Default)] +pub(crate) struct G18DrmConfig { + /// Admit compute commands (module parameter `g18_admit_compute`, default 0). + pub(crate) admit_compute: bool, + /// Admit exactly the native 4x MSAA render set (module parameter + /// `g18_admit_msaa4`, default 0; msaa163). Off refuses every + /// multisampled render as before. + pub(crate) admit_msaa4: bool, + /// Admit renders carrying `DRM_ASAHI_RENDER_G18_RESOURCES` (module + /// parameter `g18_admit_graphics_scratch`, default 0). Off leaves the + /// feature unadvertised and refuses the flag, so Mesa keeps refusing + /// draws whose programs need a thread-private frame. + pub(crate) admit_graphics_scratch: bool, +} + +pub(crate) struct G18DrmGpu { + runtime: Arc, + /// Serializes job-sink calls across every queue of this device. + submit_lock: Arc>, + config: G18DrmConfig, + revision: u32, + ids: gpu::SequenceIDs, +} + +impl G18DrmGpu { + /// `observed` is the SGX identity read after firmware start, if the + /// runtime has one. It must match the published topology exactly. + pub(crate) fn new(runtime: Arc, observed: Option<&hw::GpuIdConfig>, + config: G18DrmConfig) -> Result { + let revision = match observed { + Some(id) => { + if id.gpu_gen != hw::GpuGen::G18 || id.gpu_variant != hw::GpuVariant::G + || id.num_dies != 1 || id.num_clusters != G18_CLUSTERS + || id.num_cores != G18_CORES_PER_CLUSTER + || id.usc_generation != G18_USC_GENERATION + || id.core_masks.as_slice() != [G18_CORE_MASK; G18_CLUSTERS as usize] { + pr_err!("G18_DRM: observed identity {:?} is not the J873 G18G topology\n", id); + return Err(ENODEV); + } + id.gpu_rev as u32 + } + None => G18_REVISION_B1, + }; + let submit_lock = Arc::pin_init(new_mutex!(()), GFP_KERNEL)?; + Ok(Self { runtime, submit_lock, config, revision, ids: Default::default() }) + } +} + +// Module 1134 reached G18 through its `DrmGpu` trait (drm_gpu.rs) and +// computed GET_PARAMS in file.rs from `DrmGpu::params`; here the same facts +// fill `drm_asahi_params_global` through `gpu::Gpu::get_params`. file.rs +// adds VM_STATUS (supports_vm_status), leaves out VM_STATUS_MIRROR +// (supports_vm_status_mirror is false, as in 1134) and INDEPENDENT_QUEUES +// (no queue limits), and adds SOFT_FAULTS when fault_control is 0xb. +impl gpu::Gpu for G18DrmGpu { + fn as_any(&self) -> &dyn core::any::Any { self } + fn arc_as_any(self: Arc) -> Arc { self } + fn manager(&self) -> Option<&dyn gpu::GpuManager> { None } + fn init(&self) -> Result { + if self.is_crashed() { Err(ENODEV) } else { Ok(()) } + } + fn ids(&self) -> &gpu::SequenceIDs { &self.ids } + fn is_crashed(&self) -> bool { self.runtime.is_crashed() } + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + let mut core_masks = [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize]; + for mask in core_masks.get_mut(..G18_CLUSTERS as usize).ok_or(EIO)? { + *mask = G18_CORE_MASK; + } + params.gpu_generation = hw::GpuGen::G18 as u32; + params.gpu_variant = hw::GpuVariant::G as u32; + params.gpu_revision = self.revision; + params.chip_id = G18_CHIP_ID; + params.num_dies = 1; + params.num_clusters_total = G18_CLUSTERS; + params.num_cores_per_cluster = G18_CORES_PER_CLUSTER; + params.core_masks = core_masks.map(|mask| mask.into()); + // No G18 clock policy is implemented by this adapter. + params.max_frequency_khz = 0; + params.usc_generation = G18_USC_GENERATION; + params.gpu_hal_generation = G18_UAPI_HAL_GENERATION; + params.max_commands_per_submission = 1; + if self.config.admit_graphics_scratch { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH as u64; + } + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES as u64; + Ok(()) + } + fn supports_vm_status(&self) -> bool { true } + fn supports_vm_status_mirror(&self) -> bool { false } + /// Module 1134 validated and resolved every VM_BIND op before committing. + fn batched_vm_bind(&self) -> bool { true } + /// Module 1134 recorded `submission_error()` (EIO once crashed) on every VM_STATUS query. + fn submission_error(&self) -> Option { self.is_crashed().then_some(EIO) } + /// Module 1134 copied the command buffer before it parsed the syncs. + fn copy_commands_before_syncs(&self) -> bool { true } + /// Module 1134 refused HIGH and REALTIME queues with EINVAL. + fn allows_elevated_queue_priority(&self) -> bool { false } + /// Module 1134 had no GEM_MADVISE ioctl. + fn supports_gem_madvise(&self) -> bool { false } + /// Module 1134 refused bind flag bit 4 with EINVAL, and the T8152 UAT maps no + /// sparse tiles: a failed tile bind would clear the whole range, driver aliases + /// included. + fn supports_sparse_tiles(&self) -> bool { false } + /// `DRM_IOCTL_ASAHI_GET_TIME` scales CNTPCT_EL0 by this rate + /// (`file.rs:1225-1241`); report the counter's own frequency, as G17P + /// does (`g17_drm.rs:1209-1222`). J873 runs it at 24 MHz. + fn base_clock_hz(&self) -> u32 { counter_frequency() as u32 } + /// 42-bit T8152 user space: user range UAT_PGSZ..USER_END with the dummy + /// page at USER_END, as in module 1134. + fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } + fn new_vm(&self, kernel_range: Range) -> Result { + if self.is_crashed() { return Err(ENODEV); } + m5_vm::compact_range(&kernel_range).ok_or(EINVAL)?; + let vm = self.runtime.new_user_vm(self.ids.vm.next(), kernel_range)?; + // Module 1134 VM_CREATE: a simplefb alias is T8140-only; any other + // GPU refuses the VM when the diagnostic parameter is set. + if *crate::module_parameters::g17p_simplefb_iova.value() != 0 { + return Err(ENOTSUPP); + } + Ok(vm) + } + fn new_queue(&self, vm: mmu::Vm, + _ualloc: Arc>, + _ualloc_priv: Arc>, + priority: u32, usc_exec_base: u64) -> Result> { + if priority > 3 { return Err(EINVAL); } + let window = g17_uapi::QueueUscWindow { + base: usc_exec_base, user_start: m5_vm::DRIVER_BASE, user_end: USER_END, + }; + window.validate().map_err(|_| EINVAL)?; + let queue = G18Queue { + runtime: self.runtime.clone(), submit_lock: self.submit_lock.clone(), + config: self.config, vm, window, + fences: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, + }; + Ok(KBox::new(queue, GFP_KERNEL)?) + } + /// K8. Mesa binds `128 * sizeof(struct agx_timestamps)` of a WRITEBACK + /// BO at screen creation and enables timer queries only if this works. + fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) + -> Result { + if self.is_crashed() { return Err(ENODEV); } + if range.is_empty() { return Err(EINVAL); } + self.runtime.map_timestamp_buffer(bo, range) + } +} + +/// CNTFRQ_EL0, or Apple's 24 MHz if boot firmware left it unprogrammed: +/// GET_TIME divides by this rate (`file.rs:1225-1229`) and must never see 0. +fn counter_frequency() -> u64 { + let frequency: u64; + // SAFETY: CNTFRQ_EL0 is a read-only system register. + unsafe { core::arch::asm!("mrs {x}, CNTFRQ_EL0", x = out(reg) frequency) }; + if frequency == 0 { 24_000_000 } else { frequency } +} + +/// The counter `DRM_IOCTL_ASAHI_GET_TIME` reads, ordered after prior code. +fn counter_ticks() -> u64 { + let ticks: u64; + // SAFETY: the ISB only orders the read; CNTPCT_EL0 is read-only. + unsafe { core::arch::asm!("isb", "mrs {x}, CNTPCT_EL0", x = out(reg) ticks) }; + ticks +} + +/// GET_TIME's conversion (`file.rs:1225-1241`): floor(ticks * 1e9 / f), the +/// same rational as its gcd-reduced form. Split at whole seconds so no step +/// overflows (the remainder term is below f * 1e9 < 2^64 for any u32 rate) +/// and no 128-bit division (`__udivti3`) reaches the module. +fn ticks_to_ns(ticks: u64, frequency: u64) -> u64 { + const NSEC_PER_SEC: u64 = 1_000_000_000; + (ticks / frequency).wrapping_mul(NSEC_PER_SEC) + .wrapping_add(ticks % frequency * NSEC_PER_SEC / frequency) +} + +struct G18SubmissionFence; +#[vtable] +impl FenceOps for G18SubmissionFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("g18-submit") } +} + +/// Keep the original sync vector: creating a second output list would add an +/// allocation failure after userspace has already supplied its wait targets. +struct OutputPublication { + fence: UserFence, + syncs: KVec, + input_count: usize, + result: Result, +} + +impl Drop for OutputPublication { + fn drop(&mut self) { + if let Err(error) = self.result { self.fence.set_error(error); } + let fence = Fence::from_fence(&self.fence); + for mut sync in self.syncs.drain(self.input_count..) { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj.add_point(chain, &fence, sync.timeline_value); + } else { sync.syncobj.replace_fence(Some(&fence)); } + } + self.fence.signal(); + } +} + +struct UserAddressSpace<'a> { vm: &'a mmu::Vm } +impl g17_uapi::GpuAddressSpace for UserAddressSpace<'_> { + fn covers(&self, address: u64, size: u64, access: g17_uapi::GpuAccess) -> bool { + let (read, write) = match access { + g17_uapi::GpuAccess::Read => (true, false), + g17_uapi::GpuAccess::Write => (false, true), + g17_uapi::GpuAccess::ReadWrite => (true, true), + }; + m5_vm::user_range_contains(&self.vm.kernel_range(), address, size, USER_END) + && self.vm.covers_range(address, size, read, write) + } +} + +/// One UAPI timestamp destination inside a bound timestamp buffer. +struct TimestampSlot { + mapping: Arc, + offset: usize, +} + +impl TimestampSlot { + /// Resolve before any GPU work, so a bad handle fails the submit cleanly. + fn resolve(objects: Pin<&xarray::XArray>>, + timestamp: g17_uapi::UapiTimestamp) -> Result> { + if timestamp.handle == 0 { + return if timestamp.offset == 0 { Ok(None) } else { Err(EINVAL) }; + } + let guard = objects.lock(); + let object = guard.get(timestamp.handle.try_into()?).ok_or(ENOENT)?.clone(); + core::mem::drop(guard); + match object { + file::Object::TimestampBuffer(mapping) => { + let offset: usize = timestamp.offset.try_into()?; + if offset.checked_add(8).ok_or(EINVAL)? > mapping.size() { + return Err(ERANGE); + } + Ok(Some(Self { mapping, offset })) + } + } + } + + /// Publish one timestamp in the UAPI unit, GET_TIME nanoseconds. + fn write(&self, ticks: u64, frequency: u64) -> Result { + let value = ticks_to_ns(ticks, frequency); + self.mapping.with_cpu_bytes(|bytes| { + let end = self.offset.checked_add(8).ok_or(EINVAL)?; + bytes.get_mut(self.offset..end).ok_or(EINVAL)? + .copy_from_slice(&value.to_le_bytes()); + Ok(()) + }).inspect_err(|error| pr_err!( + "G18 timestamp not published: {:?} offset={:#x} mapping={:#x}\n", + error, self.offset, self.mapping.size()))?; + fence(Ordering::Release); + Ok(()) + } +} + +/// Write the requested stage timestamps after retirement. Without job stage +/// ticks, the CPU bracket `[t0, t1]` around the synchronous job bounds every +/// stage: starts get t0, ends get t1. +fn publish_timestamps(slots: &[Option], stage_ticks: &[u64]) -> Result { + let frequency = counter_frequency(); + for (slot, &ticks) in slots.iter().zip(stage_ticks) { + if let Some(slot) = slot { slot.write(ticks, frequency)?; } + } + Ok(()) +} + +struct G18Queue { + runtime: Arc, + submit_lock: Arc>, + config: G18DrmConfig, + vm: mmu::Vm, + window: g17_uapi::QueueUscWindow, + fences: FenceContexts, +} + +impl queue::Queue for G18Queue { + fn submit(&mut self, _submission_id: u64, syncs: KVec, + in_sync_count: usize, cmdbuf_raw: &[u8], + objects: Pin<&xarray::XArray>>) -> Result { + if in_sync_count > syncs.len() { return Err(EINVAL); } + let fence = self.fences.new_fence(0, G18SubmissionFence)?.into(); + let mut publication = OutputPublication { + fence, syncs, input_count: in_sync_count, result: Err(ECANCELED), + }; + let result = self.submit_one(&publication.syncs[..in_sync_count], cmdbuf_raw, objects); + // The runtime records and quarantines a failed published VM before + // this output fence is installed/signalled by the publication guard. + publication.result = result; + result + } +} + +impl G18Queue { + fn submit_one(&self, inputs: &[file::SyncItem], cmdbuf_raw: &[u8], + objects: Pin<&xarray::XArray>>) -> Result { + if self.vm.status().map_or(0, |s| s.get()) != 0 { return Err(EIO); } + if self.runtime.is_crashed() { return Err(ENODEV); } + // Wait outside the submission lock; another client may own the job + // whose fence satisfies this submission's dependency. + for sync in inputs { + if let Some(fence) = &sync.fence { + // SAFETY: both calls borrow the strongly held input fence. + let waited = unsafe { bindings::dma_fence_wait_timeout( + fence.raw(), true, kernel::task::MAX_SCHEDULE_TIMEOUT) }; + if waited < 0 { return Err(Error::from_errno(waited as i32)); } + if waited == 0 { return Err(ETIMEDOUT); } + let status = unsafe { bindings::dma_fence_get_status(fence.raw()) }; + if status < 0 { return Err(Error::from_errno(status)); } + } + } + // An empty command buffer is a synchronization-only submission. Input + // fences have completed above; the normal publication guard installs + // and signals its output fence, preserving timeline chaining. Never + // manufacture a hardware job or relax hardware retirement checks. + if cmdbuf_raw.is_empty() { + let _serialized = self.submit_lock.lock(); + if self.vm.status().map_or(0, |s| s.get()) != 0 { return Err(EIO); } + if self.runtime.is_crashed() { return Err(ENODEV); } + pr_info!("G18_SYNC_ONLY inputs={}\n", inputs.len()); + return Ok(()); + } + let _vm_job = self.vm.retain_t8140_job()?; + // Parsing and translation frames end here, before the job runs its + // own register builders on this stack. + let admitted = self.admit(cmdbuf_raw, objects)?; + let _serialized = self.submit_lock.lock(); + // Commands run in submission order and each retires before the next + // starts, so every barrier (an earlier command of this submission, or + // any earlier submission) has completed when its waiter begins. + for command in admitted { + if self.runtime.is_crashed() { return Err(ENODEV); } + let t0 = counter_ticks(); + match command { + Admitted::Render(render, slots) => { + let retired = self.runtime.submit_render(&self.vm, &render)?; + let t1 = counter_ticks(); + publish_timestamps(&slots, &retired.stage_ticks.unwrap_or([t0, t1, t0, t1]))?; + } + Admitted::Compute(command, slots) => { + let retired = self.runtime.submit_compute(&self.vm, &command)?; + let t1 = counter_ticks(); + let [start, _, _, end] = retired.stage_ticks.unwrap_or([t0, t1, t0, t1]); + publish_timestamps(&slots, &[start, end])?; + } + } + } + Ok(()) + } + + /// Parse every hardware command of a submission, apply the G18 admission + /// rules and resolve their timestamp destinations, all before any runs: + /// a submission is admitted whole or not at all. Nothing here touches + /// the GPU. A compute prepass and the render that consumes it (transform + /// feedback, geometry and tessellation draws) arrive as one submission. + #[inline(never)] + fn admit(&self, cmdbuf_raw: &[u8], objects: Pin<&xarray::XArray>>) + -> Result> { + let malformed = |error: g17_uapi::UapiTranslateError| { + pr_err!("G18 submit admission rejected: {:?}\n", error); + EINVAL + }; + let mut parser = g17_uapi::UapiCommandParser::new(cmdbuf_raw); + let mut admitted = KVec::new(); + while let Some(command) = parser.next_hardware().map_err(malformed)? { + let command = match command { + g17_uapi::ParsedHardwareCommand::Render { + payload, vertex_attachments, fragment_attachments, .. + } => self.admit_render(&payload, vertex_attachments, fragment_attachments, objects)?, + g17_uapi::ParsedHardwareCommand::Compute { payload, attachments, .. } => + self.admit_compute(&payload, attachments, objects)?, + }; + admitted.push(command, GFP_KERNEL)?; + } + parser.finish().map_err(malformed)?; + Ok(admitted) + } + + #[inline(never)] + fn admit_render(&self, payload: &g17_uapi::UapiRenderCommand, + vertex_attachments: g17_uapi::UapiAttachmentList, + fragment_attachments: g17_uapi::UapiAttachmentList, + objects: Pin<&xarray::XArray>>) -> Result { + let address_space = UserAddressSpace { vm: &self.vm }; + let scratch = if payload.flags & g17_uapi::RENDER_G18_RESOURCES != 0 { + Some(self.admit_render_scratch(payload)?) + } else { + None + }; + // The shared translator refuses the G18 flag and never sees the + // suffix. Only this adapter consumes and strips a validated suffix. + let mut untagged = *payload; + untagged.flags &= !g17_uapi::RENDER_G18_RESOURCES; + untagged.g18_resources = [0; 20]; + let mut render = g18_application::translate_g18_render_admitting(&untagged, + vertex_attachments, fragment_attachments, self.window, &address_space, + self.config.admit_msaa4) + .map_err(|refusal| refuse_render(payload, refusal))?; + render.scratch = scratch; + let render = KBox::new(render, GFP_KERNEL)?; + if !g18_application::job_owned_fields_clear(&render.parameters, &render.controls) { + pr_err!("G18 render admission rejected: job-owned fields set\n"); + return Err(EINVAL); + } + let slots = [ + TimestampSlot::resolve(objects, payload.vertex_timestamps.start)?, + TimestampSlot::resolve(objects, payload.vertex_timestamps.end)?, + TimestampSlot::resolve(objects, payload.fragment_timestamps.start)?, + TimestampSlot::resolve(objects, payload.fragment_timestamps.end)?, + ]; + Ok(Admitted::Render(render, slots)) + } + + /// Derive the TA/3D private-memory selection for a flagged render. Only + /// single-sample, single-layer renders with one-page tile storage are in + /// the evidenced envelope (g18_graphics_scratch.rs). The selection never + /// resizes the owned pool: ideal when both kicks fit it, else the + /// native minimum result, else ENOSPC. Nothing is published on refusal. + #[inline(never)] + fn admit_render_scratch(&self, payload: &g17_uapi::UapiRenderCommand) + -> Result { + use crate::{g18::uapi::graphics_scratch::G18RenderMetadata, g18::uapi::render_policy as policy}; + if !self.config.admit_graphics_scratch { + pr_err!("G18 render admission rejected: graphics scratch is disabled (g18_admit_graphics_scratch=0)\n"); + return Err(ENOTSUPP); + } + if payload.samples != 1 || payload.layers != 1 + || (payload.utile_width, payload.utile_height) != (32, 32) { + pr_err!("G18 render admission rejected: graphics scratch outside single-sample 32x32 envelope samples={} layers={} utiles={}x{}\n", + payload.samples, payload.layers, payload.utile_width, payload.utile_height); + return Err(ENOTSUPP); + } + let tile_bytes = 32 * 32 * u32::from(payload.sample_size); + let metadata = G18RenderMetadata::decode(&payload.g18_resources, tile_bytes) + .ok_or_else(|| { + pr_err!("G18 render admission rejected: invalid graphics scratch metadata words={:x?} tile_bytes={}\n", + payload.g18_resources, tile_bytes); + EINVAL + })?; + let plan = metadata.plan(policy::RENDER_POOL_BLOCK_BYTES).ok_or_else(|| { + pr_err!("G18 render admission rejected: graphics scratch exceeds the native pool capacity\n"); + ENOSPC + })?; + let (owned_bytes, owned_chunks) = policy::owned_scratch_capacity(); + let selected = plan.select(owned_bytes, owned_chunks).ok_or_else(|| { + pr_err!("G18 render admission rejected: graphics scratch needs {:?} beyond owned pool {} bytes / {} chunks\n", + plan.minimum, owned_bytes, owned_chunks); + ENOSPC + })?; + pr_info!("G18_RENDER_SCRATCH vertex={:?} fragment={:?} tiles={:?} tile_bytes={} mode={:?} ta={:?} d3={:?}\n", + metadata.vertex, metadata.fragment, metadata.tiles, tile_bytes, selected.mode, + selected.stages[0], selected.stages[1]); + Ok(selected) + } + + #[inline(never)] + fn admit_compute(&self, payload: &g17_uapi::UapiComputeCommand, + attachments: g17_uapi::UapiAttachmentList, + objects: Pin<&xarray::XArray>>) -> Result { + if !self.config.admit_compute { + pr_err!("G18 compute admission rejected: compute is disabled (g18_admit_compute=0)\n"); + return Err(ENOTSUPP); + } + let stream_bytes = payload.control_stream_end.saturating_sub(payload.control_stream_base); + let g18_flags = G18_COMPUTE_TEXTURE_STATE | g17_uapi::COMPUTE_G18_RESOURCES; + // ADD3 and every flag outside the G18 contract remain refused. + if payload.flags & !g18_flags != 0 || payload.sampler_heap != 0 + || payload.sampler_count != 0 || stream_bytes > G18_COMPUTE_STREAM_BYTES { + pr_err!("G18 compute admission rejected flags={:#x} sampler_heap={:#x} samplers={} stream_bytes={} (admits at most {} bytes)\n", + payload.flags, payload.sampler_heap, payload.sampler_count, stream_bytes, + G18_COMPUTE_STREAM_BYTES); + return Err(ENOTSUPP); + } + let resource_descriptor = if payload.flags & g17_uapi::COMPUTE_G18_RESOURCES != 0 { + let metadata = crate::g18::uapi::compute_metadata::G18ComputeMetadata::decode(payload.g18_resources) + .ok_or_else(|| { + pr_err!("G18 compute admission rejected: invalid resource metadata\n"); + EINVAL + })?; + if crate::g18::uapi::compute_plan::G18ComputePlan::from_descriptor(&metadata.descriptor).is_none() { + pr_err!("G18 compute admission rejected: resource plan exceeds supported capacity\n"); + return Err(ENOSPC); + } + // Main static TPS uses the metadata-derived owned pool and SMA + // launch pair. Mesa's G18 compute encoder carries the same aligned + // frame in both SPECTPR scratch fields. Constant-program scratch + // and SCS still need their separate launch contract. + if metadata.demand.constant_scratch_bytes != 0 || payload.scs_layout != 0 + { + pr_err!("G18 compute admission rejected: constant scratch or SCS integration pending\n"); + return Err(ENOTSUPP); + } + crate::g18_kick_pr_info!("G18_COMPUTE_METADATA stream={:#x} registers={} scratch_bytes={} spectpr_bytes={} constant_scratch={} local={:?}\n", + payload.control_stream_base, payload.g18_resources[2], + metadata.demand.scratch_bytes, payload.g18_resources[4], + metadata.demand.constant_scratch_bytes, &payload.g18_resources[12..15]); + Some(metadata.descriptor) + } else { + None + }; + // The shared G17 translator refuses G18 flags and nonzero resource + // words. Only this adapter may consume and strip a validated suffix. + let mut untagged = *payload; + untagged.flags &= !g18_flags; + if resource_descriptor.is_some() { untagged.g18_resources = [0; 16]; } + let address_space = UserAddressSpace { vm: &self.vm }; + let mut translated = g17_uapi::translate_compute_command(untagged, attachments, + self.window, &address_space).map_err(|error| { + pr_err!("G18 compute admission rejected {:?}: helper={:#x}/{:#x}/{:#x} flags={:#x}\n", + error, payload.helper.binary, payload.helper.config, payload.helper.data, payload.flags); + EINVAL + })?; + translated.g18_texture_state = payload.flags & G18_COMPUTE_TEXTURE_STATE != 0; + translated.g18_resource_descriptor = resource_descriptor; + let slots = [ + TimestampSlot::resolve(objects, payload.timestamps.start)?, + TimestampSlot::resolve(objects, payload.timestamps.end)?, + ]; + Ok(Admitted::Compute(KBox::new(translated, GFP_KERNEL)?, slots)) + } +} + +/// An admitted command, heap-held so the submit frame stays small. +enum Admitted { + Render(KBox, [Option; 4]), + Compute(KBox, [Option; 2]), +} + +/// Log a render refusal with the fields that decided it and map it to the +/// M5 errno convention: EINVAL for a malformed command, ENOTSUPP otherwise. +fn refuse_render(payload: &g17_uapi::UapiRenderCommand, refusal: G18ApplicationRefusal) -> Error { + pr_err!("G18 render admission rejected {:?}: size={}x{} layers={} utiles={}x{} samples={} msaa={:#x} flags={:#x} zls={:#x} depth={:#x}/{:#x} stencil={:#x} bgobjvals={:#x} samplers={}\n", + refusal, payload.width, payload.height, payload.layers, payload.utile_width, + payload.utile_height, payload.samples, payload.multisample_control, payload.flags, + payload.zls_control, payload.depth.base, payload.depth.compression_base, + payload.stencil.base, payload.stencil_clear, payload.sampler_count); + if refusal.is_invalid() { EINVAL } else { ENOTSUPP } +} + +static FOREIGN_IMPORT_REPORTED: AtomicBool = AtomicBool::new(false); + +/// Parent `struct device` of the G18 DRM device, set before it registers. +static G18_PARENT: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); + +/// Record the platform device of the G18 DRM device, so that +/// [`asahi_gem_prime_import`] applies module 1134's import rule to it alone. +pub(crate) fn note_g18_parent(parent: *mut bindings::device) { + G18_PARENT.store(parent, Ordering::Release); +} + +/// `drm_driver.gem_prime_import` for asahi. A device other than G18 gets the +/// DRM core's default import, exactly what the core does without this hook +/// (t6050-gpu has none). The G18 device re-imports only dma-bufs that it +/// exported. Anything else is refused with EINVAL, the DRM +/// core's answer for a driver without import support +/// (`drm_prime.c:987-988`). +/// +/// Without this hook the core's default import builds a bare C +/// `drm_gem_shmem_object` (no `gem_create_object`, `shmem.rs:335-339`), +/// and every asahi ioctl that looks a handle up reads it as an +/// `Object` (`file.rs:989`, `shmem.rs:320-327`). +/// +/// # Safety +/// +/// The DRM core calls this with a valid `drm_device` and a dma-buf it holds +/// a reference on, for the duration of the call. +pub(crate) unsafe extern "C" fn asahi_gem_prime_import( + dev: *mut bindings::drm_device, + dma_buf: *mut bindings::dma_buf, +) -> *mut bindings::drm_gem_object { + // SAFETY: `dev` is a valid drm_device (caller contract); its parent is + // set at allocation and does not change while the device exists. + let parent = unsafe { (*dev).dev }; + if parent.is_null() || parent != G18_PARENT.load(Ordering::Acquire) { + // SAFETY: caller contract; this is the call the DRM core makes + // itself when a driver has no gem_prime_import hook. + return unsafe { bindings::drm_gem_prime_import(dev, dma_buf) }; + } + // SAFETY: both pointers are valid (caller contract). The helper compares + // `dma_buf->ops` with DRM's own PRIME ops before it reads `priv` as a GEM + // object (`drm_prime.c:945-951`). + if unsafe { bindings::drm_gem_is_prime_exported_dma_buf(dev, dma_buf) } { + // SAFETY: for a dma-buf this device exported, the helper only takes + // a reference on the existing GEM object (`drm_prime.c:977-983`), + // which is an `Object` created by this driver. + return unsafe { bindings::drm_gem_prime_import(dev, dma_buf) }; + } + if !FOREIGN_IMPORT_REPORTED.swap(true, Ordering::Relaxed) { + pr_warn!("asahi: refusing a dma-buf another driver exported; only asahi's own buffers can be imported\n"); + } + EINVAL.to_ptr() +} diff --git a/drivers/gpu/drm/asahi/g18/fixture/compute_payload.rs b/drivers/gpu/drm/asahi/g18/fixture/compute_payload.rs new file mode 100644 index 0000000000000..0fa3860c7c871 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fixture/compute_payload.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Private 26A428 compute-add capture, with a poisoned Linux output buffer. +//! All records are user virtual pages, never captured physical/kernel mappings. +//! This fixed diagnostic format does not grant permission to submit a GPU job. + +#![allow(dead_code)] + +use crate::g18::fw::compute::{G18ComputeCommandInputs, G18ComputeRegisterInputs}; +use crate::g18::fw::compute_state::{G18ComputeState, G18_COMPUTE_STATE_BYTES}; + +pub(crate) const HEADER_BYTES: usize = 64; +pub(crate) const PAGE_BYTES: usize = 0x4000; +pub(crate) const RECORD_BYTES: usize = PAGE_BYTES + 16; +pub(crate) const PAGE_COUNT: usize = 46; +pub(crate) const INPUT_A: u64 = 0x100_0003_0000; +pub(crate) const INPUT_B: u64 = 0x100_0003_8000; +pub(crate) const OUTPUT: u64 = 0x100_0004_0000; +pub(crate) const POISON: u32 = 0xdead_beef; + +// Exact allocations in submission 1. Submission 2 is the post-compute blit. +const RANGES: [(u64, usize); 15] = [ + (0, 0x10000), + (0x100_0000_0000, 0x10000), (0x100_0001_8000, 0x10000), + (INPUT_A, PAGE_BYTES), (INPUT_B, PAGE_BYTES), (OUTPUT, PAGE_BYTES), + (0x100_0004_8000, 0x20000), (0x100_0007_0000, 0x20000), + (0x100_0009_8000, 0x8000), (0x100_000a_8000, 0x8000), + (0x100_000b_8000, 0x8000), (0x100_000c_8000, 0x8000), + (0x100_000d_8000, 0x8000), (0x100_000e_8000, 0x8000), + (0x100_000f_8000, 0xc000), +]; + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum PayloadError { Header, Length, Checksum, Page, Input, Output, State } + +pub(crate) struct ComputePayload<'a> { bytes: &'a [u8] } +pub(crate) struct PayloadPage<'a> { pub(crate) address: u64, pub(crate) bytes: &'a [u8] } + +fn u32_at(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} +fn u64_at(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) +} + +/// Transfer corruption check only. Capture provenance is in the SHA256 receipt. +pub(crate) fn fnv64(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |h, b| + (h ^ u64::from(*b)).wrapping_mul(0x100_0000_01b3)) +} + +/// Build the captured addition's state and default device control fields. +/// Original 26A428 setup 8bf1280/8bf1414 supplies the spill inputs to 8bf16cc: +/// d518/d520/d524 and d5a8/d5b0/d5d8 are all zero for submission 1. Native +/// configureDevice stores at 8bc73fc, 8c492a0, 8bc93bc and 8bc7774 establish +/// the register defaults below. Optional tuning and debug policies stay off. +/// Context, pool, scratch, scheduler, completion and addresses belong to the +/// retained owners; this function never publishes or submits a command. +pub(crate) fn addition_inputs(state: &[u8]) + -> Result<(G18ComputeCommandInputs, G18ComputeRegisterInputs), PayloadError> { + if state.len() != G18_COMPUTE_STATE_BYTES || fnv64(state) != 0x956a_f8e1_8848_5ce8 { + return Err(PayloadError::State); + } + let mut command = G18ComputeCommandInputs::default(); + let mut registers = G18ComputeRegisterInputs { + a10aac: 1, + a6d0: 1, + clusters: 2, + cluster_width: 6, + ..G18ComputeRegisterInputs::default() + }; + G18ComputeState::parse(state).ok_or(PayloadError::State)?.apply(&mut command, &mut registers); + Ok((command, registers)) +} + +/// Exact IEEE754 encoding of a small signed integer divided by four. Keeps +/// float operations out of kernel code; the qualified values fit in 13 bits. +fn quarter_bits(value: i32) -> u32 { + if value == 0 { return 0; } + let magnitude = value.unsigned_abs(); + let top = 31 - magnitude.leading_zeros(); + (u32::from(value < 0) << 31) | ((top + 125) << 23) + | ((magnitude << (23 - top)) & 0x7f_ffff) +} + +impl<'a> ComputePayload<'a> { + pub(crate) fn parse(bytes: &'a [u8]) -> Result { + let expected = HEADER_BYTES + G18_COMPUTE_STATE_BYTES + PAGE_COUNT * RECORD_BYTES; + if bytes.len() != expected { return Err(PayloadError::Length); } + if &bytes[..8] != b"G18CMP01" || u32_at(bytes, 8) != 1 + || u32_at(bytes, 12) != HEADER_BYTES as u32 + || u32_at(bytes, 16) != PAGE_BYTES as u32 + || u32_at(bytes, 20) != PAGE_COUNT as u32 + || u32_at(bytes, 24) != G18_COMPUTE_STATE_BYTES as u32 + || u32_at(bytes, 28) != 4096 + || u64_at(bytes, 32) != INPUT_A || u64_at(bytes, 40) != INPUT_B + || u64_at(bytes, 48) != OUTPUT + { return Err(PayloadError::Header); } + if fnv64(&bytes[HEADER_BYTES..]) != u64_at(bytes, 56) { + return Err(PayloadError::Checksum); + } + let payload = Self { bytes }; + let mut index = 0; + for (base, size) in RANGES { + for offset in (0..size).step_by(PAGE_BYTES) { + let record = HEADER_BYTES + G18_COMPUTE_STATE_BYTES + index * RECORD_BYTES; + let page = payload.page(index).ok_or(PayloadError::Page)?; + if page.address != base + offset as u64 || u32_at(bytes, record + 8) != 3 + || u32_at(bytes, record + 12) != PAGE_BYTES as u32 + { return Err(PayloadError::Page); } + if page.address == OUTPUT { + if page.bytes.chunks_exact(4).any(|v| u32_at(v, 0) != POISON) { + return Err(PayloadError::Output); + } + } else if page.address == INPUT_A || page.address == INPUT_B { + for (i, word) in page.bytes.chunks_exact(4).enumerate() { + let value = if page.address == INPUT_A { 2 * i as i32 } else { 4000 - i as i32 }; + if u32_at(word, 0) != quarter_bits(value) { return Err(PayloadError::Input); } + } + } + index += 1; + } + } + if index != PAGE_COUNT { return Err(PayloadError::Page); } + let mut c = G18ComputeCommandInputs::default(); + let mut r = G18ComputeRegisterInputs::default(); + payload.state().apply(&mut c, &mut r); + if r.program != [0x100_000f_8000, 0x100_000c_8000, 0x100_000f_9b80, + 0x100_000f_9b88, 0x100_000f_9b90, 0x100_000f_9b98] + || c.d4e0 != 0x100_000c_802c || r.aliases != [0; 6] || r.d548 != 0 + { return Err(PayloadError::State); } + // All six program pointers and the command-stream end must be backed. + for address in r.program.into_iter().chain([c.d4e0]) { + if !RANGES.iter().any(|(base, size)| *base <= address && address < *base + *size as u64) { + return Err(PayloadError::State); + } + } + Ok(payload) + } + + pub(crate) fn page_count(&self) -> usize { PAGE_COUNT } + + pub(crate) fn state(&self) -> G18ComputeState<'a> { + G18ComputeState::parse(&self.bytes[HEADER_BYTES..HEADER_BYTES + G18_COMPUTE_STATE_BYTES]).unwrap() + } + + pub(crate) fn state_bytes(&self) -> &[u8] { + &self.bytes[HEADER_BYTES..HEADER_BYTES + G18_COMPUTE_STATE_BYTES] + } + + pub(crate) fn page(&self, index: usize) -> Option> { + if index >= PAGE_COUNT { return None; } + let offset = HEADER_BYTES + G18_COMPUTE_STATE_BYTES + index * RECORD_BYTES; + Some(PayloadPage { address: u64_at(self.bytes, offset), + bytes: &self.bytes[offset + 16..offset + RECORD_BYTES] }) + } +} + +/// Byte evidence only: callers additionally require a completed, fault-free +/// live submission before interpreting a correct output as GPU execution. +#[derive(Debug, Default, Copy, Clone)] +pub(crate) struct ComputeResult { + pub(crate) checked: usize, + pub(crate) correct: usize, + pub(crate) untouched: usize, +} + +impl ComputeResult { + pub(crate) fn inspect(bytes: &[u8]) -> Option { + if bytes.len() != PAGE_BYTES { return None; } + let mut result = Self::default(); + for (i, word) in bytes.chunks_exact(4).enumerate() { + let value = u32_at(word, 0); + result.checked += 1; + result.correct += usize::from(value == quarter_bits(4000 + i as i32)); + result.untouched += usize::from(value == POISON); + } + Some(result) + } +} diff --git a/drivers/gpu/drm/asahi/g18/fixture/compute_triangle.rs b/drivers/gpu/drm/asahi/g18/fixture/compute_triangle.rs new file mode 100644 index 0000000000000..0a9457a8cbb9a --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fixture/compute_triangle.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Fixed, private compute-triangle patch and post-completion pixel checks. +//! Failure model: the triangle plan, section 2.7. This format only +//! patches four known sites in an unpublished, verified compute-add VM. + +use crate::g18::fixture::compute_payload::{fnv64, PAGE_BYTES, POISON}; + +pub(crate) const PATCH_COUNT: usize = 4; +pub(crate) const OUTPUT_FNV: u64 = 0x5a2c_fd9c_8627_b035; +const BASE_FNV: u64 = 0x4263_044c_89a6_b244; +const PATCH_FNV: u64 = 0xe40f_1f0c_5b96_cb69; +const HEADER_BYTES: usize = 64; +const PATCH_BYTES: usize = 16574; +// Address, length, original bytes' FNV. Patch provenance is SHA256-pinned in +// the host build receipt; these checks reject damaged or different payloads. +const SITES: [(u64, usize, u64); PATCH_COUNT] = [ + (0x100_0000_8000, PAGE_BYTES, 0x9c1b_da7f_8c87_2325), + (0x100_000a_8038, 10, 0x3def_155f_b602_ffc3), + (0x100_000a_8042, 12, 0xa522_f63b_a76a_995c), + (0x100_0001_8020, 8, 0xa8c7_f832_281a_39c5), +]; + +fn u32_at(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} +fn u64_at(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) +} + +pub(crate) struct TrianglePatch<'a> { bytes: &'a [u8] } +pub(crate) struct PatchSite<'a> { + pub(crate) address: u64, + pub(crate) before_fnv: u64, + pub(crate) bytes: &'a [u8], +} + +impl<'a> TrianglePatch<'a> { + pub(crate) fn parse(bytes: &'a [u8], base_fnv: u64) -> Option { + if bytes.len() != PATCH_BYTES || &bytes[..8] != b"G18TRI01" + || u32_at(bytes, 8) != 1 || u32_at(bytes, 12) != HEADER_BYTES as u32 + || u32_at(bytes, 16) != PATCH_COUNT as u32 + || u32_at(bytes, 20) != 64 || u32_at(bytes, 24) != 64 + || u32_at(bytes, 28) != 1568 || u64_at(bytes, 32) != OUTPUT_FNV + || base_fnv != BASE_FNV || u64_at(bytes, 40) != BASE_FNV + || u64_at(bytes, 48) != fnv64(&bytes[HEADER_BYTES..]) + || u64_at(bytes, 56) != 0 || fnv64(bytes) != PATCH_FNV { + return None; + } + let mut cursor = HEADER_BYTES; + for (address, length, before_fnv) in SITES { + let end = cursor.checked_add(24 + length)?; + if end > bytes.len() || u64_at(bytes, cursor) != address + || u32_at(bytes, cursor + 8) as usize != length + || u32_at(bytes, cursor + 12) != 0 + || u64_at(bytes, cursor + 16) != before_fnv { + return None; + } + cursor = end; + } + if cursor != bytes.len() { return None; } + Some(Self { bytes }) + } + + pub(crate) fn site(&self, index: usize) -> Option> { + if index >= PATCH_COUNT { return None; } + let cursor = HEADER_BYTES + SITES[..index].iter().map(|s| 24 + s.1).sum::(); + let (address, length, before_fnv) = SITES[index]; + Some(PatchSite { address, before_fnv, bytes: &self.bytes[cursor + 24..cursor + 24 + length] }) + } +} + +#[derive(Debug)] +pub(crate) struct TriangleResult { + pub(crate) digest: u64, + pub(crate) foreground: usize, + pub(crate) background: usize, + pub(crate) untouched: usize, + pub(crate) zero: usize, +} + +impl TriangleResult { + /// Only interpret after independent completion; this function neither + /// generates pixels nor establishes GPU execution by itself. + pub(crate) fn inspect(bytes: &[u8]) -> Option { + if bytes.len() != PAGE_BYTES { return None; } + let mut result = Self { digest: fnv64(bytes), foreground: 0, background: 0, + untouched: 0, zero: 0 }; + for word in bytes.chunks_exact(4) { + match u32_at(word, 0) { + POISON => result.untouched += 1, + 0 => result.zero += 1, + 0xff20_2020 => result.background += 1, + _ => result.foreground += 1, + } + } + Some(result) + } + + pub(crate) fn passed(&self) -> bool { + self.digest == OUTPUT_FNV && self.foreground == 1568 && self.background == 2528 + && self.untouched == 0 && self.zero == 0 + } +} diff --git a/drivers/gpu/drm/asahi/g18/fixture/mod.rs b/drivers/gpu/drm/asahi/g18/fixture/mod.rs new file mode 100644 index 0000000000000..ce562d91f0887 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fixture/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fixed G18 diagnostic captures (bring-up stages 8 and 9). + +#[allow(dead_code)] +pub(crate) mod compute_payload; +#[allow(dead_code)] +pub(crate) mod compute_triangle; +#[allow(dead_code)] +pub(crate) mod render_pixels; diff --git a/drivers/gpu/drm/asahi/g18/fixture/render_pixels.rs b/drivers/gpu/drm/asahi/g18/fixture/render_pixels.rs new file mode 100644 index 0000000000000..018ca8f0477a1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fixture/render_pixels.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Streaming SHA-256 of the fixed H1 output. This is not a completion signal. +pub(crate) struct Pixels { h: [u32;8], count: u64, pub poison: u32, pub background: u32 } +impl Pixels { + pub(crate) fn new() -> Self { Self { h: [0x6a09e667,0xbb67ae85,0x3c6ef372,0xa54ff53a, + 0x510e527f,0x9b05688c,0x1f83d9ab,0x5be0cd19], count: 0, poison: 0, background: 0 } } + fn block(&mut self, b: &[u8]) { + const K:[u32;64]=[0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2]; + let mut w=[0u32;64]; + for (i,v) in w[..16].iter_mut().enumerate() { *v=u32::from_be_bytes(b[i*4..i*4+4].try_into().unwrap()); } + for i in 16..64 { let x=w[i-15]; let y=w[i-2]; + w[i]=w[i-16].wrapping_add(x.rotate_right(7)^x.rotate_right(18)^(x>>3)) + .wrapping_add(w[i-7]).wrapping_add(y.rotate_right(17)^y.rotate_right(19)^(y>>10)); } + let [mut a,mut b,mut c,mut d,mut e,mut f,mut g,mut h]=self.h; + for i in 0..64 { + let t=h.wrapping_add(e.rotate_right(6)^e.rotate_right(11)^e.rotate_right(25)) + .wrapping_add((e&f)^(!e&g)).wrapping_add(K[i]).wrapping_add(w[i]); + let u=(a.rotate_right(2)^a.rotate_right(13)^a.rotate_right(22)).wrapping_add((a&b)^(a&c)^(b&c)); + h=g;g=f;f=e;e=d.wrapping_add(t);d=c;c=b;b=a;a=t.wrapping_add(u); + } + for (v,x) in self.h.iter_mut().zip([a,b,c,d,e,f,g,h]) { *v=v.wrapping_add(x); } + } + pub(crate) fn page(&mut self, data: &[u8]) -> bool { + if data.len()!=0x4000 || self.count>=0x40000 { return false; } + for w in data.chunks_exact(4) { self.poison+=u32::from(w==[0xa5;4]); self.background+=u32::from(w==[16,16,16,255]); } + for b in data.chunks_exact(64) { self.block(b); } + self.count+=data.len() as u64; true + } + pub(crate) fn finish(mut self) -> Option<[u32;8]> { + if self.count!=0x40000 { return None; } + let mut padding=[0u8;64];padding[0]=0x80;padding[56..].copy_from_slice(&(self.count*8).to_be_bytes()); + self.block(&padding);Some(self.h) + } +} +pub(crate) const COLOR: [u32;8] = [0xeb547f16,0x6fc6e2b6,0x9d35aca7,0xaa219eec,0xa3405d99,0x47c52e92,0xc18a8642,0xe0298e06]; diff --git a/drivers/gpu/drm/asahi/g18/fw/compute.rs b/drivers/gpu/drm/asahi/g18/fw/compute.rs new file mode 100644 index 0000000000000..2df516579d569 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/compute.rs @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 compute command serialization, derived from the original 26A428 HAL400. +//! +//! This encodes the submission fields and registers of the 0x1020-byte command. +//! It does not create resource owners, publish a queue entry or execute a kick. +//! The caller must own and validate all GPU addresses before publication. Native +//! offsets retained in field names are deliberately not assigned guessed meanings. +//! Register overrides, optional SMAPP preemption objects and random/alternating +//! tuning policies are outside this initial contract. + +#![allow(dead_code)] + +pub(crate) const G18_COMPUTE_COMMAND_BYTES: usize = 0x1020; + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ComputeRegisterInputs { + pub command_gpu: u64, + /// Native descriptor +4d8, +480, +4e8, +4f0, +4f8, +500, in emission order. + pub program: [u64; 6], + /// Native descriptor +578 through +5a0, in emission order. + pub aliases: [u64; 6], + pub d550: u64, + pub d558: u64, + pub d548: u64, + pub d5a8: u64, + pub d5b0: u64, + pub d540: u8, + pub d5d8: u8, + pub d5d9: u8, + pub d62a: u8, + pub d62b: u8, + pub d420: u8, + pub d418: u32, + pub d774: u32, + pub d5c0: u32, + pub d5c4: u32, + pub d5c8: u32, + pub d50c: u32, + pub d510: u32, + pub d514: u32, + pub d630: u32, + pub context_id: u32, + pub pool_slot: u32, + /// 0x10241[63:16]: AGXUMADataRec+0x100, the per-dispatch SMA result + /// (kext 8c4bdb8 kind 2). Not the pool binding's FList generation. + pub sma_units: u64, + pub operands: [u64; 2], + pub support_control: u64, + pub clusters: u32, + pub cluster_width: u32, + pub a6b7: u8, + pub a6b9: u8, + pub a6d0: u8, + pub a758: u32, + pub a10aac: u64, + pub a10ab4: u64, + pub a10aec: u32, + pub a10af0: u8, + pub a10af4: u32, + pub a10b04: u8, + pub a10b08: u32, + pub a10b18: u32, + pub policy54: u8, + pub policy2c: u8, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum G18ComputeRegisterError { + InvalidTopology, + AddressOverflow, + InvalidCompletionState, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18ComputeRegisterSummary { + pub main_gpu: u64, + pub main_count: u16, + pub total_main_count: u16, + pub secondary_gpu: u64, + pub secondary_count: u16, +} + +/// Inputs to the native per-queue completion calculation. `generation` is the +/// selected stamp half's u64 generation; selection happens in the owner layer. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ComputeCompletionState { + pub stamp: u32, + pub index: u32, + pub entries: u32, + pub generation: u64, + pub base: u64, +} + +/// The remaining native submission inputs. Offset names refer to the native +/// host descriptor, not firmware command offsets. Ownership is a caller duty. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ComputeCommandInputs { + pub sequence: u64, + pub completion: G18ComputeCompletionState, + pub stamp_addresses: [u64; 2], + pub channel_ordinal: u32, + pub has_wait: bool, + pub d100: u64, + pub d150: u32, + pub d158: u64, + pub d198: u64, + pub d1b0: u64, + pub d1b8: u64, + pub d488: u64, + pub d490: u64, + pub d498: u64, + pub d4a0: u64, + pub d4a8: u64, + pub d4b0: u32, + pub d4b8: u64, + pub d4c0: u32, + pub d4d0: u32, + pub d4e0: u64, + pub d508: u8, + pub d518: u64, + pub d520: u32, + pub d524: u32, + pub d628: u8, + pub d629: u8, + pub d770: u8, + pub d778: u64, + pub d780: u64, + pub d788: u64, + pub d790: u64, + pub d7b8: u64, + pub d7d4: u8, + pub d8b0: u8, + pub extra_addresses: [u64; 4], + pub extra_policy: u8, +} + +/// Original HAL400 completion expression. Preserve the native u64 division, +/// u32 arithmetic and final 8-bit index independently; a wide modulo gives +/// different bytes around counter wrap. G18's native scratch payload is zero. +pub(crate) fn g18_compute_completion( + p: &G18ComputeCompletionState, +) -> Result { + if p.entries == 0 || p.index >= 128 { + return Err(G18ComputeRegisterError::InvalidCompletionState); + } + let value = (u64::from(p.stamp >> 8) | p.generation << 24).wrapping_sub(1); + let quotient = (value / u64::from(p.entries)) as u32; + let remainder = (value as u32).wrapping_sub(quotient.wrapping_mul(p.entries)); + let slot = (p.base as u32 & 0xff) + (remainder & 0xff); + let carry = slot / p.entries; + let index = (slot - carry * p.entries) & 0xff; + let phase = ((p.base >> 8) as u32) + .wrapping_add(quotient) + .wrapping_add(carry); + Ok(u64::from(index) | u64::from(phase) << 8) +} + +/// Encode the native submission field writer and register program. This does +/// not publish a ring entry or imply that any pointer is backed. The caller +/// initializes the command allocation, rejects errors and retains all owners. +/// Aperture entries and optional SMAPP preemption resources are not supported. +pub(crate) fn encode_g18_compute_command( + out: &mut [u8; G18_COMPUTE_COMMAND_BYTES], + p: &G18ComputeCommandInputs, + registers: &G18ComputeRegisterInputs, +) -> Result { + let completion = g18_compute_completion(&p.completion)?; + put32(out, 0, 3); + put64(out, 4, p.sequence); + put32(out, 0xc, registers.context_id); + for (offset, value) in [ + (0x10, p.d198), + (0xfb5, p.d158), + (0xe80, p.d518), + (0xeb8, p.d488), + (0xec0, p.d490), + (0xec8, p.d498), + (0xed0, p.d4a0), + (0xed8, registers.program[0]), + (0xee0, p.d4e0), + (0xee8, p.d4a8), + (0xef8, p.d4b8), + (0xf2c, 0), + (0xf34, 0), + (0xf40, p.stamp_addresses[0]), + (0xf48, p.stamp_addresses[1]), + (0xf60, completion), + (0xf68, p.d100), + (0xf74, 0), + (0xf7c, p.d1b0), + (0xf84, p.d1b8), + (0xf8c, p.d778), + (0xf94, p.d780), + (0xf9c, p.d788), + (0xfa4, p.d790), + (0xfac, p.d7b8), + ] { + put64(out, offset, value); + } + for (offset, value) in [ + (0xf70, p.d150), + (0xe88, p.d520), + (0xe90, p.d524), + (0xeac, 0), + (0xef0, p.d4b0), + (0xf00, p.d4c0), + (0xf10, p.d4d0), + (0xf1c, u32::from(p.d508 & 1)), + (0xf20, registers.d50c), + (0xf24, registers.d510), + (0xf28, registers.d514), + (0xf50, p.completion.stamp), + (0xf54, p.completion.index), + (0xf58, p.channel_ordinal), + (0xf5c, u32::from(p.has_wait)), + (0xfdd, 0), + (0xfe1, 0), + ] { + put32(out, offset, value); + } + out[0xfb4] = p.d7d4 & 1; + out[0xf3c] = p.d628; + out[0xf3d] = p.d629 & 1; + out[0xf3e] = p.d770 & 1; + out[0xf3f] = p.d8b0 & 1; + let summary = encode_g18_compute_registers(out, registers)?; + out[0x1005] &= 0xf2; + if p.extra_addresses.iter().any(|&value| value != 0) { + out[0x1005] |= 1 | ((p.extra_policy & 3) << 2); + for (index, value) in p.extra_addresses.iter().enumerate() { + put64(out, 0xfe5 + index * 8, *value); + } + } + Ok(summary) +} + +fn put16(out: &mut [u8], offset: usize, value: u16) { + out[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn read32(out: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(out[offset..offset + 4].try_into().unwrap()) +} + +fn append(out: &mut [u8], base: usize, count: &mut u16, number: u32, value: u64) { + let offset = base + usize::from(*count) * 12; + let header = (read32(out, offset) & 0xfffc0006) | (number & 0x3fff9); + put32(out, offset, header); + put64(out, offset + 4, value); + *count += 1; +} + +/// Fill only native generateRegisterList outputs; leave all other command bytes +/// untouched. The full-command builder sets command+0xf68 before this call. +pub(crate) fn encode_g18_compute_registers( + out: &mut [u8; G18_COMPUTE_COMMAND_BYTES], + p: &G18ComputeRegisterInputs, +) -> Result { + if p.clusters == 0 || p.clusters > 16 || p.cluster_width == 0 || p.cluster_width > 16 { + return Err(G18ComputeRegisterError::InvalidTopology); + } + let main_gpu = p + .command_gpu + .checked_add(0x40) + .ok_or(G18ComputeRegisterError::AddressOverflow)?; + let secondary_gpu = p + .command_gpu + .checked_add(0x760) + .ok_or(G18ComputeRegisterError::AddressOverflow)?; + let mode_table = [ + 0x154006201u64, + 0x154004201, + 0x154002201, + 0x154002201, + 0x154002201, + 0x154002201, + 0x154000201, + ]; + let mut control = 0x154024200u64; + if p.a10b04 & 1 != 0 { + control = mode_table + .get(p.a10b08.wrapping_sub(1) as usize) + .copied() + .unwrap_or(0x154002201); + control = (control | (u64::from(p.a10b18) << 17)) & 0x1ffffffee6200; + } + let cs_mode = if p.a10af0 & 1 == 0 { + p.a10aec + } else { + u32::from(p.a10af4 == 0) + }; + control = ((p.a10aac >> 28 | p.a10aac << 9) & 0x1e1fffffe30) + | ((control | u64::from(cs_mode) << 20) & 0xfffe1fffe6001) + | u64::from((!p.d540) & 1) + | ((p.a10ab4 & 0x87ffffff) << 6) + | ((p.a10ab4 >> 32) << 33); + if p.a6d0 == 1 { + control |= 0x30; + } + let mut count = 0u16; + macro_rules! emit { + ($number:expr, $value:expr) => { + append(out, 0x40, &mut count, $number, $value) + }; + } + if p.d5d8 & 1 != 0 { + emit!(0x81e9, 1); + } + if p.d5d9 & 1 != 0 { + emit!(0x81f9, 1); + } + for (number, value) in [0x1a510, 0x1a420, 0x1a4d0, 0x1a4d8, 0x1a4e0, 0x1a4e8] + .into_iter() + .zip(p.program) + { + emit!(number, value); + } + emit!(0x1a440, control); + put64(out, 0xf08, control); + emit!(0xa5e1, p.d550 | p.d558 << 16); + if p.d548 != 0 { + emit!(0xa0f1, p.d548 & !0xfff); + } + let mut mode = if p.d62a & 1 == 0 { + 0x10c08860 + } else { + 0x14c08a60 + }; + if p.d62b & 1 != 0 { + mode |= 0x80; + } + if p.clusters > 2 || p.a6b9 & 1 != 0 || p.d548 != 0 { + mode |= 0x1c; + } + emit!(0x1a458, mode); + emit!(0x101d9, u64::from(p.d774 & 0x7f)); + emit!(0xa111, 0xffff00000000); + emit!(0xa131, 0x7000000000); + emit!(0xa151, 0x7100000000); + for (number, value) in [0x1a079, 0x1a081, 0x1a049, 0x1a051, 0x1a0a9, 0x1a0b1] + .into_iter() + .zip(p.aliases) + { + emit!(number, value); + } + emit!(0x101d1, (p.d5a8 & 0xffffffffc00fffff) | p.d5b0 << 20); + emit!(0x1a0d9, u64::from(p.d5c4 & 0xfffffff8)); + let next = p.d5c8.wrapping_add(1); + emit!(0x1a109, u64::from(if next < 32 { next } else { 0 })); + let mut dispatch = if p.d5d8 & 1 == 0 { + 0xff0004 + } else { + (u64::from(p.d5c0) << 16) | 0x1000004 + }; + if p.a6b7 & 1 == 0 { + dispatch = (dispatch & 0xfffffffb) | (u64::from(p.a758) << 32); + } else { + dispatch &= 0xffffffff; + } + dispatch |= u64::from(p.d5d9 & 1) << 25; + emit!(0x107a1, dispatch); + put64(out, 0x1006, dispatch); + put32(out, 0x100e, 0); + let identity = u64::from(read32(out, 0xf68)) | u64::from(p.d50c) << 32; + emit!(0x1a540, identity); + emit!(0x11209, identity); + emit!(0xa351, identity); + if p.policy54 & 1 != 0 { + if p.policy2c & 8 != 0 { + emit!(0x1a430, 0x3c0f); + } + let dimensions = p.d510.min(0x3fff) | p.d514.min(0x3fff) << 16; + for number in [0x1a4f0, 0x1a4f8, 0x1a500, 0x1a508] { + emit!(number, u64::from(dimensions)); + } + } + put32(out, 0x20, 0); + let mode = if p.d420 & 1 == 0 { + 0 + } else if p.d418 == 1 { + 3 + } else { + 1 + }; + emit!(0xa731, mode); + let context = u64::from((p.d630 & 0x3f) | p.context_id.wrapping_shl(8)); + emit!(0x10201, context); + emit!(0x10428, context); + emit!( + 0x10241, + (p.pool_slot as i32 as i64 as u64) | p.sma_units << 16 + ); + emit!(0xa611, 0); + emit!( + 0x10229, + (p.support_control & 0x7ffffffffc0) | 0x1000000000000000 + ); + if p.d5d8 & 1 != 0 { + emit!(0x81e1, 1); + } + if p.d5d9 & 1 != 0 { + emit!(0x81f1, 1); + } + let extent = u64::from(p.cluster_width + 10 * p.clusters - 10) * 0x1800; + let a = p.operands[0] & 0x7fffffffc00; + let b = p.operands[1] & 0x7fffffffc00; + emit!(0x10099, a | 5); + emit!(0x10091, a + extent); + emit!(0xa5c1, b | 5); + emit!(0xa5c9, b + extent); + let main_count = count; + put16(out, 0x18, main_count); + emit!(0xa731, 0); + emit!(0x1a440, control | 8); + put16(out, 0x1a, count); + put16(out, 0x1c, count); + put64(out, 0x740, main_gpu); + put16(out, 0x748, count); + let mut secondary_count = 0; + for (number, value) in [ + (0x10099, b | 5), + (0x10091, b + extent), + (0xa5c1, a | 5), + (0xa5c9, a + extent), + ] { + append(out, 0x760, &mut secondary_count, number, value); + } + put64(out, 0xe60, secondary_gpu); + put16(out, 0xe68, secondary_count); + Ok(G18ComputeRegisterSummary { + main_gpu, + main_count, + total_main_count: count, + secondary_gpu, + secondary_count, + }) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/compute_memory.rs b/drivers/gpu/drm/asahi/g18/fw/compute_memory.rs new file mode 100644 index 0000000000000..07fbf7999686e --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/compute_memory.rs @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Native G18 / 26A428 compute channel allocation geometry. +//! +//! AGXCLChannel::init 0x8be3934..0x8be3970 computes the extents; +//! 0x8be399c / 0x8be39ac form the allocation size, and +//! 0x8be3ad0..0x8be3ae8 bind its four pointers. These working buffers +//! are distinct from process UAT page tables and scheduler records. + +#![allow(dead_code)] + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18ComputeChannelLayout { + pub(crate) clusters: u32, + pub(crate) width: u32, + pub(crate) operand_bytes: u64, + pub(crate) control_bytes: u64, + pub(crate) tail_bytes: u64, + pub(crate) allocation_bytes: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18ComputeChannelPointers { + /// Native channel +0x178, +0x180, in that order. + pub(crate) operands: [u64; 2], + /// Native channel +0x188 and +0x190. + pub(crate) control: u64, + pub(crate) tail: u64, +} + +impl G18ComputeChannelLayout { + pub(crate) fn new(clusters: u32, width: u32) -> Option { + // The register encoder admits the same bounded topology. Arithmetic + // cannot wrap within these bounds; native's u32 operations agree. + if clusters == 0 || clusters > 16 || width == 0 || width > 16 { + return None; + } + let operand_bytes = + (u64::from(width + 10 * clusters - 10) * 0x1800 + u64::from(clusters) * 64 + 0x3ff) + & !0x3ff; + let control_bytes = u64::from(clusters) * 0x800; + let tail_bytes = 0x400; + let allocation_bytes = (2 * operand_bytes + control_bytes) | tail_bytes; + Some(Self { + clusters, + width, + operand_bytes, + control_bytes, + tail_bytes, + allocation_bytes, + }) + } + + /// The native register fields consume lower GPU VAs. A firmware-high + /// address must not be silently truncated into a different lower address. + pub(crate) fn bind(&self, base: u64) -> Option { + let end = base.checked_add(self.allocation_bytes)?; + if base == 0 || base & 0x3ff != 0 || end > 1u64 << 42 { + return None; + } + Some(G18ComputeChannelPointers { + operands: [base + self.operand_bytes, base], + control: base + 2 * self.operand_bytes, + tail: base + 2 * self.operand_bytes + self.control_bytes, + }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/compute_params.rs b/drivers/gpu/drm/asahi/g18/fw/compute_params.rs new file mode 100644 index 0000000000000..647c496321dbf --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/compute_params.rs @@ -0,0 +1,1619 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8140/G17P compute descriptor construction. +//! +//! The descriptor is the one-page selector-3 object executed by the CL path. +//! The layout combines the m1n1 builder with inspected native producers. The +//! caller supplies queue-owned object addresses and command-owned UAPI values; +//! this module only validates and encodes them. + +pub(crate) const COMPUTE_DESCRIPTOR_SIZE: usize = 0x4000; +/// Registers a kick with no SCS scratch request emits. +pub(crate) const COMPUTE_REGISTER_COUNT: usize = 40; +/// One more slot for the SCS enable register `generateCLRegisterListG16Common` +/// PREPENDS when the descriptor carries a non-zero scratch request +/// (mac17p `0xfffffe0008a5fa90`..`0xfffffe0008a5fac8`). +pub(crate) const COMPUTE_REGISTER_MAX: usize = COMPUTE_REGISTER_COUNT + 1; +pub(crate) const COMPUTE_REGISTER_CAPACITY: usize = 128; +pub(crate) const COMPUTE_REGISTER_SIZE: usize = 0x0c; +pub(crate) const COMPUTE_QUEUE_GRAPH_SIZE: usize = 0x8000; +pub(crate) const COMPUTE_QUEUE_CONTEXT_SIZE: usize = 0x4000; +pub(crate) const COMPUTE_QUEUE_CONTEXT_EXTENT: usize = 8 * COMPUTE_QUEUE_CONTEXT_SIZE; +pub(crate) const COMPUTE_OPTIONAL_SIZE: usize = 0xc0; +pub(crate) const COMPUTE_EVENT_SIZE: usize = 0x400; +pub(crate) const COMPUTE_CHANNEL_WRITE_INDEX: u32 = 3; +pub(crate) const COMPUTE_QUEUE_POINTERS: usize = 0x100; +const COMPUTE_QUEUE_CONTEXT_ITEM: usize = 0x200; +/// Same offset, exported so a repeat can refresh the item the first bind wrote. +pub(crate) const COMPUTE_QUEUE_CONTEXT_ITEM_OFFSET: usize = COMPUTE_QUEUE_CONTEXT_ITEM; +pub(crate) const COMPUTE_ITEM_RING: usize = 0x4000; +pub(crate) const COMPUTE_ITEM_RING_SIZE: usize = 0x500 * 8; +pub(crate) const COMPUTE_OPTIONAL: usize = 0x6800; +pub(crate) const COMPUTE_EVENT: usize = 0x6c00; +/// Storage for the tag-14 `KsmKickQueueAddKicks` channel command. +/// +/// Native allocates this record from a firmware-owned pool +/// (`AGXFirmware::requestChannelCommandKSMKickQueueAddKicks`, +/// mac17p `0xfffffe0008a2d664`). Linux keeps one retained slot per queue graph, +/// placed after the tag-16 window exactly as tags 15 and 16 are placed after the +/// command-pointer ring. A000 `0xfffffc0000011710` cache-invalidates `0x18` +/// bytes of the record before reading it, so the reserved window is zeroed as a +/// whole even though the host writes only `0x15` bytes. +/// Records one steady-state submission publishes into the item ring: +/// the tag-3 command pointer, tag 15, tag 14, tag 16. +pub(crate) const COMPUTE_REPEAT_RING_RECORD_COUNT: usize = 4; +pub(crate) const COMPUTE_ADD_KICKS: usize = 0x7000; +pub(crate) const COMPUTE_ADD_KICKS_SIZE: usize = 0x40; +const _: () = assert!(COMPUTE_ITEM_RING + COMPUTE_ITEM_RING_SIZE <= COMPUTE_OPTIONAL); +const _: () = assert!(COMPUTE_OPTIONAL + COMPUTE_OPTIONAL_SIZE <= COMPUTE_EVENT); +const _: () = assert!(COMPUTE_EVENT + COMPUTE_EVENT_SIZE <= COMPUTE_ADD_KICKS); +const _: () = assert!(COMPUTE_ADD_KICKS + COMPUTE_ADD_KICKS_SIZE <= COMPUTE_QUEUE_GRAPH_SIZE); +const _: () = assert!(COMPUTE_REGISTER_MAX <= COMPUTE_REGISTER_CAPACITY); + +const COMPUTE_SELECTOR: u32 = 3; +const COMPUTE_REGISTER_START: usize = 0x40; +const COMPUTE_SECONDARY_REGISTER_START: usize = 0x760; +const COMPUTE_PRIMARY_COUNT: usize = 0x748; +const COMPUTE_ENCODER_PARAMS: usize = 0xf14; +const COMPUTE_SAMPLER_ARRAY: usize = COMPUTE_ENCODER_PARAMS + 0x18; +const COMPUTE_SAMPLER_COUNT: usize = COMPUTE_ENCODER_PARAMS + 0x20; +const COMPUTE_SAMPLER_MAX: usize = COMPUTE_ENCODER_PARAMS + 0x24; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ComputeBuildError { + BufferTooSmall, + AddressOverflow, + CdmTerminatorBeforeStream, + SubmissionIndexZero, + QueueSubmissionZero, + InvalidSksmEntryAliases, + SamplerCountOutOfRange, + SamplerPair, + SamplerAlignment, + ScsRequestOutOfRange, + RegisterProgramShape, + HardwareTimestampOutOfRange, +} + +/// Low and high GPU aliases of one retained SKSM entry backing. +/// +/// Native tag-15 ConfigUpdate reads these from the same KSM queue object at +/// `+0x80/+0x88`. Keeping them typed prevents the classic CL2 queue-context +/// allocation from being substituted for either address. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeSksmEntryAliases { + low: u64, + high: u64, + extent: usize, +} + +impl ComputeSksmEntryAliases { + pub(crate) const fn new( + low: u64, + high: u64, + extent: usize, + alignment: u64, + ) -> Result { + if low == 0 + || high == 0 + || low == high + || extent == 0 + || alignment == 0 + || alignment & (alignment - 1) != 0 + || low & (alignment - 1) != 0 + || high & (alignment - 1) != 0 + || low.checked_add(extent as u64).is_none() + || high.checked_add(extent as u64).is_none() + { + return Err(ComputeBuildError::InvalidSksmEntryAliases); + } + Ok(Self { low, high, extent }) + } + + pub(crate) const fn low(self) -> u64 { + self.low + } + + pub(crate) const fn high(self) -> u64 { + self.high + } + + pub(crate) const fn extent(self) -> usize { + self.extent + } + + /// Return corresponding addresses within both aliases of the one backing. + pub(crate) const fn range_at( + self, + offset: usize, + size: usize, + ) -> Result<[u64; 2], ComputeBuildError> { + let end = match offset.checked_add(size) { + Some(end) => end, + None => return Err(ComputeBuildError::InvalidSksmEntryAliases), + }; + if end > self.extent { + return Err(ComputeBuildError::InvalidSksmEntryAliases); + } + Ok([self.low + offset as u64, self.high + offset as u64]) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeRegister { + pub(crate) number: u32, + pub(crate) value: u64, +} + +impl ComputeRegister { + const fn new(number: u32, value: u64) -> Self { + Self { number, value } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeRegisterParameters { + pub(crate) preempt_base: u64, + pub(crate) cdm_base: u64, + pub(crate) usc_exec_base: u64, + pub(crate) helper_binary: u64, + pub(crate) helper_data: u64, + pub(crate) helper_config: u64, + pub(crate) dispatch_identity: u64, + pub(crate) context_id: u32, + pub(crate) work_ordinal: u32, + pub(crate) robustness: u64, + pub(crate) operand_state_base: u64, + /// Native CL prepare stores the allocated FList ID at descriptor+0x798; + /// generateRegisterList emits it at 0x14028. This is not an enable gate. + pub(crate) usc_flist_id: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeDescriptorObjects { + pub(crate) scheduler_record: u64, + pub(crate) descriptor_low_alias: u64, + pub(crate) dispatch_a: u64, + pub(crate) dispatch_b: u64, + pub(crate) status_a: u64, + pub(crate) status_b: u64, + pub(crate) shared_control: u64, + pub(crate) usage_record: u64, +} + +/// Descriptor offset of the completion-stamp value firmware copies to the +/// support-page stamp word (`*(descriptor + 0x0F40)`) after the kick's +/// completion cache maintenance (b1 `AGFAcceleratorProcessTimeStampQueue`, +/// see `G17P_COMPUTE_STAMP_SUPPORT_OFFSET`). +pub(crate) const COMPUTE_COMPLETION_STAMP_VALUE: usize = 0x0f50; +/// Initial stamp for bootstrap records. User publication overwrites this with +/// the physical queue's monotonic kick value, independently of TSQ slot index. +pub(crate) const COMPUTE_COMPLETION_STAMP_CONSTANT: u32 = 0x100; + +/// Native EventMachine increments a stable event index's value by 0x100. +/// B1 coalesces timestamp records at that index before cache maintenance: +/// the newest store must cover every earlier completed kick in that prefix. +pub(crate) const fn completion_stamp_value(kick_timestamp: u64) -> u32 { + (kick_timestamp as u32).wrapping_shl(8) +} + +/// Modular prefix comparison, valid while outstanding kicks span less than +/// half the 24-bit stamp sequence (the queue's admission bound is much lower). +/// Zero is a valid wrapped store. Counter/ring identity must also pass. +pub(crate) const fn completion_stamp_covers(stamp_word: u32, kick_timestamp: u64) -> bool { + let delta = stamp_word.wrapping_sub(completion_stamp_value(kick_timestamp)); + delta & 0xff == 0 && delta < 0x8000_0000 +} + +/// Bind the native allocation epoch after building any user compute record. +/// Zero is a valid wrapped epoch; never replace it with a default value. +pub(crate) fn set_context_generation(raw: &mut [u8], generation: u8) -> Result<(), ComputeBuildError> { + if raw.len() <= 0x0fd3 { + return Err(ComputeBuildError::BufferTooSmall); + } + put_u8(raw, 0x0fd3, generation); + Ok(()) +} + +/// Bind the physical queue's exact publication prefix before exposing a record. +pub(crate) fn set_completion_stamp_value(raw: &mut [u8], kick_timestamp: u64) -> Result<(), ComputeBuildError> { + if raw.len() < COMPUTE_COMPLETION_STAMP_VALUE + 4 { + return Err(ComputeBuildError::BufferTooSmall); + } + put_u32(raw, COMPUTE_COMPLETION_STAMP_VALUE, completion_stamp_value(kick_timestamp)); + Ok(()) +} + +/// CL's native completion scratch payload is zero; its low forty bits are +/// the same converted hardware timestamp used by the KSM entry. This field +/// is a u64 at +0xf60, not a u32 ordinal or a QID-packed dependency token. +pub(crate) fn set_hardware_timestamp(raw: &mut [u8], timestamp: u64) -> Result<(), ComputeBuildError> { + if raw.len() < 0x0f68 { return Err(ComputeBuildError::BufferTooSmall); } + if timestamp >> 40 != 0 { return Err(ComputeBuildError::HardwareTimestampOutOfRange); } + put_u64(raw, 0x0f60, timestamp); + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeDescriptorMetadata { + pub(crate) submit_sequence: u64, + pub(crate) context_id: u32, + /// Event stamp index (+0xf54), distinct from the TSQ pointer slot. + pub(crate) grid_index: u32, + pub(crate) work_ordinal: u32, + pub(crate) queue_submission: u32, + /// Historical name: +0xf58 selects a pointer slot within the TSQ node. + /// It is NOT an unbounded submission ordinal. User queues use slot zero. + pub(crate) queue_ordinal: u32, + /// Bootstrap seed for the low word at +0xf60; user publication binds the + /// complete forty-bit hardware timestamp with set_hardware_timestamp. + pub(crate) submission_index: u32, + pub(crate) support_control: u32, + pub(crate) support_flags: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeDescriptorCommand { + pub(crate) cdm_terminator: u64, + pub(crate) sampler_array: u64, + pub(crate) sampler_count: u32, + pub(crate) user_timestamp_start: u64, + pub(crate) user_timestamp_end: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeQueueGraphAddresses { + pub(crate) queue_record: u64, + pub(crate) pointers: u64, + pub(crate) job_list: u64, + pub(crate) item_ring: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeChannelGroupAddresses { + pub(crate) queue: u64, + pub(crate) optional: u64, + pub(crate) event: u64, + pub(crate) write_index: u32, +} + +fn add(address: u64, offset: u64) -> Result { + address + .checked_add(offset) + .ok_or(ComputeBuildError::AddressOverflow) +} + +/// The three register values that carry a compute kick's ray-traversal (SCS) +/// scratch request. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ScsRequestRegisters { + /// 0x101d1: pages in the low half, MiB in the high half. + pub(crate) size_word: u64, + /// 0xd479: the request's "extras" count. + pub(crate) extras: u64, + /// 0x107a1: window and enable, or the disabled constant. + pub(crate) window_word: u64, + /// `AGXCLCommandDescriptor+0x5c8`, which `processComputeSetup` sets to + /// `pages != 0` (mac17p `0xfffffe0008a07ab8`..`0xfffffe0008a07ac0`). + /// `generateCLRegisterListG16Common` branches on it twice: once to prepend + /// register `0x17d9 = 1` (`0xfffffe0008a5fa90`) and once to choose the + /// 0x107a0 value (`0xfffffe0008a60280`). The three request registers alone + /// do NOT tell the two arms apart, so carry the flag rather than inferring + /// it from a value. + pub(crate) enabled: bool, +} + +/// The MiB half of the SCS request, derived from the page count exactly as +/// AGXMetalG17P derives it in `ComputeContext::endComputePass` +/// (0x216ac1f5c..0x216ac1f6c: `pages << 2`, `+ 0x3ff`, `>> 10`). +fn scs_request_mib(pages: u32) -> u32 { + ((pages as u64 + 0xff) >> 8) as u32 +} + +/// Derive the per-core SCS request from Apple's packed ScsLayoutGen1 word. +/// +/// AGXMetalG17P's `getSectionEnd(ScsSection::End)` has four fixed sections +/// totaling 0x58000 bytes, followed by per-VRID ray-stack, driver-payload and +/// user-payload storage. The virtual-ray count is the packed clique byte times +/// 32. `endComputePass` rounds that total to 4 KiB and adds one guard page. +/// +/// A layout with neither user nor driver payload is the native "no traversal +/// scratch" gate even if other fields are populated. Arithmetic is checked +/// here rather than reproducing Apple's 32-bit wrapping: a wrapped request +/// would under-allocate kernel-owned backing. +pub(crate) fn scs_pages_for_layout(layout: u64) -> Result { + let depth = layout & 0xffff; + let cliques = (layout >> 24) & 0xff; + let user = (layout >> 32) & 0xffff; + let driver = (layout >> 48) & 0xffff; + + if user == 0 && driver == 0 { + return Ok(0); + } + + let vrids = cliques + .checked_mul(32) + .ok_or(ComputeBuildError::ScsRequestOutOfRange)?; + let per_vrid = depth + .checked_mul(8) + .and_then(|value| value.checked_add(driver)) + .and_then(|value| value.checked_add(user)) + .ok_or(ComputeBuildError::ScsRequestOutOfRange)?; + let bytes = vrids + .checked_mul(per_vrid) + .and_then(|value| value.checked_add(0x58000)) + .and_then(|value| value.checked_add(0x1fff)) + .ok_or(ComputeBuildError::ScsRequestOutOfRange)? + & !0xfff; + let pages = bytes >> 12; + + if pages > 0xffff { + return Err(ComputeBuildError::ScsRequestOutOfRange); + } + + Ok(pages as u32) +} + +/// Encode an SCS scratch request the way the kext's register builder does +/// (generateCLRegisterListG16Common): the page and MiB counts share 0x101d1, +/// the extras count is its own register, and 0x107a1 either carries the window +/// with the enable bit or reads back the disabled constant 0x00ff_0000. +/// +/// `mib` of zero means "derive it the way Metal does", which is what a faithful +/// request wants; pass a non-zero value only to sweep the two halves of 0x101d1 +/// independently. A request of zero pages reproduces the register program this +/// driver has always emitted, so the default path is unchanged. +pub(crate) fn scs_request_registers( + pages: u32, + mib: u32, + extras: u32, + window: u32, +) -> Result { + if pages == 0 { + return Ok(ScsRequestRegisters { + size_word: 0, + extras: 0, + window_word: 0xff_0000, + enabled: false, + }); + } + // Keep the page and MiB halves distinct, and the window below bit 24's + // enable flag. Reject unrepresentable diagnostic inputs before publishing + // any register program instead of overlapping fields or truncating them. + if pages > 0xffff || mib > 0xffff || window > 0xff { + return Err(ComputeBuildError::ScsRequestOutOfRange); + } + let mib = if mib == 0 { scs_request_mib(pages) } else { mib }; + Ok(ScsRequestRegisters { + size_word: (pages as u64) | ((mib as u64) << 16), + extras: extras as u64, + window_word: ((window as u64) << 16) | 0x0100_0000, + enabled: true, + }) +} + +/// One compute kick's primary register program. +/// +/// The length is not fixed: `generateCLRegisterListG16Common` emits one extra +/// leading register when the descriptor carries an SCS scratch request, so the +/// descriptor's own count word at `COMPUTE_PRIMARY_COUNT` has to follow. +#[derive(Debug, Copy, Clone)] +pub(crate) struct ComputeRegisterProgram { + registers: [ComputeRegister; COMPUTE_REGISTER_MAX], + len: usize, +} + +impl ComputeRegisterProgram { + pub(crate) fn as_slice(&self) -> &[ComputeRegister] { + &self.registers[..self.len] + } + + pub(crate) fn len(&self) -> usize { + self.len + } +} + +/// Build the compute register program, taking the SCS request from the module +/// parameters. +pub(crate) fn build_compute_registers( + p: ComputeRegisterParameters, +) -> Result { + let scs = scs_request_registers( + crate::g18::fw::config::COMPUTE_SCS_PAGES, + crate::g18::fw::config::COMPUTE_SCS_MIB, + crate::g18::fw::config::COMPUTE_SCS_EXTRAS, + crate::g18::fw::config::COMPUTE_SCS_WINDOW, + )?; + build_compute_registers_with_scs(p, scs) +} + +/// Build a production compute register program from the per-command SCS ABI. +/// A zero/non-RT layout is byte-identical to the historical register program. +pub(crate) fn build_compute_registers_for_layout( + p: ComputeRegisterParameters, + layout: u64, +) -> Result { + let pages = scs_pages_for_layout(layout)?; + let scs = scs_request_registers(pages, 0, 0, 0)?; + build_compute_registers_with_scs(p, scs) +} + +/// The register program for an explicit SCS request. +/// +/// Split out of `build_compute_registers` so the enabled arm is reachable from +/// a host test without a module parameter. +pub(crate) fn build_compute_registers_with_scs( + p: ComputeRegisterParameters, + scs: ScsRequestRegisters, +) -> Result { + // Native reads CommandBufferState+8 and masks its low six bits here. + // The logical compute ordinal at descriptor+0xf70 is a separate identity. + let context_word = ((p.context_id as u64) << 8) | u64::from(p.work_ordinal & 63); + + let body = [ + ComputeRegister::new(0x1a510, p.preempt_base), + ComputeRegister::new(0x1a420, p.cdm_base), + ComputeRegister::new(0x1a4d0, add(p.preempt_base, 0x1480)?), + ComputeRegister::new(0x1a4d8, add(p.preempt_base, 0x1488)?), + ComputeRegister::new(0x1a4e0, add(p.preempt_base, 0x1490)?), + ComputeRegister::new(0x1a4e8, add(p.preempt_base, 0x1498)?), + ComputeRegister::new(0x10071, p.usc_exec_base), + ComputeRegister::new(0x11841, p.helper_binary), + ComputeRegister::new(0x11849, p.helper_data), + ComputeRegister::new(0x11f81, p.helper_config), + ComputeRegister::new(0x1a440, 0x0000_0001_5402_4201), + ComputeRegister::new(0x1a458, 0x0000_0000_10c0_8860), + ComputeRegister::new(0x101d9, 0x1c), + ComputeRegister::new(0x1a089, 0), + ComputeRegister::new(0x1a091, 0), + ComputeRegister::new(0x1a059, 0), + ComputeRegister::new(0x1a061, 0), + ComputeRegister::new(0x1a0b9, 0), + ComputeRegister::new(0x1a0c1, 0), + ComputeRegister::new(0x101d1, scs.size_word), + ComputeRegister::new(0x0d479, scs.extras), + ComputeRegister::new(0x1a0e9, 8), + ComputeRegister::new(0x107a1, scs.window_word), + // `A[47:32]` is the per-thread private-context ("PTC"/DUPM) maximum. + // `generateCLRegisterListG16Common` builds this value twice and both + // times from the same accelerator constant, so the two emissions are + // always equal; this driver's 0x132 was not: + // + // first emission `0xfffffe0008a6030c`: reads + // `AGXCLCommandDescriptor+0x548` and emits + // `CL[0x548] ? 0x0040_0020 | CL[0x548] << 32 + // : 0x0000_0060_0040_0020` (`0xfffffe0008a6032c`); + // second emission `0xfffffe0008a60b4c`: reads the accelerator fields + // directly -- `accel[0x10790] ? (u32)accel[0x10790] + // : accel[0x10778]` -- and applies the + // identical expression (`0xfffffe0008a60b7c`). + // + // `CL+0x548` is seeded from `AGXUMADataRec+0x88` + // (`AGXCLCommandDescriptor::prepare`, `0xfffffe00089fb85c`), which + // `AGXUSCPrivMemPoolUMA::allocatePagesEnd` fills with that same + // `accel[0x10790] ?: accel[0x10778]` (`0xfffffe0008aa7688`.. + // `0xfffffe0008aa76a0`). + // + // For G17P that constant is 0x60: `AGXAcceleratorG16::configureDevice` + // stores `accel[0x10778] = 0x60` unconditionally + // (`0xfffffe0008a55688`: `mov w8, #0x60`; `0xfffffe0008a5568c`: + // `str w8, [x21, #0x1b8]`, x21 = accel+0x105c0), and + // `AGXAcceleratorG17P::configureDevice` calls it as its first + // instruction (`0xfffffe0008a64df8`). Nothing in com.apple.AGXG17P + // writes `accel[0x10790]`, so the `?:` selects `accel[0x10778]`, and + // Apple's own zero-source fallback constant in the same expression is + // literally `0x60` -- the two agree. The render lists already emit + // `0x0000_0060_0040_0020` for their analogues 0x0a5a1 and 0x0a5a9. + ComputeRegister::new(0x0a599, 0x0000_0060_0040_0020), + ComputeRegister::new(0x0d411, 0x0000_0002_0000_0001), + ComputeRegister::new(0x1a540, p.dispatch_identity), + ComputeRegister::new(0x014a9, p.dispatch_identity), + ComputeRegister::new(0x0a351, p.dispatch_identity), + ComputeRegister::new(0x10201, context_word), + ComputeRegister::new(0x10428, context_word), + ComputeRegister::new(0x14028, u64::from(p.usc_flist_id)), + ComputeRegister::new(0x14070, p.robustness | 1), + ComputeRegister::new(0x10229, add(p.operand_state_base, 0x12800)?), + ComputeRegister::new(0x140a8, add(p.operand_state_base, 0x13000)?), + ComputeRegister::new(0x10099, add(p.operand_state_base, 0x9405)?), + ComputeRegister::new(0x10091, add(p.operand_state_base, 0x12400)?), + ComputeRegister::new(0x0a5c1, add(p.operand_state_base, 0x0005)?), + ComputeRegister::new(0x0a5c9, add(p.operand_state_base, 0x9000)?), + ComputeRegister::new(0x1a440, 0x0000_0001_5402_4209), + ComputeRegister::new(0x0a599, 0x0000_0060_0040_0020), + ]; + + // `generateCLRegisterListG16Common` writes its first register list entry + // at buffer offset 0 only when `AGXCLCommandDescriptor+0x5c8` is clear. + // When it is set the first entry is `0x17d9 = 1` and everything else moves + // up by one 12-byte entry (mac17p `0xfffffe0008a5fa90` takes the branch, + // `0xfffffe0008a5faa0` ORs the selector `0x17d9` into the entry word, + // `0xfffffe0008a5fab0` stores the value 1, `0xfffffe0008a5fac4` sets the + // cursor to 12 so `0x1a510` lands at entry 1; the clear arm at + // `0xfffffe0008a5facc` sets the cursor to 0 instead). The TA and 3D list + // builders do the same thing with the same selector and value, each gated + // on its own descriptor flag (`0xfffffe0008a585b8` on TA `+0x598`, + // `0xfffffe0008a5a84c` on 3D `+0xb70`). + // + // This driver had never emitted `0x17d9`, which is the only difference + // left between its `g17p_compute_scs_pages=1` program and the kext's: that + // run already sent the byte-exact 0x101d1 / 0xd479 / 0x107a1 triple and its + // shader still never launched. That `0x17d9` is what the launch was + // waiting on is an INFERENCE, not a measured result. + // + // What `0x17d9` means to the hardware is not established either. None of + // the selectors this program emits appears as an immediate anywhere in the + // B100 firmware image (checked 0x17d8/0x17d9, 0x101d0, 0xd478, 0x107a0, + // 0xa598, 0x1a510, 0x14070 and 0x1748), so the firmware replays the list + // rather than interpreting it and the meaning is the GPU's, not the + // firmware's. + let mut registers = [ComputeRegister::new(0, 0); COMPUTE_REGISTER_MAX]; + let mut len = 0; + if scs.enabled { + registers[len] = ComputeRegister::new(0x17d9, 1); + len += 1; + } + for register in body { + registers[len] = register; + len += 1; + } + Ok(ComputeRegisterProgram { registers, len }) +} + +fn put_u8(raw: &mut [u8], offset: usize, value: u8) { + raw[offset] = value; +} + +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Build the retained queue record named by every compute CL entry. +pub(crate) fn build_compute_queue_graph( + base: u64, + channel_control: u64, + raw: &mut [u8], +) -> Result { + if raw.len() < COMPUTE_QUEUE_GRAPH_SIZE { + return Err(ComputeBuildError::BufferTooSmall); + } + let addresses = ComputeQueueGraphAddresses { + queue_record: base, + pointers: add(base, 0x100)?, + job_list: add(base, 0x200)?, + item_ring: add(base, 0x4000)?, + }; + let raw = &mut raw[..COMPUTE_QUEUE_GRAPH_SIZE]; + raw.fill(0); + put_u64(raw, 0x00, addresses.pointers); + put_u64(raw, 0x08, addresses.item_ring); + put_u64(raw, 0x10, addresses.job_list); + put_u32(raw, 0x24, u32::MAX); + put_u32(raw, 0x28, 2); + put_u32(raw, 0x2c, 2); + raw[0x36..0x38].fill(0xff); + put_u32(raw, 0x40, 2); + put_u32(raw, 0x44, u32::MAX); + // Bootstrap/unowned graph. A logical bind supplies the creator's TGID. + put_u32(raw, 0x48, 0); + put_u64(raw, 0x9c, channel_control); + put_u32(raw, 0x100 + 0x50, u32::MAX); + put_u32(raw, 0x100 + 0x60, compute_item_ring_entries()); + put_u64(raw, 0x200 + 0x08, addresses.job_list); + Ok(addresses) +} + +/// Usable entries in the queue's item ring, as programmed into the graph's +/// pointer block at `+0x160` and read back by both the host publisher and the +/// firmware. +/// +/// Normally the full `0x500`. `g17p_ring_limit` shrinks it so the ring WRAP can +/// be reached in a handful of submissions instead of 320: at four records per +/// submission, a 32-entry ring wraps on the eighth. Wrap behaviour is identical +/// at any size, so this turns a 400-iteration run -- which also overflows the +/// kernel log and destroys its own evidence -- into a 20-iteration one. +pub(crate) fn compute_item_ring_entries() -> u32 { + const FULL: u32 = 0x500; + let requested = crate::g18::fw::config::RING_LIMIT; + if requested == 0 { + return FULL; + } + // Must stay a multiple of the per-submission record count so a batch never + // straddles the wrap differently than it would at full size, and must leave + // room for at least two submissions plus the always-empty slot. + let clamped = requested.clamp( + 2 * COMPUTE_REPEAT_RING_RECORD_COUNT as u32 + 4, + FULL, + ); + clamped - (clamped % COMPUTE_REPEAT_RING_RECORD_COUNT as u32) +} + +/// Identity of the configuration last published on one physical CL QID. +/// +/// Logical scheduler pages are unique and retained until processor stop. +/// Their address therefore also distinguishes two root leases that reuse the +/// same numeric context ID, including queues sharing one VM. The root lease +/// is immutable while current; submit validates that before using this key. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeConfigIdentity { + pub(crate) channel_control: u64, + pub(crate) qos_id: u8, + pub(crate) context_id: Option, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeConfigUpdates { + pub(crate) qos: bool, + pub(crate) context: bool, +} + +impl ComputeConfigIdentity { + pub(crate) fn updates_from(self, published: Option) -> ComputeConfigUpdates { + match published { + None => ComputeConfigUpdates { qos: true, context: true }, + Some(previous) => ComputeConfigUpdates { + // QID membership is scoped to its logical scheduler page. + // Policy is currently immutable for this logical lease. + qos: self.channel_control != previous.channel_control + || self.qos_id != previous.qos_id, + context: self.channel_control != previous.channel_control + || self.context_id != previous.context_id, + }, + } + } +} + +/// Encode the tag-15 `KSMKickQueueConfigUpdate` record inside a retained +/// queue graph. +/// +/// A retained submission may carry this record without reinstalling the QID +/// or changing its QoS/context. Native computes +0x52 from QoS ID, QID +/// membership and policy changes, and +0x62 from the channel's cached GART +/// allocation generation. See audit_g17_native_scheduler_owner.py. +pub(crate) fn encode_compute_config_update( + graph: &mut [u8], + sksm_entries: ComputeSksmEntryAliases, + shared_control: u64, + channel_control: u64, + queue_id: u8, + hardware_buffer_id: u8, + install_queue: bool, + updates: ComputeConfigUpdates, + context_id: Option, +) -> Result<(), ComputeBuildError> { + // AGXChannel::getPid supplies a full u32 at tag15 +0x5a. The queue's + // logical-owner bind populates the same PID at record +0x48; reading that + // retained identity avoids capturing a workqueue PID during submission. + let owner_pid = u32::from_le_bytes( + graph.get(0x48..0x4c).ok_or(ComputeBuildError::BufferTooSmall)? + .try_into().map_err(|_| ComputeBuildError::BufferTooSmall)?, + ); + let optional_raw = graph + .get_mut(COMPUTE_OPTIONAL..COMPUTE_OPTIONAL + COMPUTE_OPTIONAL_SIZE) + .ok_or(ComputeBuildError::BufferTooSmall)?; + encode_compute_config_update_record( + optional_raw, + owner_pid, + sksm_entries, + shared_control, + channel_control, + queue_id, + hardware_buffer_id, + install_queue, + updates, + context_id, + ) +} + +/// Encode one tag-15 `KSMKickQueueConfigUpdate` record into `optional_raw` +/// (`COMPUTE_OPTIONAL_SIZE` bytes), wherever that record lives. +/// +/// A queue's FIRST bind encodes it in the queue graph's optional window; every +/// later publication encodes it in its own per-publication record slot. The +/// firmware reads a record through the item-ring pointer that names it, which +/// can be long after the host has published further kicks, so a record must +/// never be rewritten while a ring entry still names it. Measured on Neo +/// 2026-09-26 19:41: a repeat encoded into the graph window while the first +/// bind's ring entry still pointed there rewrote +0x1a (queue-install) to 0; +/// the firmware then consumed the first bind with the install branch skipped, +/// left the per-queue slot unpopulated, and its tag-16 handler faulted on the +/// NULL slot (pc=0x1190c far=0x140 esr=0x96000004), the same fault shape the +/// +0x1a note below records for sending 0 directly. +pub(crate) fn encode_compute_config_update_record( + optional_raw: &mut [u8], + owner_pid: u32, + sksm_entries: ComputeSksmEntryAliases, + shared_control: u64, + channel_control: u64, + queue_id: u8, + hardware_buffer_id: u8, + install_queue: bool, + updates: ComputeConfigUpdates, + context_id: Option, +) -> Result<(), ComputeBuildError> { + let optional_raw = optional_raw + .get_mut(..COMPUTE_OPTIONAL_SIZE) + .ok_or(ComputeBuildError::BufferTooSmall)?; + // Record +0x32 is the hardware context this queue's work is declared to run + // in. It has always been the literal 2, from when every compute client ran + // in the fixed pair `T8140_COMPUTE_CTXS = [2, 3]`. + // + // Those two contexts are a SINGLE GLOBAL alias pair: + // `install_t8140_compute_context_alias` publishes the current client's VM + // root into both and records one `t8140_alias_owner`. With one compute + // client that is correct. With a second, the alias is re-pointed and the + // loser's kicks -- still declaring 2 -- no longer name their own root, + // while their real root lives in the app context the execution context + // allocated (5..62). The kick already declares that real context at + // `sgx+0x10428`; only this record still says 2. + // + // Probe-gated and default-off: 0 sends the historical constant, byte + // identical to the previous driver; 1 sends the client's real context when + // one is known. See `g17p_compute_config_context`. + let declared_context = match context_id { + Some(context) if crate::g18::fw::config::COMPUTE_CONFIG_CONTEXT != 0 => { + context + } + _ => 2u16, + }; + let flushid_fix = crate::g18::fw::config::FLUSHID_FIX != 0; + let (flushid_object, flushid_slot, flushid_index) = if flushid_fix { + (0u64, 0xffffu16, 0xffffu16) + } else { + (shared_control, 0x3fu16, crate::g18::fw::config::compute_flist_id(queue_id)) + }; + optional_raw.fill(0); + put_u32(optional_raw, 0x00, 0x0f); + put_u64(optional_raw, 0x08, sksm_entries.low()); + put_u64(optional_raw, 0x10, sksm_entries.high()); + for (offset, value) in [ + (0x18, u16::from(queue_id)), + // +0x1a is the QUEUE-INSTALL flag, and it must be non-zero on a first + // bind. Firmware RE (2026-08-31): the only writer of the per-queue slot + // `0x127628 + qid*0x28` is at 0x119b8, and that store -- together with + // slot[+0x00]=1, the per-data-master active bitmap, the sgx+0x21008 + // queue-base write and the 0x40 KTrace receipt -- is reachable only when + // this field is non-zero. Sending 0 left slot[+0x08] NULL, and the tag-16 + // handler then faulted dereferencing it (firmware exception pc=0x1190c + // far=0x340 esr=0x96000004, a level-0 translation fault), which is why + // pause bit 7 taken at 0x118c0 was never cleared at 0x11948. + // + // The two captured macOS records show 0 here because both were taken on + // already-installed queues, not on a first bind. Absence of the 0x40 + // KTrace code is the receipt that the install branch was not taken. + // Queue-install flag: 1 only on a queue's FIRST bind. Native clears it + // afterwards -- leaving it set forces a disable/reprogram/enable cycle + // on the KSMFE queue for every submission, which is why a second submit + // published fine and then never completed. + (0x1a, if install_queue { 1 } else { 0 }), + (0x1e, 2), + (0x22, 2), + (0x32, declared_context), + (0x3e, flushid_slot), + (0x46, flushid_index), + (0x52, u16::from(updates.qos)), + // AGXCLChannelSKSM::submitBuffer copies AGXQOSQueue +0x10 here. + // Native retained CL queues receive distinct HardwareBufferIDs in + // allocation order (1, 2, 3 in the captured QID 4/5/7 sequence). + // This field is therefore queue identity, not the literal 1 from the + // first output-positive queue. + (0x56, u16::from(hardware_buffer_id)), + (0x5e, 2), + (0x62, u16::from(updates.context)), + (0x66, 1), + ] { + put_u16(optional_raw, offset, value); + } + put_u64(optional_raw, 0x36, flushid_object); + put_u64(optional_raw, 0x4a, channel_control); + put_u32(optional_raw, 0x5a, owner_pid); + optional_raw[0x76..0x86].fill(0xff); + Ok(()) +} + +/// Encode one queue-context item. +/// +/// The array is a sibling of the SKSM entry array, not a single record: +/// `COMPUTE_QUEUE_CONTEXT_EXTENT` is `8 * 0x4000 = 0x20000`, the item stride is +/// `0x200`, so it holds `0x100` items -- exactly the queue geometry's +/// `stamp_count`, and exactly the size and stride of the entry backing +/// (`0x100 * 0x200 = 0x20000`). The first bind writes item 1 and consumes stamp +/// index 1, which is consistent with the item being indexed by stamp. +/// +/// That reading is INFERRED from the geometry, not proven, which is why the +/// repeat path selects between "rewrite item 1" and "write item[stamp]" by +/// parameter instead of assuming. +pub(crate) fn encode_compute_queue_context_item( + queue_context: &mut [u8], + item_index: usize, + descriptor: u64, + queue_base: u64, + queue_id: u8, +) -> Result<(), ComputeBuildError> { + let offset = item_index + .checked_mul(COMPUTE_QUEUE_CONTEXT_ITEM) + .ok_or(ComputeBuildError::AddressOverflow)?; + let end = offset + .checked_add(COMPUTE_QUEUE_CONTEXT_ITEM) + .ok_or(ComputeBuildError::AddressOverflow)?; + if end > queue_context.len() { + return Err(ComputeBuildError::BufferTooSmall); + } + let context = &mut queue_context[offset..end]; + context.fill(0); + put_u64(context, 0x00, 0x1000_1000_0000_0004); + put_u64(context, 0x10, descriptor); + put_u64(context, 0x18, queue_base); + put_u64(context, 0x20, 0xffff_0801_0000_0001); + put_u64(context, 0x28, u64::from(queue_id) << 40); + put_u64(context, 0x130, 0); + put_u64(context, 0x138, 4); + put_u64(context, 0x150, 0x0001_1003_8001_a002); + put_u64(context, 0x158, 0x0000_2003_8001_a03b); + put_u64(context, 0x178, 0x003f_ffff_ffff_ffff); + Ok(()) +} + +/// Stage the first hardware-proven direct-compute group in a retained queue. +/// +/// This is the CL_2 channel graph used by the output-positive m1n1 add3 test: +/// selector-3 descriptor, selector-0x0f optional record, selector-0x0e event, +/// followed by queue write index 3. The caller publishes the outer channel +/// slot and its producer only after this function returns. +pub(crate) fn build_initial_compute_channel_group( + base: u64, + descriptor: u64, + sksm_entries: ComputeSksmEntryAliases, + shared_control: u64, + channel_control: u64, + queue_id: u8, + hardware_buffer_id: u8, + graph: &mut [u8], + queue_context: &mut [u8], + install_queue: bool, + context_id: Option, +) -> Result { + if graph.len() < COMPUTE_QUEUE_GRAPH_SIZE + || queue_context.len() < COMPUTE_QUEUE_CONTEXT_SIZE + { + return Err(ComputeBuildError::BufferTooSmall); + } + + let optional = add(base, COMPUTE_OPTIONAL as u64)?; + let event = add(base, COMPUTE_EVENT as u64)?; + + // Record +0x48 is the process PID, not a UUID. Preserve the logical + // owner's value installed when this physical graph was bound. + for (index, address) in [descriptor, optional, event].into_iter().enumerate() { + put_u64(graph, COMPUTE_ITEM_RING + index * 8, address); + } + put_u32( + graph, + COMPUTE_QUEUE_POINTERS + 0x40, + COMPUTE_CHANNEL_WRITE_INDEX, + ); + + // B1 registers an exact pointer/cookie at table[record+0x46]; later + // descriptor processing selects that same table with FList+0x08. Both + // IDs must identify our canonical compute pool, not render's ID zero. + // The cookie is stable for this retained session incarnation. All queues + // register the same pointer/cookie; their scheduler identity is separate. + encode_compute_config_update( + graph, + sksm_entries, + shared_control, + channel_control, + queue_id, + hardware_buffer_id, + install_queue, + ComputeConfigUpdates { qos: true, context: true }, + context_id, + )?; + + let event_raw = &mut graph[COMPUTE_EVENT..COMPUTE_EVENT + COMPUTE_EVENT_SIZE]; + event_raw.fill(0); + put_u32(event_raw, 0x00, 0x0e); + put_u32(event_raw, 0x04, (1u32 << 16) | u32::from(queue_id)); + put_u32(event_raw, 0x08, 0x0102); + put_u32(event_raw, 0x10, 0x0200); + + queue_context[..COMPUTE_QUEUE_CONTEXT_SIZE].fill(0); + // Item index 1, which is also the stamp index the first bind consumes + // (the geometry seeds `current_timestamp` at 1). See + // `encode_compute_queue_context_item` for why that coincidence matters. + encode_compute_queue_context_item(queue_context, 1, descriptor, base, queue_id)?; + + Ok(ComputeChannelGroupAddresses { + queue: base, + optional, + event, + write_index: COMPUTE_CHANNEL_WRITE_INDEX, + }) +} + +fn encode_registers(raw: &mut [u8], offset: usize, registers: &[ComputeRegister]) { + for (index, register) in registers.iter().enumerate() { + let entry = offset + index * COMPUTE_REGISTER_SIZE; + put_u32(raw, entry, register.number); + put_u64(raw, entry + 4, register.value); + } +} + +fn register_value(registers: &[ComputeRegister], number: u32, occurrence: usize) -> u64 { + registers + .iter() + .filter(|register| register.number == number) + .nth(occurrence) + .expect("fixed G17P register program lost a required register") + .value +} + +fn register_count_through( + registers: &[ComputeRegister], + number: u32, + occurrence: usize, +) -> Result { + let index = registers + .iter() + .enumerate() + .filter(|(_, register)| register.number == number) + .nth(occurrence) + .map(|(index, _)| index) + .ok_or(ComputeBuildError::RegisterProgramShape)?; + u16::try_from(index + 1).map_err(|_| ComputeBuildError::RegisterProgramShape) +} + +/// Resume checkpoints consumed by B1 after a nonterminal CL completion. +/// +/// The no-extra G16 common producer records the current register count after +/// the first 0x0a5c9, the final 0x1a440 (twice), and the final 0x0a599. B1 +/// reads descriptor +0x1a/+0x1c on status 2 and copies the selected low seven +/// bits into the selected SKSM kick-ring entry at +0x150. Derive the counts from the +/// actual program so an SCS 0x17d9 prefix, or a later list edit, cannot leave +/// stale resume indices in the descriptor. +pub(crate) fn compute_resume_checkpoints( + registers: &[ComputeRegister], +) -> Result<[u16; 4], ComputeBuildError> { + let first_tail = register_count_through(registers, 0x0a5c9, 0)?; + let final_control = register_count_through(registers, 0x1a440, 1)?; + let final_tail = register_count_through(registers, 0x0a599, 1)?; + Ok([first_tail, final_control, final_control, final_tail]) +} + +/// Encode a selector-3 descriptor without dropping sampler, timestamp, or CDM +/// command state. +pub(crate) fn build_compute_descriptor( + program: &ComputeRegisterProgram, + objects: ComputeDescriptorObjects, + metadata: ComputeDescriptorMetadata, + command: ComputeDescriptorCommand, + raw: &mut [u8], +) -> Result<[u16; 4], ComputeBuildError> { + let registers = program.as_slice(); + if raw.len() < COMPUTE_DESCRIPTOR_SIZE { + return Err(ComputeBuildError::BufferTooSmall); + } + if command.cdm_terminator < register_value(registers, 0x1a420, 0) { + return Err(ComputeBuildError::CdmTerminatorBeforeStream); + } + if metadata.submission_index == 0 { + return Err(ComputeBuildError::SubmissionIndexZero); + } + if metadata.queue_submission == 0 { + return Err(ComputeBuildError::QueueSubmissionZero); + } + if command.sampler_count == u32::MAX { + return Err(ComputeBuildError::SamplerCountOutOfRange); + } + if (command.sampler_array == 0) != (command.sampler_count == 0) { + return Err(ComputeBuildError::SamplerPair); + } + if command.sampler_array & 7 != 0 { + return Err(ComputeBuildError::SamplerAlignment); + } + + let raw = &mut raw[..COMPUTE_DESCRIPTOR_SIZE]; + raw.fill(0); + put_u32(raw, 0x00, COMPUTE_SELECTOR); + put_u64(raw, 0x04, metadata.submit_sequence); + put_u32(raw, 0x0c, metadata.context_id); + put_u64(raw, 0x10, objects.scheduler_record); + let resume_checkpoints = compute_resume_checkpoints(registers)?; + for (index, value) in resume_checkpoints.into_iter().enumerate() { + put_u16(raw, 0x18 + index * 2, value); + } + + encode_registers(raw, COMPUTE_REGISTER_START, registers); + put_u64( + raw, + 0x740, + add(objects.descriptor_low_alias, COMPUTE_REGISTER_START as u64)?, + ); + // The count word follows the program, which grows by one entry when the + // kick carries an SCS request -- exactly as the kext's own + // `[sp,#0x58]`/`[sp,#0x5a]` byte-size and entry-count pair does + // (`0xfffffe0008a5fab4` seeds them with 12 and 1 on the request arm). + put_u32( + raw, + COMPUTE_PRIMARY_COUNT, + ((program.len() * COMPUTE_REGISTER_SIZE) as u32) << 16 | program.len() as u32, + ); + + let secondary = [ + ComputeRegister::new(0x10099, register_value(registers, 0x0a5c1, 0)), + ComputeRegister::new(0x10091, register_value(registers, 0x0a5c9, 0)), + ComputeRegister::new(0x0a5c1, register_value(registers, 0x10099, 0)), + ComputeRegister::new(0x0a5c9, register_value(registers, 0x10091, 0)), + ]; + encode_registers(raw, COMPUTE_SECONDARY_REGISTER_START, &secondary); + put_u64( + raw, + 0x0e60, + add( + objects.descriptor_low_alias, + COMPUTE_SECONDARY_REGISTER_START as u64, + )?, + ); + put_u32(raw, 0x0e68, 0x0030_0004); + put_u32(raw, 0x0f28, u32::MAX); + + put_u64(raw, 0x0ed8, register_value(registers, 0x1a510, 0)); + put_u64(raw, 0x0ee0, command.cdm_terminator); + put_u64(raw, 0x0f08, register_value(registers, 0x1a440, 0)); + put_u32( + raw, + 0x0f20, + (register_value(registers, 0x1a540, 0) >> 32) as u32, + ); + put_u64(raw, COMPUTE_SAMPLER_ARRAY, command.sampler_array); + put_u32(raw, COMPUTE_SAMPLER_COUNT, command.sampler_count); + put_u32( + raw, + COMPUTE_SAMPLER_MAX, + if command.sampler_count == 0 { + 0 + } else { + command.sampler_count + 1 + }, + ); + put_u64(raw, 0x0f40, objects.dispatch_a); + put_u64(raw, 0x0f48, objects.dispatch_b); + // Bootstrap seed only. Every user publication route binds the actual kick + // timestamp separately; queue_ordinal is a TSQ slot, not a stamp sequence. + put_u32(raw, 0x0f50, COMPUTE_COMPLETION_STAMP_CONSTANT); + put_u32(raw, 0x0f54, metadata.grid_index); + put_u32(raw, 0x0f58, metadata.queue_ordinal); + put_u32(raw, 0x0f60, metadata.submission_index); + put_u64( + raw, + 0x0f68, + register_value(registers, 0x1a540, 0) & 0xffff_ffff, + ); + put_u32(raw, 0x0f70, metadata.work_ordinal); + put_u64(raw, 0x0f7c, objects.status_a); + put_u64(raw, 0x0f84, objects.status_b); + put_u64(raw, 0x0f8c, command.user_timestamp_start); + put_u64(raw, 0x0f94, command.user_timestamp_end); + + put_u16(raw, 0x0fb0, 0x001a); + put_u64(raw, 0x0fb2, objects.shared_control); + put_u32(raw, 0x0fba, metadata.support_control); + put_u32(raw, 0x0fbe, metadata.support_flags); + put_u8(raw, 0x0fc5, 0x9f); + put_u32(raw, 0x0fc8, (metadata.work_ordinal & 3) << 30); + put_u64(raw, 0x0fcb, objects.usage_record); + put_u8(raw, 0x0fd3, 1); + Ok(resume_checkpoints) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn register_work_state_does_not_overwrite_the_context_id() { + for word in [0, 62, 64, 0x1234_5678, u32::MAX - 1] { + let mut p = register_parameters(); + p.context_id = 9; + p.work_ordinal = word; + let program = build_compute_registers(p).unwrap(); + let expected = (9 << 8) | u64::from(word & 63); + assert_eq!(register_value(program.as_slice(), 0x10201, 0), expected); + assert_eq!(register_value(program.as_slice(), 0x10428, 0), expected); + } + } + + #[test] + fn hardware_timestamp_keeps_all_forty_bits_and_zero_payload() { + for timestamp in [0, 1, 0xffff_ffff, 0x1_0000_0000, 0xff_ffff_ffff] { + let mut raw = [0xa5u8; 0x1000]; + set_hardware_timestamp(&mut raw, timestamp).unwrap(); + assert_eq!(u64::from_le_bytes(raw[0xf60..0xf68].try_into().unwrap()), timestamp); + assert!(raw[..0xf60].iter().all(|&v| v == 0xa5)); + assert!(raw[0xf68..].iter().all(|&v| v == 0xa5)); + } + let mut raw = [0xa5u8; 0x1000]; + assert_eq!(set_hardware_timestamp(&mut raw, 1 << 40), Err(ComputeBuildError::HardwareTimestampOutOfRange)); + assert!(raw.iter().all(|&v| v == 0xa5)); + assert_eq!(set_hardware_timestamp(&mut [0; 4], 1), Err(ComputeBuildError::BufferTooSmall)); + } + + #[test] + fn allocation_epoch_changes_only_its_byte_and_accepts_wrap() { + for epoch in [0, 1, 2, 3, 127, 255] { + let mut raw = [0xa5u8; 0x1000]; + set_context_generation(&mut raw, epoch).unwrap(); + assert_eq!(raw[0xfd3], epoch); + assert!(raw[..0xfd3].iter().all(|&b| b == 0xa5)); + assert!(raw[0xfd4..].iter().all(|&b| b == 0xa5)); + } + assert_eq!(set_context_generation(&mut [0; 0xfd3], 1), Err(ComputeBuildError::BufferTooSmall)); + } + + #[test] + fn completion_stamp_covers_only_the_flushed_prefix_including_wrap() { + for kick in [1u64, 0x100, 0x3628, 0x3629, 0xff_ffff, 0x100_0000, 0x123_4567_89ab] { + let stamp = completion_stamp_value(kick); + assert!(completion_stamp_covers(stamp, kick)); + assert!(!completion_stamp_covers(stamp, kick + 1)); + assert!(completion_stamp_covers(completion_stamp_value(kick + 16), kick)); + assert!(!completion_stamp_covers(stamp | 1, kick)); + } + assert!(!completion_stamp_covers(0, 1)); + assert!(completion_stamp_covers(0, 0x100_0000)); + assert!(completion_stamp_covers(0, 0xff_ffff)); + assert!(!completion_stamp_covers(0xffff_ff00, 0x100_0000)); + assert!(!completion_stamp_covers(0x8000_0100, 1)); + let mut raw = [0u8; COMPUTE_COMPLETION_STAMP_VALUE + 4]; + set_completion_stamp_value(&mut raw, 0x3629).unwrap(); + assert_eq!( + u32::from_le_bytes(raw[0x0f50..0x0f54].try_into().unwrap()), + 0x362900, + ); + let mut raw2 = [0u8; COMPUTE_COMPLETION_STAMP_VALUE + 4]; + set_completion_stamp_value(&mut raw2, 1).unwrap(); + assert_ne!(raw2, raw); + assert_eq!(set_completion_stamp_value(&mut [0; 4], 1), Err(ComputeBuildError::BufferTooSmall)); + } + + fn fnv1a64(raw: &[u8]) -> u64 { + let mut value = 0xcbf2_9ce4_8422_2325u64; + for byte in raw { + value ^= *byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + value + } + + fn register_parameters() -> ComputeRegisterParameters { + ComputeRegisterParameters { + preempt_base: 0x100_0404_0000, + cdm_base: 0x100_0410_0000, + usc_exec_base: 0x1_0000_0000, + helper_binary: 0, + helper_data: 0, + helper_config: 0, + dispatch_identity: 0x1234_5678_0000_0001, + context_id: 3, + work_ordinal: 7, + robustness: 0x100_0430_0000, + operand_state_base: 0x70_0022_8000, + usc_flist_id: 2, + } + } + + #[test] + fn register_program_matches_the_hardware_tested_m1n1_builder() { + let program = build_compute_registers(register_parameters()).unwrap(); + let registers = program.as_slice(); + assert_eq!(program.len(), COMPUTE_REGISTER_COUNT); + let mut raw = [0u8; COMPUTE_REGISTER_COUNT * COMPUTE_REGISTER_SIZE]; + encode_registers(&mut raw, 0, registers); + // Re-pinned when 0x0a599's first emission was corrected from 0x132 + // to the 0x60 both of the kext's emissions carry. + assert_eq!(fnv1a64(&raw), 0x2d33_2bc4_ecd2_8bdd); + assert_eq!(registers[0], ComputeRegister::new(0x1a510, 0x100_0404_0000)); + assert_eq!(registers[39], ComputeRegister::new(0x0a599, 0x6000_4000_20)); + } + + #[test] + fn both_0xa599_emissions_carry_the_same_accelerator_constant() { + // `generateCLRegisterListG16Common` reads one accelerator quantity for + // both 0xa598 emissions (`0xfffffe0008a6030c` through + // `AGXCLCommandDescriptor+0x548`, and `0xfffffe0008a60b4c` straight + // from `accel[0x10790] ?: accel[0x10778]`), so they cannot disagree. + // This driver's two writes did; the first carried 0x132. + let program = build_compute_registers(register_parameters()).unwrap(); + let first = register_value(program.as_slice(), 0x0a599, 0); + let second = register_value(program.as_slice(), 0x0a599, 1); + assert_eq!(first, second); + // 0x60 is `AGXAcceleratorG16::configureDevice`'s + // `accel[0x10778] = 0x60` (`0xfffffe0008a5568c`), and also the kext's + // own zero-source fallback constant in the same expression. + assert_eq!(first, 0x0000_0060_0040_0020); + } + + #[test] + fn an_scs_request_prepends_the_0x17d9_enable_register() { + let scs = scs_request_registers(89, 1, 0, 0).unwrap(); + assert!(scs.enabled); + let program = + build_compute_registers_with_scs(register_parameters(), scs).unwrap(); + assert_eq!(program.len(), COMPUTE_REGISTER_MAX); + // Entry 0 is the enable register, and the historical program follows + // it unchanged -- the kext moves its cursor to 12 before writing + // 0x1a510 (`0xfffffe0008a5fac4`). + assert_eq!(program.as_slice()[0], ComputeRegister::new(0x17d9, 1)); + assert_eq!( + program.as_slice()[1], + ComputeRegister::new(0x1a510, 0x100_0404_0000) + ); + // Nothing else moves: the same registers in the same order, and only + // the three request registers change value. + let disabled = build_compute_registers(register_parameters()).unwrap(); + assert_eq!(program.as_slice().len(), disabled.as_slice().len() + 1); + for (enabled, disabled) in program.as_slice()[1..].iter().zip(disabled.as_slice()) { + assert_eq!(enabled.number, disabled.number); + if !matches!(enabled.number, 0x101d1 | 0x0d479 | 0x107a1) { + assert_eq!(enabled.value, disabled.value); + } + } + // The three request registers still carry the request itself. + assert_eq!(register_value(program.as_slice(), 0x101d1, 0), 0x0001_0059); + assert_eq!(register_value(program.as_slice(), 0x107a1, 0), 0x0100_0000); + assert_eq!( + compute_resume_checkpoints(program.as_slice()).unwrap(), + [39, 40, 40, 41] + ); + } + + #[test] + fn no_scs_request_emits_no_enable_register() { + let program = build_compute_registers(register_parameters()).unwrap(); + assert!(program + .as_slice() + .iter() + .all(|register| register.number != 0x17d9)); + assert_eq!(register_value(program.as_slice(), 0x107a1, 0), 0xff_0000); + } + + #[test] + fn an_empty_scs_request_reproduces_the_historical_registers() { + // Zero pages is the kext's "this pass asks for no ray-traversal + // scratch" branch, and is what this driver emitted before the request + // became settable. register_program_matches_the_hardware_tested_m1n1_builder + // above pins the same thing through the whole program's hash. + let scs = scs_request_registers(0, 0, 0, 0).unwrap(); + assert_eq!(scs.size_word, 0); + assert_eq!(scs.extras, 0); + assert_eq!(scs.window_word, 0xff_0000); + // The window is ignored while no pages are requested. + assert_eq!(scs_request_registers(0, 7, 3, 9), scs_request_registers(0, 7, 3, 0)); + } + + #[test] + fn a_nonempty_scs_request_packs_pages_mib_extras_and_window() { + // 89 pages is the four-section floor (0x58000 bytes) plus the guard + // page AGXMetalG17P adds; one MiB covers it ((89 * 4 + 0x3ff) >> 10). + let scs = scs_request_registers(89, 1, 0, 0).unwrap(); + assert_eq!(scs.size_word, 0x0001_0059); + assert_eq!(scs.extras, 0); + assert_eq!(scs.window_word, 0x0100_0000); + + let wide = scs_request_registers(815, 4, 2, 0x20).unwrap(); + assert_eq!(wide.size_word, 0x0004_032f); + assert_eq!(wide.extras, 2); + assert_eq!(wide.window_word, 0x0120_0000); + } + + #[test] + fn an_unset_mib_is_derived_from_the_page_count_like_metal_does() { + // AGXMetalG17P computes MiB as (pages * 4 + 0x3ff) >> 10 and never + // leaves it zero next to a non-zero page count, so the driver must not + // either: an operator who sets only the page count still gets the + // register value a real ray-tracing kick carries. + // 0x59 pages = the four fixed sections plus the guard page. + assert_eq!(scs_request_registers(0x59, 0, 0, 0).unwrap().size_word, 0x0001_0059); + // 0x32f pages = the intersector layout + // (traversal depth 0x70, driver payload 0x48, 96 cliques). + assert_eq!(scs_request_registers(0x32f, 0, 0, 0).unwrap().size_word, 0x0004_032f); + // 0x3c8 pages = the intersection_query<> layout (driver payload 0x114). + assert_eq!(scs_request_registers(0x3c8, 0, 0, 0).unwrap().size_word, 0x0004_03c8); + // 0x6a7 pages = intersection_query on a + // top-level structure, the Vulkan rayQueryEXT shape + // (traversal depth 0xe8, driver payload 0x128). + assert_eq!(scs_request_registers(0x6a7, 0, 0, 0).unwrap().size_word, 0x0007_06a7); + // An explicit value still wins, so the halves stay separable. + assert_eq!(scs_request_registers(0x59, 9, 0, 0).unwrap().size_word, 0x0009_0059); + // Zero pages keeps the historical register exactly. + assert_eq!(scs_request_registers(0, 0, 0, 0).unwrap().size_word, 0); + } + + #[test] + fn scs_layout_sizes_match_the_measured_metal_shapes() { + let layout = |depth: u64, driver: u64| depth | (96 << 24) | (driver << 48); + + assert_eq!(scs_pages_for_layout(layout(0x70, 0x48)).unwrap(), 0x32f); + assert_eq!(scs_pages_for_layout(layout(0x70, 0x114)).unwrap(), 0x3c8); + assert_eq!(scs_pages_for_layout(layout(0xe8, 0x128)).unwrap(), 0x6a7); + + // The native endComputePass gate suppresses a request when both + // payload sizes are zero, even if depth/clique fields are populated. + assert_eq!(scs_pages_for_layout(0x70 | (96 << 24)).unwrap(), 0); + assert_eq!(scs_pages_for_layout(0).unwrap(), 0); + } + + #[test] + fn scs_layout_rejects_an_unrepresentable_request() { + assert_eq!( + scs_pages_for_layout(u64::MAX), + Err(ComputeBuildError::ScsRequestOutOfRange) + ); + } + + #[test] + fn descriptor_preserves_cdm_sampler_timestamp_and_queue_state() { + let registers = build_compute_registers(register_parameters()).unwrap(); + let mut raw = [0u8; COMPUTE_DESCRIPTOR_SIZE]; + build_compute_descriptor( + ®isters, + ComputeDescriptorObjects { + scheduler_record: 0xffff_fc20_c099_8100, + descriptor_low_alias: 0x70_0034_0000, + dispatch_a: 0xffff_fc20_001c_8028, + dispatch_b: 0xffff_fc20_c07c_0028, + status_a: 0xffff_fc20_0002_4c68, + status_b: 0xffff_fc20_0002_4c70, + shared_control: 0xffff_fc20_c095_0000, + usage_record: 0xffff_fc20_0171_0000, + }, + ComputeDescriptorMetadata { + submit_sequence: 5, + context_id: 3, + grid_index: 2, + work_ordinal: 7, + queue_submission: 8, + queue_ordinal: 7, + submission_index: 9, + support_control: 0xe0a0_0001, + support_flags: 0, + }, + ComputeDescriptorCommand { + cdm_terminator: 0x100_0410_002c, + sampler_array: 0x100_0800_0000, + sampler_count: 4, + user_timestamp_start: 0x200_0000_4000, + user_timestamp_end: 0x200_0000_4008, + }, + &mut raw, + ) + .unwrap(); + + // Re-pinned with the register-derived B1 resume checkpoints, and again + // with the constant completion stamp at +0x0F50 (COMPUTE_COMPLETION_STAMP_CONSTANT). + assert_eq!(fnv1a64(&raw), 0x8a56_8b1c_2a27_0524); + assert_eq!(u64::from_le_bytes(raw[0x0ee0..0x0ee8].try_into().unwrap()), 0x100_0410_002c); + assert_eq!(u64::from_le_bytes(raw[COMPUTE_SAMPLER_ARRAY..COMPUTE_SAMPLER_ARRAY + 8].try_into().unwrap()), 0x100_0800_0000); + assert_eq!(u32::from_le_bytes(raw[COMPUTE_SAMPLER_COUNT..COMPUTE_SAMPLER_COUNT + 4].try_into().unwrap()), 4); + assert_eq!(u64::from_le_bytes(raw[0x0f8c..0x0f94].try_into().unwrap()), 0x200_0000_4000); + // The raw builder retains its bootstrap seed. User publication binds + // the physical queue's actual completion prefix after this step. + assert_eq!(u32::from_le_bytes(raw[0x0f50..0x0f54].try_into().unwrap()), COMPUTE_COMPLETION_STAMP_CONSTANT); + assert_eq!( + [ + u16::from_le_bytes(raw[0x18..0x1a].try_into().unwrap()), + u16::from_le_bytes(raw[0x1a..0x1c].try_into().unwrap()), + u16::from_le_bytes(raw[0x1c..0x1e].try_into().unwrap()), + u16::from_le_bytes(raw[0x1e..0x20].try_into().unwrap()), + ], + [38, 39, 39, 40] + ); + } + + #[test] + fn invalid_command_relations_are_caught_before_encoding() { + let registers = build_compute_registers(register_parameters()).unwrap(); + let objects = ComputeDescriptorObjects { + scheduler_record: 1, + descriptor_low_alias: 0x70_0000_0000, + dispatch_a: 2, + dispatch_b: 3, + status_a: 4, + status_b: 5, + shared_control: 6, + usage_record: 7, + }; + let metadata = ComputeDescriptorMetadata { + submit_sequence: 1, + context_id: 3, + grid_index: 2, + work_ordinal: 0, + queue_submission: 1, + queue_ordinal: 0, + submission_index: 1, + support_control: 0xe0a0_0001, + support_flags: 0, + }; + let mut raw = [0u8; COMPUTE_DESCRIPTOR_SIZE]; + assert_eq!( + build_compute_descriptor( + ®isters, + objects, + metadata, + ComputeDescriptorCommand { + cdm_terminator: register_parameters().cdm_base - 4, + sampler_array: 0, + sampler_count: 0, + user_timestamp_start: 0, + user_timestamp_end: 0, + }, + &mut raw, + ), + Err(ComputeBuildError::CdmTerminatorBeforeStream) + ); + assert_eq!( + build_compute_descriptor( + ®isters, + objects, + metadata, + ComputeDescriptorCommand { + cdm_terminator: register_parameters().cdm_base, + sampler_array: 0, + sampler_count: 1, + user_timestamp_start: 0, + user_timestamp_end: 0, + }, + &mut raw, + ), + Err(ComputeBuildError::SamplerPair) + ); + } + + #[test] + fn queue_graph_matches_the_hardware_tested_m1n1_builder() { + let mut raw = [0u8; COMPUTE_QUEUE_GRAPH_SIZE]; + let addresses = build_compute_queue_graph( + 0xffff_fc20_c100_0000, + 0xffff_fc20_c200_0000, + &mut raw, + ) + .unwrap(); + assert_eq!( + addresses, + ComputeQueueGraphAddresses { + queue_record: 0xffff_fc20_c100_0000, + pointers: 0xffff_fc20_c100_0100, + job_list: 0xffff_fc20_c100_0200, + item_ring: 0xffff_fc20_c100_4000, + } + ); + assert_eq!(u32::from_le_bytes(raw[0x48..0x4c].try_into().unwrap()), 0); + // Normalize only the captured process identity for the historical hash. + put_u32(&mut raw, 0x48, 0x1aa); + assert_eq!(fnv1a64(&raw), 0x3490_ef31_add0_ac88); + } + + #[test] + fn initial_channel_group_matches_the_output_positive_add3_builder() { + let base = 0xffff_fc20_c100_0000; + let descriptor = 0xffff_fc20_c200_0000; + let mut graph = [0u8; COMPUTE_QUEUE_GRAPH_SIZE]; + build_compute_queue_graph(base, 0xffff_fc20_c500_0000, &mut graph).unwrap(); + put_u32(&mut graph, 0x48, 0x1234_5678); + let mut context = vec![0u8; COMPUTE_QUEUE_CONTEXT_EXTENT]; + let addresses = build_initial_compute_channel_group( + base, + descriptor, + ComputeSksmEntryAliases::new( + 0x70_0040_0000, + 0xffff_fc20_c300_0000, + 0x8000, + 0x100, + ) + .unwrap(), + 0xffff_fc20_c400_0000, + 0xffff_fc20_c500_0000, + 4, + 1, + &mut graph, + &mut context, + true, + None, + ) + .unwrap(); + assert_eq!( + addresses, + ComputeChannelGroupAddresses { + queue: base, + optional: base + 0x6800, + event: base + 0x6c00, + write_index: 3, + } + ); + // The compute pool uses ID 1 consistently in registration and FList. + // Normalize only the captured PID for the original whole-graph golden. + assert_eq!(&graph[COMPUTE_OPTIONAL + 0x46..COMPUTE_OPTIONAL + 0x48], &[5, 0]); + assert_eq!(&graph[0x48..0x4c], &0x1234_5678u32.to_le_bytes()); + assert_eq!(&graph[COMPUTE_OPTIONAL + 0x5a..COMPUTE_OPTIONAL + 0x5e], &0x1234_5678u32.to_le_bytes()); + put_u32(&mut graph, 0x48, 0x170); + put_u32(&mut graph, COMPUTE_OPTIONAL + 0x5a, 0x170); + assert_eq!(fnv1a64(&graph), 0x66c2_81b4_ef4e_f1db); + assert_eq!(fnv1a64(&context), 0x5ba5_b355_8c90_0783); + } + + #[test] + fn config_update_uses_physical_graph_flist_index_not_qos() { + let mut graph = [0u8; COMPUTE_QUEUE_GRAPH_SIZE]; + let aliases = ComputeSksmEntryAliases::new( + 0x70_0040_0000, + 0xffff_fc20_c300_0000, + 0x8000, + 0x100, + ) + .unwrap(); + encode_compute_config_update( + &mut graph, + aliases, + 0xffff_fc20_c400_0000, + 0xffff_fc20_c500_0000, + 5, + 2, + true, + ComputeConfigUpdates { qos: true, context: true }, + None, + ) + .unwrap(); + assert_eq!(&graph[COMPUTE_OPTIONAL + 0x46..COMPUTE_OPTIONAL + 0x48], &[6, 0]); + assert_eq!( + &graph[COMPUTE_OPTIONAL + 0x56..COMPUTE_OPTIONAL + 0x58], + &[2, 0] + ); + } + + #[test] + fn sksm_entry_aliases_preserve_one_backing_offset() { + const EXTENT: usize = 0x8000; + let aliases = ComputeSksmEntryAliases::new( + 0x70_004d_8000, + 0xffff_fc20_0027_8000, + EXTENT, + 0x100, + ) + .unwrap(); + assert_eq!( + aliases.range_at(0x200, 0x200), + Ok([0x70_004d_8200, 0xffff_fc20_0027_8200]) + ); + assert_eq!( + aliases.range_at(EXTENT - 0x200, 0x200), + Ok([0x70_004d_fe00, 0xffff_fc20_0027_fe00]) + ); + assert_eq!( + aliases.range_at(EXTENT, 1), + Err(ComputeBuildError::InvalidSksmEntryAliases) + ); + assert_eq!( + ComputeSksmEntryAliases::new(0x70_004d_8000, 0x70_004d_8000, EXTENT, 0x100), + Err(ComputeBuildError::InvalidSksmEntryAliases) + ); + assert_eq!( + ComputeSksmEntryAliases::new( + 0x70_004d_8001, + 0xffff_fc20_0027_8000, + EXTENT, + 0x100, + ), + Err(ComputeBuildError::InvalidSksmEntryAliases) + ); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/compute_state.rs b/drivers/gpu/drm/asahi/g18/fw/compute_state.rs new file mode 100644 index 0000000000000..6050c0ee2d915 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/compute_state.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Fields copied by the native 26A428 compute-state writer (8bf16cc..8bf18a8). +//! This is a bounded state view, not a command-stream or ownership validator. +//! Host policy, context, pool, scheduler and completion fields stay with callers. + +#![allow(dead_code)] + +use crate::g18::fw::compute::{G18ComputeCommandInputs, G18ComputeRegisterInputs}; + +pub(crate) const G18_COMPUTE_STATE_BYTES: usize = 0x268; + +pub(crate) struct G18ComputeState<'a>(&'a [u8; G18_COMPUTE_STATE_BYTES]); + +impl<'a> G18ComputeState<'a> { + pub(crate) fn parse(bytes: &'a [u8]) -> Option { + Some(Self(bytes.try_into().ok()?)) + } + + fn q(&self, offset: usize) -> u64 { + u64::from_le_bytes(self.0[offset..offset + 8].try_into().unwrap()) + } + + fn w(&self, offset: usize) -> u32 { + u32::from_le_bytes(self.0[offset..offset + 4].try_into().unwrap()) + } + + fn flag(&self, offset: usize) -> u8 { self.0[offset] & 1 } + + /// Replace only the fields sourced from userspace compute state. This + /// deliberately does not synthesize any native host-owned resource fields. + pub(crate) fn apply(&self, c: &mut G18ComputeCommandInputs, r: &mut G18ComputeRegisterInputs) { + c.d488 = self.q(0x08); + c.d490 = self.q(0x10); + c.d498 = self.q(0x18); + c.d4a0 = self.q(0x20); + c.d4a8 = self.q(0x28); + c.d4b0 = self.w(0x30); + c.d4b8 = self.q(0x38); + c.d4c0 = self.w(0x40); + c.d4d0 = self.w(0x50); + c.d4e0 = self.q(0x60); + c.d508 = self.flag(0xc4); + c.d628 = self.0[0xd0]; + c.d629 = self.flag(0x249); + c.d770 = self.flag(0x24c); + c.d8b0 = self.flag(0x24e); + r.program = [self.q(0x58), self.q(0), self.q(0x70), self.q(0x78), self.q(0x80), self.q(0x88)]; + r.aliases = [self.q(0x1f8), self.q(0x200), self.q(0x208), self.q(0x210), self.q(0x218), self.q(0x220)]; + r.d540 = self.flag(0x1f4); + r.d548 = self.q(0xa8); + r.d5d9 = self.flag(0x24f); + r.d62a = self.flag(0x24a); + r.d62b = self.flag(0x24b); + r.d418 = self.w(0x260); + r.d774 = self.w(0xa4); + // The native 16-byte store starts at descriptor +0x5b8, not +0x5c0. + r.d5c0 = self.w(0x230); + r.d5c4 = self.w(0x234); + r.d5c8 = self.w(0x238); + r.d50c = self.w(0xc0); + r.d510 = self.w(0xc8); + r.d514 = self.w(0xcc); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/config.rs b/drivers/gpu/drm/asahi/g18/fw/config.rs new file mode 100644 index 0000000000000..b532b92938c0a --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/config.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fixed choices for the currently qualified G17P firmware path. +//! +//! Values preserve the former module-parameter defaults. Firmware ABI +//! alternatives require a code change and qualification, not a boot-time mix +//! of independent experimental switches. Debugging and timeout controls remain +//! module parameters in `asahi.rs`. + +pub(crate) const RENDER_SLOT_AFFINITY: u32 = 1; +pub(crate) const RENDER_SLOT_GLOBALS: u32 = 3; +pub(crate) const RENDER_NOP_RETIRE: u32 = 1; +pub(crate) const REG_DUMP: u32 = 0; +pub(crate) const SCAN_GATE: u32 = 0; +pub(crate) const NATIVE_ORDER: u32 = 1; +pub(crate) const SUBMIT_POLLS: u32 = 500; +pub(crate) const PROBE_LOW_RANGE: u32 = 0; +pub(crate) const LATE_SKSM: u32 = 0; +pub(crate) const FLUSHID_FIX: u32 = 0; +/// Session compute FList identity; render uses the separate pool ID zero. +pub(crate) const COMPUTE_FLIST_ID: u16 = 1; + +/// Published CL graphs retain distinct pools; render owns zero. The physical +/// QID lease makes this mapping stable across logical owner turnover. +pub(crate) const fn compute_flist_id(queue_id: u8) -> u16 { + queue_id as u16 + 1 +} +pub(crate) const SKIP: u32 = 0; +pub(crate) const DEFER_ADD_KICKS: u32 = 0; +pub(crate) const CTX2_ROOT: u32 = 3; +pub(crate) const PROGRESS_WATCHDOG: u32 = 1; +pub(crate) const POWER_HOLD: u32 = 0; +pub(crate) const POWER_WIRE: u32 = 0; +pub(crate) const IDLE_SELECTOR: u32 = 0; +pub(crate) const RENDER_ABSOLUTE_POINTERS: u32 = 0; +pub(crate) const RENDER_STATUS_FLAG: u32 = 0; +pub(crate) const RENDER_GATE_PROBE: u32 = 0; +pub(crate) const RENDER_KSM_PROBE: u32 = 0; +pub(crate) const RENDER_HEADER_FLAGS: u32 = 0; +pub(crate) const RENDER_GENERATION_BIAS: u32 = 0; +pub(crate) const RENDER_BG_PREFIX: u32 = 0; +pub(crate) const RENDER_EOT_BIND_ZERO: u32 = 1; +pub(crate) const RENDER_MCACHE_HWSID: u32 = 24; +pub(crate) const RENDER_MCACHE_MODE: u32 = 1; +pub(crate) const RENDER_MCACHE_SELECTOR: u32 = 64; +pub(crate) const RENDER_KSM_MCACHE: u32 = 1; +pub(crate) const RENDER_NO_BARRIER: u32 = 0; +pub(crate) const RENDER_SPLIT_LAUNCH: u32 = 0; +pub(crate) const RENDER_NATIVE_DOORBELLS: u32 = 0; +pub(crate) const RENDER_NATIVE_POOL_A_FIRST_RECORD: u32 = 1; +pub(crate) const RENDER_NATIVE_FRAGMENT_HWPB: u32 = 1; +pub(crate) const RENDER_3D_RCE: u32 = 0xff; +pub(crate) const SEED_WORK_CHANNEL: u32 = 0; +pub(crate) const RENDER_3D_STROBE: u32 = 0; +pub(crate) const IRQ_BURST: u32 = 0; +pub(crate) const SLOT_BURST: u32 = 0; +pub(crate) const PIPE_SWEEP: u32 = 0; +pub(crate) const RENDER_TA_DM: u32 = 0xff; +pub(crate) const RENDER_3D_DM: u32 = 0xff; +pub(crate) const FW_TRACE_CLASSES: u32 = 0; +pub(crate) const KSM_COLD_ARM: u32 = 0; +pub(crate) const ADMIT_ARM: u32 = 0; +pub(crate) const RENDER_NO_INSTALL: u32 = 0; +pub(crate) const FRAG_ORACLE: u32 = 0; +pub(crate) const TA_ORACLE: u32 = 0; +pub(crate) const RETAIN_STAGING: u32 = 1; +pub(crate) const RENDER_USC_FREELIST: u32 = 1; +pub(crate) const RECOVERY_ALLOW_ACTIVE_SCHEDULER: u32 = 1; +pub(crate) const RENDER_SERVICE_EVENTS: u32 = 1; +pub(crate) const COMPUTE_SKSM_ENTRY: u32 = 1; +pub(crate) const REPORT_DRAIN: u32 = 0; +pub(crate) const REPORT_DRAIN_BUDGET: u32 = 0; +pub(crate) const RENDER_RECOVERY_RETIRE: u32 = 0; +pub(crate) const COMPUTE_CONFIG_CONTEXT: u32 = 1; +// bit0: scope a faulted async-compute queue's admission poison to its own +// client instead of the whole device (landed with f6a71e767d55a). +// bit1: scope a lost render submission the same way. When a firmware +// stamp-blamed recovery is serviced to a clean final clear (state=0, +// epoch=1) but the driver cannot prove the exact lost render was +// firmware-retired, fail ONLY that client's VM and retire its physical +// slot; keep the session up for every other client. Without this bit the +// render settle path calls note_session_recovery_pending and the whole +// GPU session is declared crashed -- and on G17P a crashed session's +// rebuild never re-runs the RTKit HELLO handshake (4/4 ETIME), so the +// GPU is dead until reboot. Measured 2026-09-26 (07:48, 07:54, and the +// real desktop at 08:12): a single unprivileged client's first render +// submission killed the GPU for everyone. Both scopings are the correct +// fault-containment default, not a diagnostic opt-in: they never touch +// the healthy path's batching or concurrency. +pub(crate) const QUARANTINE_SCOPE: u32 = 3; +pub(crate) const RENDER_NATIVE_REGS: u32 = 0; +pub(crate) const RENDER_NATIVE_BYTES: u32 = 0; +pub(crate) const RENDER_TVB: u32 = 2; +pub(crate) const DEP_RECORDS: u32 = 0; +pub(crate) const NO_IMPLICIT_DEP: u32 = 0; +pub(crate) const RENDER_TA_QID_FIELD: u32 = 1; +pub(crate) const RENDER_SLOT_TEMPLATE: u32 = 0; +pub(crate) const RENDER_SEED_TS: u32 = 0; +pub(crate) const RENDER_KICK_QOS: u32 = 0; +pub(crate) const RENDER_TAG16: u32 = 0; +pub(crate) const RENDER_POPULATED_HIGH_ROOT: u32 = 0; +pub(crate) const RENDER_SKSM: u32 = 3; +pub(crate) const IRQ_COMPLETION: u32 = 1; +pub(crate) const RING_NORMALIZE: u32 = 0; +pub(crate) const RING_LIMIT: u32 = 0; +pub(crate) const RING_WRAP: u32 = 0; +pub(crate) const COMPUTE_SCS_PAGES: u32 = 0; +pub(crate) const COMPUTE_SCS_MIB: u32 = 0; +pub(crate) const COMPUTE_SCS_EXTRAS: u32 = 0; +pub(crate) const COMPUTE_SCS_WINDOW: u32 = 0; +pub(crate) const COMPLETION_STAMP: u32 = 1; +pub(crate) const REPEAT_CONTEXT: u32 = 1; +pub(crate) const REPEAT_RING: u32 = 1; +pub(crate) const REPEAT_ACTIVATION: u32 = 0; +pub(crate) const QUEUE_RECYCLE: u32 = 2; +/// Diagnostic only: 0 allocates a fresh QID graph for each new async-compute +/// owner so retained graph rebinding can be isolated from execution faults. +/// Production must remain 1 once the handoff defect is fixed. +pub(crate) const ASYNC_COMPUTE_REBIND: u32 = 1; +/// Diagnostic only: retain the pre-publication B1 descriptor/FList snapshot +/// so the first halted fault can be compared with the exact bytes it consumed. +pub(crate) const COMPUTE_RESOURCE_SNAPSHOT: u32 = 0; +pub(crate) const ZERO_DURATION: u32 = 1; +pub(crate) const GATE_QUIESCE_MS: u32 = 0; +pub(crate) const RECOVERY_RESYNC: u32 = 1; +pub(crate) const ASC_RESTART: u32 = 0; +pub(crate) const ASC_GATE_SETTLE_MS: u32 = 10; +pub(crate) const ASC_TEARDOWN: u32 = 0; +pub(crate) const ABANDON_RECOVERY: u32 = 1; +pub(crate) const SKSM_MMIO: u32 = 0; + +/// Default-off diagnostic: ordinary Job graphs with a shared per-submit TSQ +/// use native pending-IO-fence-free TA policy. This defers the early TA drain; +/// it does not remove maintenance or relax any post-maintenance fence gate. +/// Bootstrap/private-node paths retain their qualified policy. Do not promote +/// without mixed/failure/chunking liveness and throughput qualification. +pub(crate) const RENDER_NATIVE_TA_DRAIN_PROBE: bool = cfg!(g17p_native_ta_drain_probe); diff --git a/drivers/gpu/drm/asahi/g18/fw/events.rs b/drivers/gpu/drm/asahi/g18/fw/events.rs new file mode 100644 index 0000000000000..123543037ddd0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/events.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Decoder of the firmware completion-stamp event that G18 reads from its report ring. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. + +pub(crate) const G17P_FIRMWARE_EVENT_ENTRY_SIZE: usize = 0x48; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum CompletionDecodeError { + RecordSize, + ChannelNumber, + EventType, + TraceShape, + CompletionSelector, + CompletionCommandType, + CompletionCapacity, + CompletionCursor, + CursorRange, + QueueCounterOrder, + QueueCounterRegression, + PublishedPrefixBeyondWrite, + RenderWindow, + RenderCursorOutsideWindow, + CompletionIrqSource, + CompletionOrdinal, + CompletionNotificationCount, + CompletionQueueId, + CompletionTimestamp, + CompletionTimestampOrder, + FirmwareEventType, + FirmwareEventDiagnosticCount, + FirmwareEventStampIndex, + FirmwareEventStampMissing, + FirmwareEventUmaPoolIndex, + FirmwareEventTvbRequest, + FirmwareEventUmaAddress, + CompletionOutstanding, + CompletionRouteUnavailable, + FirmwareEventEndpoint, + FirmwareEventMessage, + FirmwareEventCursor, + FirmwareEventEmpty, +} + +fn get_u16(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) +} + +fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) +} + +/// Firmware event type 1, delivered through the ordinary EP0x20 event ring. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PFirmwareStampEvent { + pub(crate) masks: [u64; 2], + pub(crate) diagnostic_pair_count: u16, +} + +pub(crate) fn decode_g17p_firmware_stamp_event( + raw: &[u8], +) -> Result { + if raw.len() != G17P_FIRMWARE_EVENT_ENTRY_SIZE { + return Err(CompletionDecodeError::RecordSize); + } + if get_u32(raw, 0x00) != 1 { + return Err(CompletionDecodeError::FirmwareEventType); + } + let diagnostic_pair_count = get_u16(raw, 0x14); + if diagnostic_pair_count >= 6 { + return Err(CompletionDecodeError::FirmwareEventDiagnosticCount); + } + Ok(G17PFirmwareStampEvent { + masks: [get_u64(raw, 0x04), get_u64(raw, 0x0c)], + diagnostic_pair_count, + }) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/graphics_payload.rs b/drivers/gpu/drm/asahi/g18/fw/graphics_payload.rs new file mode 100644 index 0000000000000..71a82541685e0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/graphics_payload.rs @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded private pre-submit color capture for G18 graphics bring-up. +//! +//! This reader preserves user virtual pages and the userspace render segment. +//! It neither creates a GPU VM nor submits a job. Native TA/3D command builders, +//! kernel-owned graphics resources and completion remain separate prerequisites. +//! Resource usage bits are capture metadata, NOT native PTE permissions. + +#![allow(dead_code)] + +pub(crate) const HEADER_BYTES: usize = 64; +pub(crate) const PAGE_BYTES: usize = 0x4000; +pub(crate) const RECORD_BYTES: usize = 16 + PAGE_BYTES; +pub(crate) const COMMAND_BYTES: usize = 0xac0; +pub(crate) const GEARS_COMMAND_BYTES: usize = 0xad8; +pub(crate) const PAGE_COUNT: usize = 214; +pub(crate) const GEARS_PAGE_COUNT: usize = 231; +pub(crate) const WITNESS_PAGE_COUNT: usize = 215; +pub(crate) const WITNESS: u64 = 0x100_000a_0000; +pub(crate) const WITNESS_DATA_BYTES: usize = 4 + 256 * 256 / 8; +pub(crate) const OUTPUT: u64 = 0x100_0005_8000; +pub(crate) const OUTPUT_BYTES: u64 = 256 * 256 * 4; + +// Root allocation address, length, and captured usage for submission 1 in the +// two independent native color runs. Their shader heaps need not be identical. +const RANGES: [(u64, usize, u32); 24] = [ + (0, 0x10000, 128), + (0x18000, 0x8000, 1), (0x28000, 0x8000, 1), + (0x38000, 0x8000, 1), (0x48000, 0x8000, 1), + (0x58000, 0x8000, 1024), (0x68000, 0xc000, 1024), + (0x100_0000_0000, 0x10000, 128), + (0x100_0001_8000, 0x20000, 129), + (0x100_0004_0000, 0x10000, 128), (OUTPUT, 0x40000, 12), + (0x100_000a_0000, 0x20000, 128), + (0x100_000c_8000, 0x6c000, 128), + (0x100_0013_8000, 0x8000, 1), + (0x100_0014_8000, 0x8000, 4), + (0x100_0015_8000, 0x8000, 1), + (0x100_0016_8000, 0x8000, 4), + (0x100_0017_8000, 0x100000, 4), + (0x100_0028_0000, 0x8000, 4), + (0x100_0029_0000, 0x8000, 1), + (0x100_002a_0000, 0x8000, 1), + (0x100_002b_0000, 0x8000, 1), + (0x100_002c_0000, 0x8000, 1), + (0x100_002d_0000, 0xc0000, 4), +]; + +const GEARS_RANGES: [(u64, usize, u32); 25] = [ + (0, 0x10000, 128), (0x18000, 0x8000, 1), (0x28000, 0x8000, 1), + (0x38000, 0x8000, 1), (0x48000, 0x8000, 1), (0x58000, 0x8000, 1024), + (0x68000, 0xc000, 1024), (0x100_0000_0000, 0x10000, 128), + (0x100_0001_8000, 0x20000, 129), (0x100_0004_0000, 0x10000, 128), + (OUTPUT, 0x40000, 12), (0x100_000a_0000, 0x20000, 128), + (0x100_000c_8000, 0x44000, 12), (0x100_0011_0000, 0x6c000, 128), + (0x100_0018_0000, 0x8000, 1), (0x100_0019_0000, 0x8000, 4), + (0x100_001a_0000, 0x8000, 1), (0x100_001b_0000, 0x8000, 4), + (0x100_001c_0000, 0x100000, 4), (0x100_002c_8000, 0x8000, 4), + (0x100_002d_8000, 0x8000, 1), (0x100_002e_8000, 0x8000, 1), + (0x100_002f_8000, 0x8000, 1), (0x100_0030_8000, 0x8000, 1), + (0x100_0031_8000, 0xc0000, 4), +]; + +// Separate native fragment-witness capture: its extra 16-KiB buffer shifts +// every subsequent pipeline allocation. Never mix these with colour-a/b. +const WITNESS_RANGES: [(u64, usize, u32); 25] = [ + (0, 0x10000, 128), (0x18000, 0x8000, 1), (0x28000, 0x8000, 1), + (0x38000, 0x8000, 1), (0x48000, 0x8000, 1), (0x58000, 0x8000, 1024), + (0x68000, 0xc000, 1024), (0x100_0000_0000, 0x10000, 128), + (0x100_0001_8000, 0x20000, 129), (0x100_0004_0000, 0x10000, 128), + (OUTPUT, 0x40000, 12), (WITNESS, 0x4000, 12), + (0x100_000a_8000, 0x20000, 128), (0x100_000d_0000, 0x6c000, 128), + (0x100_0014_0000, 0x8000, 1), (0x100_0015_0000, 0x8000, 4), + (0x100_0016_0000, 0x8000, 1), (0x100_0017_0000, 0x8000, 4), + (0x100_0018_0000, 0x100000, 4), (0x100_0028_8000, 0x8000, 4), + (0x100_0029_8000, 0x8000, 1), (0x100_002a_8000, 0x8000, 1), + (0x100_002b_8000, 0x8000, 1), (0x100_002c_8000, 0x8000, 1), + (0x100_002d_8000, 0xc0000, 4), +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Capture { ColorA = 1, ColorB = 2, GearsG1 = 3, FragmentWitness = 4 } + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum PayloadError { Length, Header, Checksum, Page, Command, Output, Witness } + +pub(crate) struct GraphicsPayload<'a> { + bytes: &'a [u8], capture: Capture, command_bytes: usize, page_count: usize, +} +pub(crate) struct PayloadPage<'a> { + pub(crate) address: u64, + pub(crate) usage: u32, + pub(crate) bytes: &'a [u8], +} + +fn u32_at(b: &[u8], off: usize) -> u32 { + u32::from_le_bytes(b[off..off + 4].try_into().unwrap()) +} +fn u64_at(b: &[u8], off: usize) -> u64 { + u64::from_le_bytes(b[off..off + 8].try_into().unwrap()) +} + +/// Transfer integrity only; the offline receipt carries SHA256 provenance. +pub(crate) fn fnv64(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |h, b| + (h ^ u64::from(*b)).wrapping_mul(0x100_0000_01b3)) +} + +impl<'a> GraphicsPayload<'a> { + pub(crate) fn parse(bytes: &'a [u8]) -> Result { + if bytes.len() < HEADER_BYTES { return Err(PayloadError::Length); } + let capture = match u64_at(bytes, 56) { + 1 => Capture::ColorA, 2 => Capture::ColorB, 3 => Capture::GearsG1, + 4 => Capture::FragmentWitness, + _ => return Err(PayloadError::Header), + }; + let gears = capture == Capture::GearsG1; + let witness = capture == Capture::FragmentWitness; + let command_bytes = if gears || witness { GEARS_COMMAND_BYTES } else { COMMAND_BYTES }; + let page_count = if gears { GEARS_PAGE_COUNT } else if witness { WITNESS_PAGE_COUNT } else { PAGE_COUNT }; + let size = HEADER_BYTES + command_bytes + page_count * RECORD_BYTES; + if bytes.len() != size { return Err(PayloadError::Length); } + if &bytes[..8] != b"G18GFX01" || u32_at(bytes, 8) != 2 + || u32_at(bytes, 12) != HEADER_BYTES as u32 + || u32_at(bytes, 16) != PAGE_BYTES as u32 + || u32_at(bytes, 20) != page_count as u32 + || u32_at(bytes, 24) != 256 || u32_at(bytes, 28) != 256 + || u32_at(bytes, 32) != 1024 + || u32_at(bytes, 36) != command_bytes as u32 + || u64_at(bytes, 40) != OUTPUT + { return Err(PayloadError::Header); } + if fnv64(&bytes[HEADER_BYTES..]) != u64_at(bytes, 48) { + return Err(PayloadError::Checksum); + } + let p = Self { bytes, capture, command_bytes, page_count }; + let cmd = p.command(); + // Fixture identity is bound to the actual userspace command; the two + // colour shader heaps must never be interchanged by relabelling a file. + let command_fnv = match capture { + Capture::ColorA => 0x89e7_aca5_9572_cb78, + Capture::ColorB => 0x75e0_2b53_1fcc_46f2, + Capture::GearsG1 => 0x23d3_27af_95ce_9eb7, + Capture::FragmentWitness => 0x5eed_ef2d_0bb3_31af, + }; + if u32_at(cmd, 0) != 0x10000 || u32_at(cmd, 4) != command_bytes as u32 + || u64_at(cmd, command_bytes - 24) != OUTPUT || fnv64(cmd) != command_fnv + { return Err(PayloadError::Command); } + let mut index = 0; + let mut output_pages = 0; + let ranges: &[(u64, usize, u32)] = if gears { &GEARS_RANGES } + else if witness { &WITNESS_RANGES } else { &RANGES }; + for &(base, length, usage) in ranges { + for offset in (0..length).step_by(PAGE_BYTES) { + let page = p.page(index).ok_or(PayloadError::Page)?; + let record = HEADER_BYTES + command_bytes + index * RECORD_BYTES; + if page.address != base + offset as u64 || page.usage != usage + || u32_at(bytes, record + 12) != PAGE_BYTES as u32 + { return Err(PayloadError::Page); } + if (OUTPUT..OUTPUT + OUTPUT_BYTES).contains(&page.address) { + if page.bytes.iter().any(|&b| b != 0xa5) { + return Err(PayloadError::Output); + } + output_pages += 1; + } + if witness && page.address == WITNESS + && (page.bytes[..WITNESS_DATA_BYTES].iter().any(|&b| b != 0) + || page.bytes[WITNESS_DATA_BYTES..].iter().any(|&b| b != 0xa5)) + { return Err(PayloadError::Witness); } + index += 1; + } + } + if index != page_count || output_pages != 16 { return Err(PayloadError::Page); } + Ok(p) + } + + pub(crate) fn capture(&self) -> Capture { self.capture } + pub(crate) fn body_digest(&self) -> u64 { u64_at(self.bytes, 48) } + pub(crate) fn page_count(&self) -> usize { self.page_count } + pub(crate) fn command(&self) -> &'a [u8] { + &self.bytes[HEADER_BYTES..HEADER_BYTES + self.command_bytes] + } + pub(crate) fn page(&self, index: usize) -> Option> { + if index >= self.page_count { return None; } + let off = HEADER_BYTES + self.command_bytes + index * RECORD_BYTES; + Some(PayloadPage { address: u64_at(self.bytes, off), usage: u32_at(self.bytes, off + 8), + bytes: &self.bytes[off + 16..off + RECORD_BYTES] }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/initdata.rs b/drivers/gpu/drm/asahi/g18/fw/initdata.rs new file mode 100644 index 0000000000000..a1e5e06bf25b8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/initdata.rs @@ -0,0 +1,6549 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! Evidence-backed T8140/G17P firmware init-data layout. +//! +//! HISTORY / PROVENANCE. The previous revision of this module was derived from +//! the **M5 / G17S** on-disk firmware `GFX.a010.macho` (parser at VA +//! 0xfffffc000002127c) — a different chip and a different firmware build from +//! the target, **A18 Pro / T8140 / G17P**. G17S and G17P share a generation +//! name but not an ABI: every structure size in that model was wrong, the five +//! `+0x254` pointers were modeled as independent allocations instead of views +//! into one bundle, and the M5 magic 0x0c8d_e339_0728_04c0 is rejected by the +//! T8140 firmware with "Host and FW Interface is mismatched". This revision is +//! re-derived from: +//! - live T8140 hardware runs (Mac17,5 / J700 / T8140), which booted both +//! firmware instances with a source-constructed descriptor; +//! - our own J700 firmware image: the version-word gate is the MOVZ/MOVK +//! chain at `__TEXT+0x1e8a8` (`mov x10,#0x4c0; movk #0x396,lsl 16; +//! movk #0xa322,lsl 32; movk #0xc8a,lsl 48`), and the same routine +//! confirms the endpoint-0x20 doorbell contract +//! (`(msg & 0xbf<<48) == 0x81<<48`, address rebuilt as +//! `msg | 0xfffff0_0000_0000_0000`); +//! - the executable serializers in the m1n1 tree +//! (`proxyclient/m1n1/agx/g17p.py`, `g17p_initdata.py`) and +//! `docs/t8140-g17p-firmware-abi-spec.md`, both validated by byte-exact +//! rebuilds of pre-handoff captures. +//! +//! The object graph: each firmware instance is handed one root (`0xb8` bytes +//! for the primary, `0xc8` for the secondary). The root carries the version +//! tuple, the full UAT geometry (page size, level count, and three 0x20-byte +//! level descriptors), and points at a per-instance main configuration +//! (`0x600`), the shared region A (16 KiB), the shared region C (`0x1000`), +//! and per-instance status blocks. The main configuration points at the shared +//! hardware-data object (`0x3db4`, at the base of a contiguous bundle +//! allocation of at least `0x10000`) and carries a 17-entry channel table. +//! The five bare pointers at main `+0x254` are views into that one bundle; +//! allocating them independently breaks the overlap and offset relations +//! firmware relies on. +//! +//! Byte slices are used instead of packed Rust structures because several +//! proven fields are unaligned (main `+0x254`, main `+0x471`, and the region +//! record address at `+0x0c`). + +use core::ops::Range; +use core::result::Result; + +/// The recovered fields are sufficient for offline validation and encoding. +pub(crate) const GROUNDED_LAYOUT_AVAILABLE: bool = true; +/// The init-data object graph the T8140 firmware *accepts and boots with* — +/// per-role root sizes, the version tuple, the UAT geometry block, the main +/// configuration, the bundle-view relations, the 17-entry channel table, the +/// hardware-data acceptance block, region C, and the status blocks — is +/// grounded by live T8140 boots of a source-constructed descriptor. This is +/// what `INITDATA_GRAPH` gates. +pub(crate) const PARSE_ACCEPTED_GRAPH_AVAILABLE: bool = true; +/// Whether every hardware-data scalar the firmware reads is grounded to a +/// concrete *derivation* (rather than a recorded value). It is not: the +/// performance ladders/voltage columns and the recorded opaque platform +/// constants ([`HW_DATA_OPAQUE_RUNS`], [`REGION_C_RUNS`]) still lack an ADT +/// derivation, and this module does not encode the performance tables at all +/// (see [`PERF_TABLE_VALUES_ENCODED`]). +pub(crate) const ALL_HW_GLOBALS_VALUES_GROUNDED: bool = false; +/// The complete hardware-validated T8140 performance tables are encoded. +/// Their eventual ADT derivation remains future work, so they are retained as +/// target-scoped measured values rather than inferred from an AGX2 config. +pub(crate) const PERF_TABLE_VALUES_ENCODED: bool = true; +/// The sparse *content* of the five bundle views and of the bundle's third +/// page (`MAIN_ADDR_OBJECTS` / `HWDATA_BUNDLE_STATIC_RUNS` in `g17p.py`, +/// several KiB of recorded runs) is not yet carried by this module; only the +/// view topology is. A live boot needs those runs staged into the bundle. +pub(crate) const BUNDLE_VIEW_CONTENT_CARRIED: bool = true; +/// The primary status-B pointer names a large (`0xeb00`) status/configuration +/// object, not merely the 0x80-byte block modeled here: it holds FW-control +/// state/ring pointers at `+0x48e0`/`+0x48e8`, a lifecycle header at +/// `+0xe434`, and sparse configuration from `+0xe440` +/// (`NATIVE_PRIMARY_STATUS_B_CONFIG_RUNS` in `g17p.py`). Not yet carried. +pub(crate) const PRIMARY_STATUS_B_FULL_OBJECT_CARRIED: bool = true; + +// --- Object sizes ------------------------------------------------------------ + +/// Root size for the primary (`gfx-asc`) instance. +pub(crate) const ROOT_SIZE_PRIMARY: usize = 0x0b8; +/// Root size for the secondary (`gfx1-asc`) instance; the extra 0x10 bytes are +/// the two secondary-only pointers at `+0xb8`/`+0xc0`. +pub(crate) const ROOT_SIZE_SECONDARY: usize = 0x0c8; +pub(crate) const MAIN_CONFIG_SIZE: usize = 0x600; +pub(crate) const HW_DATA_SIZE: usize = 0x3db4; +/// Native QoS manager block published through hardware data. +pub(crate) const G17P_QOS_RESOURCE_OFFSET: usize = 0x27a4; +/// The QoS table pointer as the LIVE 25G83 build reads it. +/// +/// KEXT_PROVEN (25G83 decompile): the live initdata object is 0x2708 bytes and +/// no instruction in that build references 0x27a4 or 0x27ac at all -- both of +/// our writes land past its end, in space the firmware never reads. The real +/// field is +0x26ac, written from `AGXQOSManager + 0x28`. The neighbouring +/// live fields are +0x2614 (the 4 x 0x20 KSM completion table, which we do +/// populate), +0x2694 (a 0x40-entry table fwva), +0x2604 (deferred-kick-mode +/// selector) and the flags at +0x2600/+0x2608/+0x2610. +/// +/// We keep writing 0x27a4 as well: if that offset really is dead space the +/// extra store is inert, and leaving it in place keeps this a single-variable +/// change. +pub(crate) const G17P_QOS_RESOURCE_LIVE_OFFSET: usize = 0x26ac; +pub(crate) const G17P_QOS_RESOURCE_SIZE: usize = 0xe40; +/// Native global SKSM/QID configuration block in hardware data. +pub(crate) const G17P_SKSM_QID_CONFIG_2700_OFFSET: usize = 0x2700; +pub(crate) const G17P_SKSM_QID_CONFIG_2708_OFFSET: usize = 0x2708; +pub(crate) const G17P_SKSM_QID_RESOURCE_OFFSET: usize = 0x27ac; +/// Host->firmware transport selector, `initdata[0x26f8] = accel[0x4f8] & 1` +/// (`AGXFirmware::initFirmwareData` @0xfffffe0008a20940). 1 = the host +/// publishes channel commands; 0 = the host drives the MMIO kick bridge. +const HW_DATA_TRANSPORT_SELECTOR_OFFSET: usize = 0x26f8; +/// MEASURED, live 25G83 hardware-data block: native reads 0 here. +/// initFirmwareData clears the u32 at +0x26fe, which covers +0x2700..+0x2701, +/// and never sets it. Our 1 was a non-native write inside the live object -- +/// the last non-perf difference in the whole 0x2708-byte structure. +pub(crate) const G17P_SKSM_QID_CONFIG_2700: u64 = 0; +pub(crate) const G17P_SKSM_QID_CONFIG_2708: u32 = 0x100; +/// Firmware log enable word inside main config. +pub(crate) const MAIN_FIRMWARE_LOG_ENABLE: usize = 0x250; + +pub(crate) const G17P_SKSM_QID_RESOURCE_SIZE: usize = 0x2000; +pub(crate) const REGION_A_SIZE: usize = 0x4000; +pub(crate) const REGION_C_SIZE: usize = 0x1000; +pub(crate) const STATUS_BLOCK_SIZE: usize = 0x80; + +/// The hardware-data bundle: one contiguous allocation with the hardware-data +/// object at its base. Its principal content spans `0xc000`, but the repeated +/// writable region at `+0xc500` (named by main `+0x08`/`+0x10`) lies past +/// that, so the allocation must reach at least `0x10000`. +pub(crate) const HW_DATA_BUNDLE_SIZE: usize = 0xc000; +pub(crate) const HW_DATA_BUNDLE_MIN_ALLOC: usize = 0x10000; +/// The native placement additionally hosts the twelve work rings and both main +/// configuration objects inside the same cluster, which then spans this much +/// (see [`WORK_RING_BUNDLE_OFFSETS`], [`NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET`]). +pub(crate) const NATIVE_SHARED_CLUSTER_SIZE: usize = 0x28000; +/// Firmware-high base of the complete hardware-data/main-config/work-ring +/// cluster in every output-positive G17P capture. +pub(crate) const NATIVE_SHARED_CLUSTER_GPU_VA: u64 = 0xffff_fc20_c078_8000; + +// --- Version tuple ------------------------------------------------------------ + +/// Root `+0x00`: four u16 version words, `{0x04c0, 0x0396, 0xa322, 0x0c8a}`. +/// Grounded twice: read out of every live T8140 root, and independently +/// re-derived from the MOVZ/MOVK chain at `__TEXT+0x1e8a8` of our own J700 +/// firmware image. A mismatch is rejected with "Host and FW Interface is +/// mismatched" (the M5/G17S image wants 0x0c8d_e339_0728_04c0 instead — that +/// value must never be sent to T8140 firmware). +pub(crate) const INITDATA_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; +/// The same tuple as the little-endian u64 the firmware compares in one go. +pub(crate) const INITDATA_VERSION_WORD: u64 = 0x0c8a_a322_0396_04c0; + +/// Target/build-specific root encoding. This is a serialization choice, not +/// permission to start firmware or reuse the other target's hardware tables. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RootProfile { + J700G17P, + /// J813 Mac17,3 / 25G76, RTKit-3255.160.4.release. Both roots were captured + /// before INBOX1 publication. Independently confirmed by the loaded GFX + /// version comparison at +0x1fa5c and GFX1 at +0x21140. + J813G17G25G76, + /// J873 Mac18,5 / T8152 / G18G / 26A428, RTKit_release-3514.0.15 + /// (armfw_g18g-b000 on GFX, armfw1_g18g-b010 on GFX1). No capture of a + /// native root exists; the encoding is source-constructed from the 26A428 + /// kext and firmware (see the "J873 / G18G" section below). Selecting it + /// changes nothing for the other two profiles. + J873G18G26A428, +} + +impl RootProfile { + pub(crate) const fn version_word(self) -> u64 { + match self { + Self::J700G17P => INITDATA_VERSION_WORD, + Self::J813G17G25G76 => 0x0c8a_a322_039e_04c0, + Self::J873G18G26A428 => G18G_26A428_INITDATA_VERSION_WORD, + } + } + + /// Bytes of the root object for `role`. J700/J813 keep the historical + /// 0xb8/0xc8 split. G18G sizes both roots to 0xc8 because both firmware + /// images copy 0xc8 bytes from each root (FW_PROVEN b000 0x21664-0x216a8). + pub(crate) const fn root_size(self, role: InstanceRole) -> usize { + match self { + Self::J873G18G26A428 => G18G_26A428_ROOT_SIZE, + Self::J700G17P | Self::J813G17G25G76 => root_size(role), + } + } + + /// Opcode of the one opening device-control record staged at main + /// `+0x4c0`. G18G renumbered the opening handler: 0x16 is now an + /// unrelated handler and 0x15 reaches the opening (FW_PROVEN jump table + /// 0xfffffc0000006c60, handler b000 0x3ed0). The secondary keeps 0x2a. + pub(crate) const fn control_opening_opcode(self, role: InstanceRole) -> u32 { + match (self, role) { + (Self::J873G18G26A428, InstanceRole::Primary) => { + G18G_26A428_CONTROL_OPENING_OPCODE_PRIMARY + } + (Self::J873G18G26A428, InstanceRole::Secondary) => { + G18G_26A428_CONTROL_OPENING_OPCODE_SECONDARY + } + (_, InstanceRole::Primary) => CONTROL_OPENING_OPCODE_PRIMARY, + (_, InstanceRole::Secondary) => CONTROL_OPENING_OPCODE_SECONDARY, + } + } + + /// The "primary register" device-control opcode (0x20 on J700/M5). + /// FW_PROVEN: G18G opcode 0x20 is in the invalid set of the 56-entry + /// jump table, so the profile has none and callers must not emit one. + pub(crate) const fn control_register_opcode(self) -> Option { + match self { + Self::J873G18G26A428 => None, + Self::J700G17P | Self::J813G17G25G76 => Some(CONTROL_OPENING_PRIMARY_REGISTER_OPCODE), + } + } + + /// Bundle offsets of the five views published at main `+0x254`. + pub(crate) const fn bundle_view_offsets(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => G18G_26A428_BUNDLE_VIEW_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_OFFSETS, + } + } + + /// Extents each view must fit inside the bundle allocation. + pub(crate) const fn bundle_view_extents(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => G18G_26A428_BUNDLE_VIEW_EXTENTS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_EXTENTS, + } + } + + /// Bundle offset named twice by main `+0x08`/`+0x10` (the empty BRN + /// table and the 0x88 block, which the allocator places together). + pub(crate) const fn repeated_region_bundle_offset(self) -> usize { + match self { + Self::J873G18G26A428 => G18G_26A428_SHARED_LAYOUT.block_88, + Self::J700G17P | Self::J813G17G25G76 => MAIN_REPEATED_REGION_BUNDLE_OFFSET, + } + } + + /// Bundle offsets of the twelve work rings in channel-table order. + pub(crate) const fn work_ring_bundle_offsets(self) -> [usize; WORK_CHANNEL_COUNT] { + match self { + Self::J873G18G26A428 => G18G_26A428_WORK_RING_BUNDLE_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => WORK_RING_BUNDLE_OFFSETS, + } + } +} + +// --- Address representability -------------------------------------------------- + +// The endpoint-0x20 doorbell carries the root address in a 44-bit field; the +// J700 firmware rebuilds it as `msg | 0xfffff0_0000_0000_0000`. Every pointer +// placed in the graph must therefore be either bare or carry exactly that +// kernel-half prefix, or it silently truncates. +const INITDATA_ADDR_MASK: u64 = 0x0000_0fff_ffff_ffff; +const INITDATA_ADDR_PREFIX: u64 = 0xffff_f000_0000_0000; + +// --- Root layout --------------------------------------------------------------- + +const ROOT_VERSION: usize = 0x00; +const ROOT_REGION_A: usize = 0x08; +/// Root `+0x10` is zero at handoff on both instances; semantics unknown. +const ROOT_ZERO_10: usize = 0x10; +const ROOT_MAIN_CONFIG: usize = 0x18; +const ROOT_REGION_C: usize = 0x20; +const ROOT_INSTANCE_KIND: usize = 0x28; +const ROOT_CONSTANT_2C: usize = 0x2c; +const ROOT_PAGE_SIZE: usize = 0x30; +const ROOT_PAGE_BITS: usize = 0x32; +const ROOT_LEVEL_COUNT: usize = 0x33; +const ROOT_LEVELS: usize = 0x34; +/// Root `+0xa8`: Status A pointer. (The M5-derived model misused this slot as +/// a role pointer.) +const ROOT_STATUS_A: usize = 0xa8; +/// Root `+0xb0`: Status B pointer. Real for the primary, zero for the +/// secondary. (The M5-derived model misused it as a cross-role pointer.) +const ROOT_STATUS_B: usize = 0xb0; +const ROOT_SECONDARY_EXTRA_0: usize = 0xb8; +const ROOT_SECONDARY_EXTRA_1: usize = 0xc0; + +// --- UAT geometry (root +0x30..+0x94) ----------------------------------------- + +pub(crate) const UAT_PAGE_SIZE: u16 = 0x4000; +pub(crate) const UAT_PAGE_BITS: u8 = 14; +pub(crate) const UAT_LEVEL_COUNT: usize = 3; +/// `(index_shift, entry_count)` for the three levels below the root, +/// confirmed against a live table walk. +pub(crate) const UAT_LEVELS: [(u8, u16); UAT_LEVEL_COUNT] = [(36, 64), (25, 2048), (14, 2048)]; +pub(crate) const UAT_LEVEL_DESC_SIZE: usize = 0x20; +/// Level descriptor `+0x10`: physical mask, identical in all three. +pub(crate) const UAT_LEVEL_PHYS_MASK: u64 = 0x0000_03ff_ffff_c000; + +// --- Main configuration layout -------------------------------------------------- + +const MAIN_HW_DATA: usize = 0x00; +/// The same repeated-region address appears at `+0x08` and `+0x10`; it is the +/// bundle base plus [`MAIN_REPEATED_REGION_BUNDLE_OFFSET`], not the bundle +/// base itself. It reads zero at handoff (firmware writes into it), so the +/// host only has to provide it mapped. +const MAIN_REPEATED_0: usize = 0x08; +const MAIN_REPEATED_1: usize = 0x10; +pub(crate) const MAIN_REPEATED_REGION_BUNDLE_OFFSET: usize = 0xc500; +const MAIN_CHANNEL_TABLE: usize = 0x20; +/// Main `+0x254`: five unaligned u64 views into the hardware-data bundle. +const MAIN_BUNDLE_VIEWS: usize = 0x254; +/// Main `+0x2d0`: six qwords of region views. +const MAIN_REGION_VIEWS: usize = 0x2d0; +/// Main `+0x300`: u32 4, secondary only. +const MAIN_SECONDARY_KIND: usize = 0x300; +const MAIN_SECONDARY_KIND_VALUE: u32 = 4; +/// Main `+0x3e0`: u32 0xff, primary only. +const MAIN_PRIMARY_MASK: usize = 0x3e0; +const MAIN_PRIMARY_MASK_VALUE: u32 = 0xff; +const MAIN_PRIMARY_SCALAR_32C: usize = 0x32c; +const MAIN_PRIMARY_SCALAR_344: usize = 0x344; +/// Main `+0x471`: unaligned u64, secondary only, pointing *into* bundle view 2 +/// at [`SECONDARY_EXTRA_VIEW_OFFSET`] (not a fresh allocation — the M5-derived +/// model had this at `+0x481` backed by an invented 0x12688-byte buffer; both +/// were wrong). +const MAIN_SECONDARY_EXTRA_PTR: usize = 0x471; +pub(crate) const SECONDARY_EXTRA_VIEW_INDEX: usize = 2; +pub(crate) const SECONDARY_EXTRA_VIEW_OFFSET: usize = 0xe40; +/// Main `+0x4c0` is both a scalar and a place: the leading u32 of the +/// device-control ring, which is embedded in the main configuration object at +/// exactly this offset on both instances. Writing the opening opcode here *is* +/// staging the opening device-control record (records are 0x40 bytes; fields +/// past the opcode are zero). The control channel's producer counter must then +/// be set to the role-specific value returned by +/// [`control_opening_producer`] before the descriptor is handed over. +const MAIN_CONTROL_RING: usize = 0x4c0; +pub(crate) const CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x16; +pub(crate) const CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; +pub(crate) const CONTROL_RECORD_SIZE: usize = 0x40; +pub(crate) const CONTROL_OPENING_PRIMARY_RECORDS: usize = 1; +pub(crate) const CONTROL_OPENING_SECONDARY_RECORDS: usize = 1; +pub(crate) const CONTROL_OPENING_PRIMARY_PRODUCER: u32 = 1; +pub(crate) const CONTROL_OPENING_SECONDARY_PRODUCER: u32 = 1; +pub(crate) const CONTROL_OPENING_PRIMARY_REGISTER_OPCODE: u32 = 0x20; +pub(crate) const CONTROL_OPENING_SECONDARY_CONTINUE_OPCODE: u32 = 0x22; +/// Hardware-buffer release emitted after the opening UMA grow request retires. +pub(crate) const CONTROL_BOOTSTRAP_UMA_RELEASE_OPCODE: u32 = 0x2e; +/// Scheduler-state (channel control) release. +/// +/// Decoded from AGXG17P 353.14 (25G83) `AGXArmFirmware::submitReleaseResource` +/// at `0xfffffe00088a722c`: the record's first word is `0x14` on this image +/// (the G17X image audited for the G17 native detach plan +/// carries `0x13`; the G17P opcode route table at `0xfffffe0007136180` sends +/// `0x14` to ring 0, the same primary control ring as `0x2e`, and `0x13` to +/// ring 1). `AGXCommandQueue::freeReclaimableResources` (`0xfffffe0008865354`) +/// submits it with the four identity bytes read back from the live scheduler +/// state page and then waits for the ring consumer to pass the record before +/// it frees the work queues and their parameter management. +pub(crate) const CONTROL_SCHEDULER_STATE_RELEASE_OPCODE: u32 = 0x14; +/// `AGXCommandQueue::processExitTimeoutHandler` (@fffffe000886a494): kill +/// the work of one scheduler-state page; acknowledged by firmware event 12 +/// (`kAGFIFirmwareEventTypeProcessExitComplete`) carrying the cookie. +pub(crate) const CONTROL_CONTEXT_KILL_OPCODE: u32 = 0x29; +/// Device-control ring capacity. FW_PROVEN (b1 +/// `AGFAcceleratorDrainDeviceControlRing` @ `0xfffffc0000003a74`): the drain +/// walks `ring + index * 0x40` and advances its consumer with +/// `(consumer + 1) & 0xff`, so the ring is exactly 256 records. The placement +/// agrees byte for byte: the primary ring starts at +/// [`NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET`] `+ 0x4c0` = bundle `+0x1ea80` and the +/// secondary main configuration begins at bundle `+0x22a80`, exactly +/// `0x100 * 0x40` bytes later. +pub(crate) const CONTROL_RING_ENTRIES: usize = 0x100; +pub(crate) const CONTROL_RING_INDEX_MASK: u32 = CONTROL_RING_ENTRIES as u32 - 1; +/// Device control `IdlePowerOff(arg)`. FW_PROVEN: the opcode jump table at +/// `0xfffffc0000005380` (indexed by `opcode - 4`) sends `0x0a` to +/// `0xfffffc0000003fd8`, which stores the record's `arg` into the idle-power-off +/// enable word `[0xfffffc0000177838]` and, **only when that arg is zero**, does +/// an unconditional `bl 0xfffffc0000021960`. That routine is the one thing in +/// the image that releases KSM pause reason bit 1 (mask `0x2`) in steady state: +/// it latches `[0xfffffc00001771f8] |= 2` and tail-calls +/// `gpu_core_power(0, 1, 0)` @ `0xfffffc000000e004`, whose power-on arm clears +/// the reason bit at `0xe488` and emits KTrace `0x46 (0, 0x2, mask)` at +/// `0xe4cc`. It is gated by nothing -- unlike the EP-0x21 work doorbell, which +/// handler `0x261a4` drops silently while `[0x1771f8] & 0x18` (RTKit +/// SLEEP/NAP) -- and it is idempotent, because `0x21960` returns immediately +/// when the powered latch is already set. +pub(crate) const DEVICE_CONTROL_OPCODE_IDLE_POWER_OFF: u32 = 0x0a; +/// Disable ordinary idle power-off and request that the GPU cores wake now. +/// +/// FW_PROVEN (25F80 B100, SHA256 83af9ae0...): opcode 0x0a stores this value +/// in `[0xfffffc0000177838]` and calls the wake path only for zero. The +/// scheduler's 0x20000000 idle event skips its descent while that policy word +/// is zero. A cold wake may reload the word from Region-C +0x30, so callers +/// which need a scoped hold must retire a second zero after the core is up. +pub(crate) const DEVICE_CONTROL_IDLE_POWER_OFF_ARG_DISABLE_IDLE_AND_WAKE: u32 = 0; +/// Restore the firmware's ordinary idle power-off policy. A non-zero value +/// permits the scheduler idle event to descend; it is not a power hold. +pub(crate) const DEVICE_CONTROL_IDLE_POWER_OFF_ARG_ENABLE_IDLE: u32 = 1; +/// Separate retained compute resources from the render pool. Native +/// populateInitialFirmwareState (8aa258c) publishes resource-derived +0x14 +/// page-list and +0x30 run-list addresses; neither is firmware-fixed. +pub(crate) const COMPUTE_FLIST_RELOCATION: u64 = 0x0200_0000; +pub(crate) const COMPUTE_FLIST_PAGE_LIST_ADDRESS: u64 = 0x70_0000_0000 + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_FLIST_RUN_TABLE_ADDRESS: u64 = CONTROL_OPERAND_TABLE_ADDRESS + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_FLIST_BUFFER_BASE: u64 = CONTROL_OPERAND_BUFFER_BASE + COMPUTE_FLIST_RELOCATION; + +pub(crate) const COMPUTE_RUNTIME_CONTROL_RECORDS: usize = 65; +pub(crate) const COMPUTE_RUNTIME_CONTROL_FINAL_COUNTER: u32 = + CONTROL_OPENING_PRIMARY_PRODUCER + COMPUTE_RUNTIME_CONTROL_RECORDS as u32; +pub(crate) const COMPUTE_RUNTIME_CLASS1_RECORD_INDEX: usize = 54; +pub(crate) const COMPUTE_RUNTIME_CLASS2_RECORD_INDEX: usize = 57; +pub(crate) const COMPUTE_RUNTIME_SUPPORT_ADDRESS: u64 = 0xffff_fc20_c086_8000; +pub(crate) const COMPUTE_RUNTIME_STATE_ADDRESS: u64 = 0xffff_fc20_0163_0000; +pub(crate) const COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS: u64 = 0x70_01ad_8000 + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_RUNTIME_PAGE_LIST_ADDRESS: u64 = 0x70_018d_0000 + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_RUNTIME_ZERO_BUFFER_0_ADDRESS: u64 = 0x70_0000_0000 + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_RUNTIME_ZERO_BUFFER_1_ADDRESS: u64 = 0x70_0010_8000 + COMPUTE_FLIST_RELOCATION; +pub(crate) const COMPUTE_RUNTIME_SUPPORT_SIZE: usize = 0x4000; +pub(crate) const COMPUTE_RUNTIME_CLASS1_TABLE_SIZE: usize = 0x4000; +pub(crate) const COMPUTE_RUNTIME_PAGE_LIST_SIZE: usize = 0xa800; +pub(crate) const COMPUTE_DISPATCH_RECORD_SIZE: usize = 0x20; + +/// Output-positive first-CL readiness suffix. These six records follow the +/// clean three-record QoS opening. Native publishes and retires each slot +/// before exposing the next. +pub(crate) const COMPUTE_READINESS_RECORDS: usize = 6; +/// Output-positive final compute graph maps and names exactly operand entries +/// 0..20. The retained opening still owns 28 backing objects. +pub(crate) const COMPUTE_READINESS_OPERAND_ENTRY_COUNT: usize = 21; +pub(crate) const COMPUTE_READINESS_FINAL_COUNTER: u32 = + CONTROL_OPENING_PRIMARY_PRODUCER + COMPUTE_READINESS_RECORDS as u32; +pub(crate) const COMPUTE_READINESS_CLASS1_SUPPORT_ADDRESS: u64 = 0xffff_fc20_c087_8000; +pub(crate) const COMPUTE_READINESS_CLASS1_STATE_ADDRESS: u64 = 0xffff_fc20_0164_8000; +pub(crate) const COMPUTE_READINESS_CLASS3_SUPPORT_ADDRESS: u64 = 0xffff_fc20_c085_0000; +pub(crate) const COMPUTE_READINESS_CLASS3_STATE_ADDRESS: u64 = 0xffff_fc20_0165_0000; +pub(crate) const COMPUTE_READINESS_PAGE_SIZE: usize = 0x4000; +pub(crate) const COMPUTE_READINESS_ACTIVATION_RECORD_INDEX: usize = 2; +pub(crate) const COMPUTE_READINESS_ACTIVATION_RECORD_OFFSET: usize = + COMPUTE_READINESS_ACTIVATION_RECORD_INDEX * CONTROL_RECORD_SIZE; + +/// One native first-CL device-control publication boundary. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeReadinessRecordBoundary { + pub(crate) index: u8, + pub(crate) producer_before: u32, + pub(crate) producer_after: u32, + pub(crate) opcode: u32, + pub(crate) sequence: u32, + pub(crate) context: u32, +} + +pub(crate) const COMPUTE_READINESS_BOUNDARIES: + [ComputeReadinessRecordBoundary; COMPUTE_READINESS_RECORDS] = [ + ComputeReadinessRecordBoundary { + index: 0, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 1, + opcode: 0x20, + sequence: 35, + context: 1, + }, + ComputeReadinessRecordBoundary { + index: 1, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER + 1, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 2, + opcode: 0x2e, + sequence: 35, + context: 1, + }, + ComputeReadinessRecordBoundary { + index: 2, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER + 2, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 3, + opcode: 0x2e, + sequence: 36, + context: 1, + }, + ComputeReadinessRecordBoundary { + index: 3, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER + 3, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 4, + opcode: 0x20, + sequence: 37, + context: 2, + }, + ComputeReadinessRecordBoundary { + index: 4, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER + 4, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 5, + opcode: 0x2e, + sequence: 37, + context: 2, + }, + ComputeReadinessRecordBoundary { + index: 5, + producer_before: CONTROL_OPENING_PRIMARY_PRODUCER + 5, + producer_after: CONTROL_OPENING_PRIMARY_PRODUCER + 6, + opcode: 0x2e, + sequence: 38, + context: 2, + }, +]; + +pub(crate) const fn compute_readiness_boundary( + index: usize, +) -> Option { + if index < COMPUTE_READINESS_RECORDS { + Some(COMPUTE_READINESS_BOUNDARIES[index]) + } else { + None + } +} + +/// Host progress for the native per-record publication handshake. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeReadinessProgress { + retired_records: u8, + pending_target: Option, +} + +impl ComputeReadinessProgress { + pub(crate) const fn new() -> Self { + Self { + retired_records: 0, + pending_target: None, + } + } + + pub(crate) const fn retired_records(self) -> usize { + self.retired_records as usize + } + + pub(crate) const fn pending_target(self) -> Option { + self.pending_target + } + + pub(crate) const fn next_boundary(self) -> Option { + if self.pending_target.is_some() { + None + } else { + compute_readiness_boundary(self.retired_records as usize) + } + } + + pub(crate) const fn begin_publication( + self, + boundary: ComputeReadinessRecordBoundary, + ) -> Option { + if self.pending_target.is_some() || boundary.index != self.retired_records { + return None; + } + Some(Self { + retired_records: self.retired_records, + pending_target: Some(boundary.producer_after), + }) + } + + pub(crate) const fn retire(self, target: u32) -> Option { + match self.pending_target { + Some(pending) if pending == target => Some(Self { + retired_records: self.retired_records + 1, + pending_target: None, + }), + _ => None, + } + } + + pub(crate) const fn complete(self) -> bool { + self.retired_records as usize == COMPUTE_READINESS_RECORDS + && self.pending_target.is_none() + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ComputeReadinessPublicationStep { + ControlRecord(u8), + InitializeContext2Activation, +} + +pub(crate) const COMPUTE_READINESS_PUBLICATION_ORDER: [ComputeReadinessPublicationStep; 7] = [ + ComputeReadinessPublicationStep::ControlRecord(0), + ComputeReadinessPublicationStep::ControlRecord(1), + ComputeReadinessPublicationStep::ControlRecord(2), + ComputeReadinessPublicationStep::InitializeContext2Activation, + ComputeReadinessPublicationStep::ControlRecord(3), + ComputeReadinessPublicationStep::ControlRecord(4), + ComputeReadinessPublicationStep::ControlRecord(5), +]; + +pub(crate) const CONTROL_SHARED_ADDRESS: u64 = 0xffff_fc20_c083_0000; +pub(crate) const CONTROL_SHARED_INNER_ADDRESS: u64 = 0xffff_fc20_0160_8000; +pub(crate) const CONTROL_SHARED_OBJECT_SIZE: usize = 0x4000; +pub(crate) const CONTROL_SHARED_CURSOR_OFFSET: usize = 0x48; +pub(crate) const CONTROL_SHARED_CURSOR_BEFORE: u32 = 0x88; +pub(crate) const CONTROL_SHARED_INNER_PRESENTED: u64 = 2; +/// Separate compact class-2 object consumed by the first-partial opening. +/// Descriptor tails and later compute control continue to use +/// [`CONTROL_SHARED_ADDRESS`]. +pub(crate) const PARTIAL_OPENING_CONTROL_SHARED_ADDRESS: u64 = 0xffff_fc20_c082_8000; +pub(crate) const PARTIAL_OPENING_CONTROL_SHARED_INNER_ADDRESS: u64 = + 0xffff_fc20_0160_0000; +/// Hardware-proven compact compute cursor. Twenty-one consecutive run qwords +/// occupy slots 0..20 and the next eight-qword boundary is 0x18. +pub(crate) const COMPUTE_PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE: u32 = 0x18; +/// Initial compact FList cursor in qwords. +/// +/// `AGXUSCPrivMemFList::prepareBufferResources` encodes one 1-MiB run for +/// each of the 17 initial pool blocks, rounding to an eight-qword slot after +/// each block. The first grow starts at qword 17 * 8 = 0x88 (byte +0x440), +/// exactly as the complete 25G83 first-render opcode-0x20 record publishes. +pub(crate) const PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE: u32 = 0x88; +pub(crate) const PARTIAL_OPENING_CONTROL_SHARED_INNER_PRESENTED: u64 = 0; +/// Linux owns one USC private-memory FList. HardwareBufferIDManager permits +/// slot zero (native's generic allocator pops 0 first); macOS observed ID 1 +/// only because another live FList already owned slot zero in that session. +pub(crate) const PARTIAL_OPENING_FREELIST_HARDWARE_BUFFER_ID: u32 = 0; +pub(crate) const CONTROL_OPERAND_TABLE_ADDRESS: u64 = 0x70_0020_8000; +pub(crate) const CONTROL_OPERAND_TABLE_SIZE: usize = 0x4000; +pub(crate) const CONTROL_OPERAND_SLOT_OFFSET: u64 = 0x440; +pub(crate) const PARTIAL_OPENING_CONTROL_OPERAND_SLOT_OFFSET: u64 = 0x640; +pub(crate) const PARTIAL_OPENING_CONTROL_REGISTER_COUNT: u32 = 0x18; +pub(crate) const PARTIAL_OPENING_CONTROL_OPERAND_ENTRY_COUNT: usize = 28; +pub(crate) const CONTROL_OPERAND_BUFFER_BASE: u64 = 0x70_0022_0000; +pub(crate) const CONTROL_OPERAND_BUFFER_STRIDE: u64 = 0x10_8000; +pub(crate) const CONTROL_OPERAND_BUFFER_SIZE: u64 = 0x10_0000; +pub(crate) const CONTROL_OPERAND_ENTRY_COUNT: usize = 22; +pub(crate) const CONTROL_OPERAND_ENTRY_STRIDE: usize = 0x40; +pub(crate) const CONTROL_OPERAND_ENTRY_FLAG: u64 = 1 << 60; +pub(crate) const COMPUTE_RUNTIME_BUFFER_ADDRESSES: [u64; 21] = [ + 0x70_0022_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_0032_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_0043_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_0053_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_0064_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_0074_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_0085_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_0095_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_00a6_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_00b6_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_00c7_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_00d7_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_00e8_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_00f8_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_0109_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_0119_8000 + COMPUTE_FLIST_RELOCATION, + 0x70_012a_0000 + COMPUTE_FLIST_RELOCATION, + 0x70_013a_8000 + COMPUTE_FLIST_RELOCATION, + COMPUTE_RUNTIME_ZERO_BUFFER_0_ADDRESS, + COMPUTE_RUNTIME_ZERO_BUFFER_1_ADDRESS, + 0x70_014b_0000 + COMPUTE_FLIST_RELOCATION, +]; + +pub(crate) const fn control_opening_producer(role: InstanceRole) -> u32 { + match role { + InstanceRole::Primary => CONTROL_OPENING_PRIMARY_PRODUCER, + InstanceRole::Secondary => CONTROL_OPENING_SECONDARY_PRODUCER, + } +} + +pub(crate) const fn control_opening_size(role: InstanceRole) -> usize { + match role { + InstanceRole::Primary => CONTROL_OPENING_PRIMARY_RECORDS * CONTROL_RECORD_SIZE, + InstanceRole::Secondary => CONTROL_OPENING_SECONDARY_RECORDS * CONTROL_RECORD_SIZE, + } +} + +fn encode_control_registration( + out: &mut [u8], + slot: usize, + control_class: u32, + sequence: u32, + object: u64, + operand: u64, + slot_offset: u64, + count: u32, + context_word: u32, +) { + let base = slot * CONTROL_RECORD_SIZE; + put_u32(out, base, CONTROL_OPENING_PRIMARY_REGISTER_OPCODE); + put_u32(out, base + 0x04, control_class); + put_u32(out, base + 0x08, 0x3f); + put_u32(out, base + 0x0c, sequence); + put_u64(out, base + 0x14, object); + put_u64(out, base + 0x1c, operand); + put_u64(out, base + 0x24, operand + slot_offset); + put_u32(out, base + 0x2c, count); + put_u32(out, base + 0x30, context_word); + put_u32(out, base + 0x34, 1); +} + +fn encode_control_tick(out: &mut [u8], slot: usize, sequence: u32) { + let base = slot * CONTROL_RECORD_SIZE; + put_u32(out, base, CONTROL_BOOTSTRAP_UMA_RELEASE_OPCODE); + put_u32(out, base + 0x04, sequence); +} + +/// Encode the exact release record produced when an Async FList grow request +/// drops its last HardwareBuffer ID reference. The captured opening uses +/// sequence zero and ID zero; accepting them as operands keeps the encoder +/// tied to the retained host ownership rather than to a raw type-13 tuple. +pub(crate) fn encode_bootstrap_uma_release_record( + sequence: u64, + hardware_buffer_id: u32, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, CONTROL_BOOTSTRAP_UMA_RELEASE_OPCODE); + put_u64(out, 0x04, sequence); + put_u32(out, 0x0c, hardware_buffer_id); + Ok(()) +} + +/// Encode the scheduler-state release record exactly as +/// `AGXArmFirmware::submitReleaseResource(va, a, b, c, d)` writes it into its +/// 64-byte device-control entry: opcode at `+0`, zero at `+4`, byte `d` at +/// `+8`, `a` at `+9`, `b` at `+0xa`, `c` at `+0xb`, and the resource's +/// firmware VA as an unaligned u64 at `+0xc`. `identity` is `[a, b, c, d]`, +/// i.e. bytes `[0]`, `[1]`, `[4]` and `[0x26]` of the live scheduler-state +/// page, as `freeReclaimableResources` loads them at `0xfffffe0008865398`. +pub(crate) fn encode_scheduler_state_release_record( + resource_va: u64, + identity: [u8; 4], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, CONTROL_SCHEDULER_STATE_RELEASE_OPCODE); + out[0x08] = identity[3]; + out[0x09] = identity[0]; + out[0x0a] = identity[1]; + out[0x0b] = identity[2]; + put_u64(out, 0x0c, resource_va); + Ok(()) +} + +/// 64-byte process-exit kill record exactly as `processExitTimeoutHandler` +/// builds it on its stack (record base sp+8): opcode at `+0`, the +/// scheduler-state page VA (queue+0x8b8) as an unaligned u64 at `+4`, and +/// the cookie as an unaligned u64 at `+0xc`; everything else zero. Firmware +/// 0x29 (fw 0x49cc) reads exactly `[x24,#4]` and `[x24,#0xc]`. +pub(crate) fn encode_context_kill_record( + scheduler_state_va: u64, + cookie: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, CONTROL_CONTEXT_KILL_OPCODE); + put_u64(out, 0x04, scheduler_state_va); + put_u64(out, 0x0c, cookie); + Ok(()) +} + +fn encode_control_tick_with_context( + out: &mut [u8], + slot: usize, + sequence: u32, + context_word: u32, +) { + encode_control_tick(out, slot, sequence); + let base = slot * CONTROL_RECORD_SIZE; + put_u32(out, base + 0x0c, context_word); +} + +/// Encode one of the six exact records immediately preceding the first CL. +pub(crate) fn encode_compute_readiness_record( + index: usize, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE || index >= COMPUTE_READINESS_RECORDS { + return Err(InitdataError::BufferSize); + } + out.fill(0); + match index { + 0 => encode_control_registration( + out, + 0, + 1, + 35, + COMPUTE_READINESS_CLASS1_SUPPORT_ADDRESS, + COMPUTE_FLIST_RUN_TABLE_ADDRESS, + 0x440, + 0x20, + 1, + ), + 1 => encode_control_tick_with_context(out, 0, 35, 1), + 2 => encode_control_tick_with_context(out, 0, 36, 1), + 3 => encode_control_registration( + out, + 0, + 3, + 37, + COMPUTE_READINESS_CLASS3_SUPPORT_ADDRESS, + COMPUTE_FLIST_RUN_TABLE_ADDRESS, + 0x5c0, + 0x28, + 2, + ), + 4 => encode_control_tick_with_context(out, 0, 37, 2), + 5 => encode_control_tick_with_context(out, 0, 38, 2), + _ => return Err(InitdataError::BufferSize), + } + Ok(()) +} + +fn encode_compute_readiness_support( + out: &mut [u8], + class: u32, + active: u32, + resource_class: u64, + cursor: u32, + state: u64, + final_kind: u32, +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_READINESS_PAGE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0x00, class); + put_u32(out, 0x08, active); + put_u32(out, 0x10, class); + put_u64(out, 0x14, COMPUTE_FLIST_PAGE_LIST_ADDRESS); + put_u32(out, 0x1c, 0x4_0000); + put_u64(out, 0x20, resource_class << 40); + put_u64(out, 0x28, resource_class << 40); + put_u64(out, 0x30, COMPUTE_FLIST_RUN_TABLE_ADDRESS); + put_u32(out, 0x40, 4); + put_u32(out, 0x48, cursor); + put_u64(out, 0x4c, state); + put_u32(out, 0x54, 0); + put_u32(out, 0x5c, 0); + put_u32(out, 0x60, final_kind); + Ok(()) +} + +pub(crate) fn encode_compute_readiness_class1_support( + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_compute_readiness_support( + out, + 1, + 1, + 0x11, + 0x88, + COMPUTE_READINESS_CLASS1_STATE_ADDRESS, + 2, + ) +} + +pub(crate) fn encode_compute_readiness_class3_support( + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_compute_readiness_support( + out, + 3, + 2, + 0x17, + 0xb8, + COMPUTE_READINESS_CLASS3_STATE_ADDRESS, + 3, + ) +} + +pub(crate) fn encode_compute_readiness_state( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_READINESS_PAGE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, 1); + Ok(()) +} + +pub(crate) fn encode_compute_readiness_activation_record( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u64(out, 0x00, 0x0000_0100_0000_ffff); + put_u64(out, 0x20, 0x0002_0000_0000_0000); + put_u64(out, 0x30, 0x0000_0000_ff00_0000); + Ok(()) +} + +pub(crate) fn encode_compute_readiness_operand_table( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_OPERAND_TABLE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for index in 0..COMPUTE_READINESS_OPERAND_ENTRY_COUNT { + let address = COMPUTE_FLIST_BUFFER_BASE + .checked_add(index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + .ok_or(InitdataError::AddressOutOfRange)?; + put_u64( + out, + index * CONTROL_OPERAND_ENTRY_STRIDE, + address | CONTROL_OPERAND_ENTRY_FLAG, + ); + } + Ok(()) +} + +/// Encode the complete control ring prefix consumed during the compute opening. +pub(crate) fn encode_control_opening( + role: InstanceRole, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != control_opening_size(role) { + return Err(InitdataError::BufferSize); + } + out.fill(0); + + match role { + InstanceRole::Primary => { + for slot in 0..CONTROL_OPENING_PRIMARY_RECORDS { + put_u32( + out, + slot * CONTROL_RECORD_SIZE, + CONTROL_OPENING_OPCODE_PRIMARY, + ); + } + } + InstanceRole::Secondary => { + let initial_records = core::cmp::min(3, CONTROL_OPENING_SECONDARY_RECORDS); + for slot in 0..initial_records { + put_u32( + out, + slot * CONTROL_RECORD_SIZE, + CONTROL_OPENING_OPCODE_SECONDARY, + ); + } + for slot in initial_records..CONTROL_OPENING_SECONDARY_RECORDS { + put_u32( + out, + slot * CONTROL_RECORD_SIZE, + CONTROL_OPENING_SECONDARY_CONTINUE_OPCODE, + ); + } + } + } + Ok(()) +} + +/// [`encode_control_opening`] for an explicit firmware build. J700/J813 are +/// byte-identical to [`encode_control_opening`]. G18G stages exactly one +/// record per role, 0x15 (primary) or 0x2a (secondary), and never a +/// continuation or register opcode: G18G 0x20 is invalid and 0x22 has not +/// been re-derived. +pub(crate) fn encode_control_opening_for_profile( + profile: RootProfile, + role: InstanceRole, + out: &mut [u8], +) -> Result<(), InitdataError> { + if profile != RootProfile::J873G18G26A428 { + return encode_control_opening(role, out); + } + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, profile.control_opening_opcode(role)); + Ok(()) +} + +/// Encode one bare device-control record: `u32 opcode @ +0x00`, +/// `u32 arg @ +0x04`, and 0x38 zero bytes. +/// +/// FW_PROVEN (b1 `0x3a74` drain): a record is 0x40 bytes at +/// `ring + index * 0x40`; the drain loads the opcode with `ldr w?, [x24]` and +/// the argument with `ldr w8, [x24, #4]`. Every opcode that carries no further +/// operands leaves the remaining 0x38 bytes untouched, exactly as the opening +/// records this module already stages do. +pub(crate) fn encode_device_control_record( + opcode: u32, + arg: u32, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0x00, opcode); + put_u32(out, 0x04, arg); + Ok(()) +} + +/// Encode one record in the output-positive 65-record runtime control tail. +/// Record zero follows the clean three-record QoS opening. +pub(crate) fn encode_compute_runtime_control_record( + index: usize, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_RECORD_SIZE || index >= COMPUTE_RUNTIME_CONTROL_RECORDS { + return Err(InitdataError::BufferSize); + } + out.fill(0); + + match index { + 0..=53 => encode_control_tick(out, 0, index as u32), + COMPUTE_RUNTIME_CLASS1_RECORD_INDEX => encode_control_registration( + out, + 0, + 1, + 54, + COMPUTE_RUNTIME_SUPPORT_ADDRESS, + COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS, + 0x480, + 0x18, + 0, + ), + 55..=56 => encode_control_tick(out, 0, index as u32 - 1), + COMPUTE_RUNTIME_CLASS2_RECORD_INDEX => encode_control_registration( + out, + 0, + 2, + 56, + COMPUTE_RUNTIME_SUPPORT_ADDRESS, + COMPUTE_FLIST_RUN_TABLE_ADDRESS, + 0x5c0, + 0x28, + 0, + ), + 58..=64 => encode_control_tick(out, 0, index as u32 - 2), + _ => return Err(InitdataError::BufferSize), + } + Ok(()) +} + +fn encode_compute_runtime_support( + out: &mut [u8], + control_class: u32, + low_buffer: u64, + operand_table: u64, + resource_class: u64, + cursor: u32, + final_kind: u32, +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_RUNTIME_SUPPORT_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0x00, control_class); + put_u32(out, 0x10, control_class); + put_u64(out, 0x14, low_buffer); + put_u32(out, 0x1c, 0x4_0000); + let resource = resource_class << 40; + put_u64(out, 0x20, resource); + put_u64(out, 0x28, resource); + put_u64(out, 0x30, operand_table); + put_u32(out, 0x40, 4); + put_u32(out, 0x48, cursor); + put_u64(out, 0x4c, COMPUTE_RUNTIME_STATE_ADDRESS); + put_u32(out, 0x60, final_kind); + Ok(()) +} + +pub(crate) fn encode_compute_runtime_class1_support( + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_compute_runtime_support( + out, + 1, + COMPUTE_RUNTIME_PAGE_LIST_ADDRESS, + COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS, + 0x12, + 0x90, + 2, + ) +} + +pub(crate) fn encode_compute_runtime_class2_support( + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_compute_runtime_support(out, 2, COMPUTE_FLIST_PAGE_LIST_ADDRESS, COMPUTE_FLIST_RUN_TABLE_ADDRESS, 0x17, 0xb8, 3) +} + +pub(crate) fn encode_compute_runtime_class1_table( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_RUNTIME_CLASS1_TABLE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for (index, address) in COMPUTE_RUNTIME_BUFFER_ADDRESSES.into_iter().enumerate() { + put_u64( + out, + index * CONTROL_OPERAND_ENTRY_STRIDE, + address | CONTROL_OPERAND_ENTRY_FLAG, + ); + } + Ok(()) +} + +pub(crate) fn encode_compute_runtime_page_inventory( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_RUNTIME_PAGE_LIST_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + let mut index = 0usize; + for base in COMPUTE_RUNTIME_BUFFER_ADDRESSES { + for offset in (0..CONTROL_OPERAND_BUFFER_SIZE).step_by(0x1000) { + put_u64(out, index * 8, base + offset); + index += 1; + } + } + if index * 8 != COMPUTE_RUNTIME_PAGE_LIST_SIZE { + return Err(InitdataError::BufferSize); + } + Ok(()) +} + +/// Populate the 21-block pool advertised by the compute UMA descriptor. +/// Like render's initial page list, this contains 4-KiB addresses, not compact +/// run descriptors. Publishing nonzero UMA counters over a zero-filled list +/// makes the allocator return address zero on its first consumption. +pub(crate) fn encode_compute_flist_page_list(out: &mut [u8]) -> Result<(), InitdataError> { + let pages_per_buffer = (CONTROL_OPERAND_BUFFER_SIZE / 0x1000) as usize; + let count = COMPUTE_READINESS_OPERAND_ENTRY_COUNT * pages_per_buffer; + if out.len() < count * 8 { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for buffer in 0..COMPUTE_READINESS_OPERAND_ENTRY_COUNT { + let base = COMPUTE_FLIST_BUFFER_BASE + buffer as u64 * CONTROL_OPERAND_BUFFER_STRIDE; + for page in 0..pages_per_buffer { + put_u64(out, (buffer * pages_per_buffer + page) * 8, base + page as u64 * 0x1000); + } + } + Ok(()) +} + +pub(crate) fn encode_compute_dispatch_record( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != COMPUTE_DISPATCH_RECORD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + // halLoadUMADesc packs the page-list VA in units of 128 bytes below + // unit_pages << 41. The native literal names render's unrelocated pool; + // slot 1 must instead name the independently backed compute page list. + put_u64(out, 0x00, (0x4_0000u64 << 41) | (COMPUTE_FLIST_PAGE_LIST_ADDRESS >> 7)); + put_u64(out, 0x08, 0x1500u64 << 33); + put_u64(out, 0x10, 0x1500); + Ok(()) +} + +/// Encode the generic shared object named by descriptor tails and later +/// compute-control records. +pub(crate) fn encode_control_shared(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != CONTROL_SHARED_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for (offset, value) in [ + (0x00, 0x0000_0000_0000_0001), + (0x10, 0x0000_0000_0000_0001), + (0x18, 0x0004_0000_0000_0070), + (0x20, 0x0000_1100_0000_0000), + (0x28, 0x0000_1100_0000_0000), + (0x30, CONTROL_OPERAND_TABLE_ADDRESS), + (0x40, 0x0000_0000_0000_0004), + (0x60, 0x0000_0000_0000_0002), + ] { + put_u64(out, offset, value); + } + put_u32(out, CONTROL_SHARED_CURSOR_OFFSET, CONTROL_SHARED_CURSOR_BEFORE); + put_u64(out, 0x4c, CONTROL_SHARED_INNER_ADDRESS); + Ok(()) +} + +/// Encode the distinct first-partial support object before opcode 0x20. +pub(crate) fn encode_partial_opening_control_shared( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_SHARED_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + // AGXUSCPrivMemFList::populateInitialFirmwareState seeds this hardware + // buffer id with -1, but AGXUSCPrivMemPool::prepareLocked replaces it with + // the allocated id before publishing the first tag-15 record. Linux owns + // the cold-opening slot zero statically, so encode the post-allocation + // state that the firmware actually receives. + put_u32( + out, + 0x08, + PARTIAL_OPENING_FREELIST_HARDWARE_BUFFER_ID, + ); + for (offset, value) in [ + (0x00, 0x0000_0000_0000_0001), + (0x10, 0x0000_0000_0000_0002), + (0x18, 0x0004_0000_0000_0070), + // The complete first-render trace has 17 initial one-MiB blocks: + // 17 MiB / 4 KiB = 0x1100 pages. Opcode 0x20 appends five blocks. + (0x20, 0x0000_1100_0000_0000), + (0x28, 0x0000_1100_0000_0000), + (0x30, CONTROL_OPERAND_TABLE_ADDRESS), + (0x40, 0x0000_0000_0000_0004), + (0x60, 0x0000_0000_0000_0003), + ] { + put_u64(out, offset, value); + } + put_u32( + out, + CONTROL_SHARED_CURSOR_OFFSET, + PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE, + ); + put_u64( + out, + 0x4c, + PARTIAL_OPENING_CONTROL_SHARED_INNER_ADDRESS, + ); + Ok(()) +} + +/// Encode the live render FList state at the native c0830000 owner address. +/// +/// The page-pool fields are the same prepared 17-block plus first-grow state +/// used by the first-partial encoder, but +0 is a pointer to the distinct +/// c01608000 inner object. Valid cold opcode-0x20, both render tag-15 records, +/// and descriptor +0x21ce all name this one object; c0828000 remains the +/// separate compact compute/first-partial state. +pub(crate) fn encode_render_control_shared(out: &mut [u8]) -> Result<(), InitdataError> { + encode_partial_opening_control_shared(out)?; + put_u64(out, 0x00, CONTROL_SHARED_INNER_ADDRESS); + put_u64(out, 0x4c, CONTROL_SHARED_INNER_ADDRESS); + Ok(()) +} + +/// Encode a per-pair render FList whose whole pool is populated up front. +/// +/// Same object as `encode_render_control_shared`, but the page counts at +/// +0x24/+0x2c and the run cursor at +0x48 cover every block, as +/// AGXUSCPrivMemFList::populateInitialFirmwareState writes them for Apple's +/// 3D pool (0x1600 pages, cursor 0xb0 for 22 blocks). Nothing is left for a +/// first opcode-0x20 grow to append. +pub(crate) fn encode_presized_render_control_shared( + out: &mut [u8], + blocks: u32, +) -> Result<(), InitdataError> { + let pages = blocks.checked_mul(0x100).ok_or(InitdataError::BufferSize)?; + encode_render_control_shared(out)?; + put_u64(out, 0x20, u64::from(pages) << 32); + put_u64(out, 0x28, u64::from(pages) << 32); + put_u32(out, CONTROL_SHARED_CURSOR_OFFSET, blocks * 8); + Ok(()) +} + +/// Encode the hardware-proven compute form of the compact class-2 object. +/// +/// This is deliberately separate from the render UMA form above. Compute's +/// successful opening uses 21 compact consecutive run descriptors, a cursor of +/// 0x18 and the original unallocated HardwareBufferID sentinel. Substituting +/// render's 22-block plus asynchronous-grow state makes compute retire without +/// executing. +pub(crate) fn encode_compute_partial_opening_control_shared( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_SHARED_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0x08, u32::MAX); + for (offset, value) in [ + (0x00, 0x0000_0000_0000_0001), + (0x10, 0x0000_0000_0000_0002), + (0x18, 0x0004_0000_0000_0070), + (0x20, 0x0000_1900_0000_0000), + (0x28, 0x0000_1900_0000_0000), + (0x30, COMPUTE_FLIST_RUN_TABLE_ADDRESS), + (0x40, 0x0000_0000_0000_0004), + (0x60, 0x0000_0000_0000_0003), + ] { + put_u64(out, offset, value); + } + put_u64(out, 0x14, COMPUTE_FLIST_PAGE_LIST_ADDRESS); + put_u32( + out, + CONTROL_SHARED_CURSOR_OFFSET, + COMPUTE_PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE, + ); + put_u64( + out, + 0x4c, + PARTIAL_OPENING_CONTROL_SHARED_INNER_ADDRESS, + ); + Ok(()) +} + +/// Keep the first-partial operand page blank through its opening and first +/// control-done notification. The host publishes the 28-entry form before +/// making either first-work producer visible. +pub(crate) fn encode_partial_opening_operand_table_pre_control( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_OPERAND_TABLE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + Ok(()) +} + +pub(crate) fn encode_partial_opening_operand_table_post_control( + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != CONTROL_OPERAND_TABLE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for index in 0..PARTIAL_OPENING_CONTROL_OPERAND_ENTRY_COUNT { + let address = CONTROL_OPERAND_BUFFER_BASE + .checked_add(index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + .ok_or(InitdataError::AddressOutOfRange)?; + put_u64( + out, + index * CONTROL_OPERAND_ENTRY_STRIDE, + address | CONTROL_OPERAND_ENTRY_FLAG, + ); + } + Ok(()) +} + +/// Encode the 22 flagged buffer pointers used by the later compute tail. +pub(crate) fn encode_control_operand_table(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != CONTROL_OPERAND_TABLE_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for index in 0..CONTROL_OPERAND_ENTRY_COUNT { + let address = COMPUTE_FLIST_BUFFER_BASE + .checked_add(index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + .ok_or(InitdataError::AddressOutOfRange)?; + put_u64( + out, + index * CONTROL_OPERAND_ENTRY_STRIDE, + address | CONTROL_OPERAND_ENTRY_FLAG, + ); + } + Ok(()) +} + +/// Bundle offsets of the five views named at main `+0x254`. They are views of +/// the common bundle, not independent objects: view 0 overlaps the tail of +/// the hardware-data object, and views 3 and 4 overlap each other by all but +/// 0x80 bytes. +pub(crate) const BUNDLE_VIEW_OFFSETS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; +/// Logical extents of the five views. +pub(crate) const BUNDLE_VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; + +/// The context-0 mappings of the PB descriptor and UMA page-pool descriptor +/// tables published at main `+0x2d8` and `+0x2e8`, respectively. The matching +/// firmware-high mappings are published at `+0x2e0` and `+0x2f0`. +pub(crate) const REGION_VIEW_LOW_ADDRS: [u64; 2] = [0x70_0183_8000, 0x70_0184_0000]; + +// --- Channel table --------------------------------------------------------------- + +pub(crate) const CHANNEL_TABLE_ENTRIES: usize = 17; +pub(crate) const CHANNEL_ENTRY_SIZE: usize = 0x20; +/// Channel entry layout: three *separate* 32-bit state-counter addresses at +/// `+0x00`/`+0x08`/`+0x10` (the counters themselves sit at 0x10 spacing in +/// their state block — do not collapse them into one state object), then the +/// ring base at `+0x18`. +pub(crate) const CHANNEL_ENTRY_STATE_COUNT: usize = 3; +pub(crate) const CHANNEL_STATE_SPACING: usize = 0x10; +pub(crate) const CHANNEL_STATE_STRIDE: usize = 0x40; +/// For a work channel, state 0 is the consumer and state 2 the producer. +pub(crate) const CHANNEL_STATE_CONSUMER: usize = 0; +pub(crate) const CHANNEL_STATE_PRODUCER: usize = 2; +/// Entries 0..11 are ordered TA_0, 3D_0, CL_0, TA_1, ... CL_3. +pub(crate) const WORK_CHANNEL_COUNT: usize = 12; +/// Entry 12 is device control. It continues the compact control-state grid +/// ([`WORK_STATE_GRID_OFFSETS`] has 13 entries, one more than there are work +/// channels), and its ring is embedded at main `+0x4c0`. The M5-derived model +/// refused this index, claiming a "cursor pair at +0x1a0/+0x1a8"; that was a +/// misread — those offsets are exactly this entry's state-counter addresses 0 +/// and 1. +pub(crate) const CONTROL_CHANNEL_INDEX: usize = 12; +/// Entries 13 and 14 are report channels with split state objects: their +/// state addresses alias off status A rather than the compact grid, and the +/// firmware peer of each host-owned counter is at `+0x20` within its object. +pub(crate) const REPORT_CHANNEL_INDICES: [usize; 2] = [13, 14]; +/// Entry 15 is partial: only its first state address, no ring. Entry 16 is +/// empty. Filling them like the rest declares channels that do not exist. +pub(crate) const PARTIAL_CHANNEL_INDEX: usize = 15; + +/// Offsets of the channel state blocks (channels 0..=12) inside the compact +/// state grid. Native state is *not* in channel-table order: within each +/// workload group the work queues sit at descending offsets, and the control +/// channel's block follows at `+0x300`. Preserving these address identities is +/// part of the descriptor ABI. +pub(crate) const WORK_STATE_GRID_OFFSETS: [usize; 13] = [ + 0x0c0, 0x1c0, 0x2c0, // TA_0, 3D_0, CL_0 + 0x080, 0x180, 0x280, // TA_1, 3D_1, CL_1 + 0x040, 0x140, 0x240, // TA_2, 3D_2, CL_2 + 0x000, 0x100, 0x200, // TA_3, 3D_3, CL_3 + 0x300, // device control +]; +/// The primary root's status-B pointer is the word immediately after that +/// compact grid: `status_b == state_grid + 0x340` in the native layout. +pub(crate) const PRIMARY_STATUS_B_STATE_GRID_OFFSET: usize = 0x340; +/// The primary status-B object runs from there to status A: `0xeb00` bytes. +pub(crate) const PRIMARY_STATUS_B_OBJECT_SIZE: usize = 0xeb00; + +/// State-counter addresses of report channels 13/14, as offsets from status A. +pub(crate) const REPORT_STATE_STATUS_A_OFFSETS: [[usize; 3]; 2] = + [[0x00040, 0x002c0, 0x00080], [0x00240, 0xa6ac0, 0x00280]]; +/// Ring bases of report channels 13/14, as offsets from status A. +pub(crate) const REPORT_RING_STATUS_A_OFFSETS: [usize; 2] = [0x04ac0, 0xafac0]; +/// Entry 15's sole state address, as an offset from status A. +pub(crate) const PARTIAL_STATE_STATUS_A_OFFSET: usize = 0x2d2c0; + +/// Native bundle offsets of the twelve work rings (0x1800 each), in +/// channel-table order. These extend past [`HW_DATA_BUNDLE_MIN_ALLOC`]; a host +/// using this placement sizes the cluster [`NATIVE_SHARED_CLUSTER_SIZE`]. +pub(crate) const WORK_RING_BUNDLE_OFFSETS: [usize; 12] = [ + 0x10dc0, 0x16dc0, 0x1cdc0, 0x0f5c0, 0x155c0, 0x1b5c0, 0x0ddc0, 0x13dc0, 0x19dc0, 0x0c5c0, + 0x125c0, 0x185c0, +]; +/// Native bundle offsets of the two main configuration objects. +pub(crate) const NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET: usize = 0x1e5c0; +pub(crate) const NATIVE_SECONDARY_MAIN_BUNDLE_OFFSET: usize = 0x22a80; + +// --- Hardware-data object --------------------------------------------------------- + +/// Register-mapping array: 53 slots of 0x28 bytes at `+0x640`. Slots may also +/// carry only the flag at `+0x20` (declared but unmapped). Register windows +/// come from the ADT; do not replace ADT-defined sizes or addresses. +/// The five qwords `AGXFirmware::initFirmwareData` writes at the head of the +/// hardware-data object: `{range-3 base, range-3 size, GART window base, GART +/// window base, reserved top-of-range-5 page}`. +pub(crate) const HW_FIRMWARE_RANGE_HEAD: &[(usize, u64)] = &[ + (0x00, 0x0000_006f_0000_0000), + (0x08, 0x0000_0000_ffc0_0000), + (0x10, 0x0000_0010_0000_0000), + (0x18, 0x0000_0010_0000_0000), + (0x20, 0x0000_02ff_ffff_8000), +]; + +pub(crate) const HW_REGISTER_MAP_OFFSET: usize = 0x640; +pub(crate) const HW_REGISTER_MAP_ENTRY_SIZE: usize = 0x28; +pub(crate) const HW_REGISTER_MAP_SLOTS: usize = 53; +/// FW device-VA aperture the register windows live in. +pub(crate) const HW_REGISTER_APERTURE: Range = 0xffff_fc21_8000_0000..0xffff_fc21_9000_0000; + +/// `+0xe90`: u32 chip identifier. +pub(crate) const HW_CHIP_ID_OFFSET: usize = 0xe90; +pub(crate) const HW_CHIP_ID_T8140: u32 = 0x8140; + +// The acceptance block at +0x258e. Bisecting by zeroing showed every *other* +// unexplained hardware-data byte can be zero, so this is the part that has to +// be right for the descriptor to be accepted. It decodes as the work-channel +// count, two {0,1,1,1} flag groups, one 0xffffffff "unassigned" word per work +// channel, and three trailing ones. +pub(crate) const HW_ACCEPT_CHANNEL_COUNT: usize = 0x258e; // u16 = 12 +pub(crate) const HW_ACCEPT_FLAG_GROUPS: [usize; 2] = [0x2590, 0x25a0]; +pub(crate) const HW_ACCEPT_FLAG_PATTERN: [u32; 4] = [0, 1, 1, 1]; +pub(crate) const HW_ACCEPT_UNASSIGNED_TABLE: usize = 0x25b4; // 12 x u32 0xffffffff +pub(crate) const HW_ACCEPT_TRAILING_ONES: [usize; 3] = [0x25f4, 0x2600, 0x2608]; +/// `+0x2630` is firmware state, not host configuration (the one byte that +/// differed between a pre-start and a running capture). It must start zero. +pub(crate) const HW_FIRMWARE_STATE_OFFSET: usize = 0x2630; + +/// Four inline G17P KSM completion descriptors at hardware-data `+0x2614`. +/// Only ordinals 0 and 2 are present. Each names low/high mappings of one +/// retained `0x800 * 0x40 == 0x20000` backing; ordinals 1 and 3 stay zero. +pub(crate) const G17P_KSM_COMPLETION_TABLE_OFFSET: usize = 0x2614; +pub(crate) const G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE: usize = 0x20; +pub(crate) const G17P_KSM_COMPLETION_PRESENT_ORDINALS: [u32; 2] = [0, 2]; +pub(crate) const G17P_KSM_COMPLETION_CAPACITY: u32 = 0x800; +pub(crate) const G17P_KSM_COMPLETION_ENTRY_SIZE: u32 = 0x40; +pub(crate) const G17P_KSM_COMPLETION_BACKING_SIZE: usize = + G17P_KSM_COMPLETION_CAPACITY as usize * G17P_KSM_COMPLETION_ENTRY_SIZE as usize; +pub(crate) const G17P_KSM_COMPLETION_LOW_VAS: [u64; 2] = + [0x70_03f0_0000, 0x70_03f2_8000]; +pub(crate) const G17P_KSM_COMPLETION_HIGH_BUNDLE_OFFSETS: [usize; 2] = [0x48000, 0x70000]; +const G17P_KSM_COMPLETION_TABLE_PREFIX_OFFSET: usize = 0x2610; +const G17P_KSM_COMPLETION_TABLE_PREFIX: u32 = 0x100; + +/// Two GPU mappings of one present completion-queue backing. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmCompletionAliases { + pub(crate) low: u64, + pub(crate) high: u64, +} + +pub(crate) const M5_25G76_COMPLETION_QUEUE_COUNT: usize = 4; +pub(crate) const M5_25G76_COMPLETION_BACKING_SIZE: usize = 0x800 * 0x40; + +/// M5 publishes all four descriptors at hardware-data +0x2614, with ordinals +/// 0,1,2,3. Each names low/high mappings of its own retained 128-KiB backing. +/// The two native pre-handoff captures agree on every descriptor byte. +/// Construction only: this does not allocate, map, publish, or poll a queue. +pub(crate) fn encode_m5_25g76_completion_table( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_COMPLETION_QUEUE_COUNT * 0x20 { + return Err(InitdataError::BufferSize); + } + let extent = M5_25G76_COMPLETION_BACKING_SIZE as u64; + // Validate every range before mutating the output. Different queues must + // not alias each other in either address space. + for (i, alias) in aliases.iter().enumerate() { + for address in [alias.low, alias.high] { + required_address(address)?; + let end = address.checked_add(extent).ok_or(InitdataError::AddressOutOfRange)?; + required_address(end - 1)?; + } + for prior in &aliases[..i] { + if (alias.low < prior.low + extent && prior.low < alias.low + extent) + || (alias.high < prior.high + extent && prior.high < alias.high + extent) + { + return Err(InitdataError::PairAliasedObject); + } + } + } + out.fill(0); + for (ordinal, alias) in aliases.iter().enumerate() { + let offset = ordinal * 0x20; + put_u64(out, offset, alias.low); + put_u64(out, offset + 8, alias.high); + put_u32(out, offset + 0x10, 0x800); + put_u32(out, offset + 0x14, 0x40); + put_u32(out, offset + 0x18, ordinal as u32); + } + Ok(()) +} + +// Performance-table geometry (values deliberately not encoded; see +// [`PERF_TABLE_VALUES_ENCODED`]). Two groups with identical internal layout, +// one per 11-entry ladder: frequency ladder, then a core-voltage column 0x40 +// on, then a memory-voltage column 0x440 on, stored as 0x40-stride per-state +// blocks. Group 0 repeats each per-state value across 16 words; group 1 +// stores it once. +pub(crate) const HW_PERF_GROUP_BASES: [usize; 2] = [0xfc8, 0x1cdc]; +pub(crate) const HW_PERF_GROUP_REPEATS: [usize; 2] = [16, 1]; +pub(crate) const HW_PERF_LADDER_ENTRIES: usize = 11; +pub(crate) const HW_PERF_VOLTAGE_DELTA: usize = 0x40; +pub(crate) const HW_PERF_MEMORY_VOLTAGE_DELTA: usize = 0x440; +pub(crate) const HW_PERF_STATE_BLOCK_STRIDE: usize = 0x40; +pub(crate) const HW_PERF_FREQ_LADDER_B: usize = 0x1808; +pub(crate) const HW_PERF_SCALE_LADDER_B: usize = 0x1848; +pub(crate) const HW_PERF_RELATIVE_LADDER_A: usize = 0x18c8; +pub(crate) const HW_PERF_RELATIVE_LADDER_B: usize = 0x1908; +pub(crate) const HW_PERF_INDEX_MAP_A: usize = 0x19c8; +pub(crate) const HW_PERF_INDEX_MAP_B: usize = 0x1a08; +const HW_PERF_FREQ_A: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 338, 492, 618, 796, 928, 1056, 1170, 1278, 1338, 1470]; +const HW_PERF_FREQ_B: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 338, 492, 618, 796, 928, 952, 1053, 1152, 1204, 1326]; +const HW_PERF_SCALE_B: [u32; HW_PERF_LADDER_ENTRIES] = [1_065_520_988; 11]; +/// Relative-performance ladder A. Native's live block reads +/// 0f 16 1d 2a 38 4a 56 at +0x18cc (states 1..7) = 15 22 29 42 56 74 86, +/// against our 16 23 30 43 57 74 85 -- off by one nearly everywhere. States +/// 8..10 are not covered by that read and keep our values. +const HW_PERF_RELATIVE_A_VALUES: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 15, 22, 29, 42, 56, 74, 86, 90, 92, 100]; +const HW_PERF_RELATIVE_B_VALUES: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 0, 13, 24, 40, 52, 63, 73, 83, 88, 100]; +const HW_PERF_INDEX_A_VALUES: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 1, 2, 3, 4, 5, 7, 9, 11, 13, 15]; +const HW_PERF_INDEX_B_VALUES: [u32; HW_PERF_LADDER_ENTRIES] = + [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 14]; +/// Per-perf-state GPU core voltage, in mV. +/// +/// MEASURED against native's live hardware-data block (25G83, hwdata.d reading +/// *(AGXArmFirmware+0x2b8)): native carries +/// 605 630 650 695 745 800 845 885 905 975 +/// for states 1..10, and this driver derived +/// 595 615 635 685 730 780 815 865 885 955 +/// -- systematically LOW at every state, by 10 to 30 mV. Both machines run the +/// same ADT, so this is our derivation and not chip binning. +/// +/// This is not cosmetic. The register this tree calls "GPC core power", +/// sgx+0xe01000 & 0xf, is G16::AGXFenderCommon::getGPCPerformanceState -- the +/// GPU PERFORMANCE state, where 0 means off. The firmware selects a state from +/// exactly these tables and raises the GPC to it, so a table it judges invalid +/// or below its floor is a direct mechanism for "firmware runs, tables loaded, +/// GPC stays 0, nothing ever executes". +const HW_PERF_CORE_VOLTAGE: [u32; HW_PERF_LADDER_ENTRIES] = + [125, 605, 630, 650, 695, 745, 800, 845, 885, 905, 975]; +/// Per-perf-state SRAM/memory-rail voltage, in mV. +/// +/// Same defect as the core table: our tail tracked our own (wrong) core values. +/// Native's live block carries 765 x6 then 800 845 885 905 975, which is the +/// ADT perf-states-sram column mapped through HW_PERF_INDEX_A. The correct +/// derivation is memory[i] = ADT perf-states-sram[INDEX_A[i]].volt, not "the +/// memory tail tracks core". +const HW_PERF_MEMORY_VOLTAGE: [u32; HW_PERF_LADDER_ENTRIES] = + [765, 765, 765, 765, 765, 765, 800, 845, 885, 905, 975]; + +// --- Region C --------------------------------------------------------------------- + +/// Byte immediately preceding the unaligned host power-assert counter. +pub(crate) const REGION_C_REQUIRED_OFFSET: usize = 0xe50; +pub(crate) const REGION_C_REQUIRED_VALUE: u32 = 0; +/// Unaligned host-owned GPU-core power-assert counter. +/// +/// FW_PROVEN (b1 `0x3aa8`/`0x3b20`): the firmware reads this **unaligned** u32 +/// and compares it with its own tally `status_b[0xe438]`, which it increments +/// once per drained record whose opcode flag byte (`__const 0x45430 + opcode`) +/// is non-zero. When the GPU is unpowered and the two disagree, `0x3b2c` calls +/// `0x21960` and powers the cores on before draining. Equal values mean "no +/// unaccounted power-asserting work", so a value pinned forever at 1 — the old +/// static initializer — can never trigger that arm after the first flagged +/// record. +/// +/// Was `0xe55`, which is off by four: that offset is read nowhere in b1. +/// The flag byte is `0` (i.e. no power assert) for opcodes `0x00`, `0x0e`, +/// `0x15`, `0x16`, `0x1b`, `0x22`, `0x2a`, `0x2e` and `0x3a..=0x3f`; note that +/// includes `0x2e`, contradicting the older comment here that named it as a +/// core-power claimant. +pub(crate) const REGION_C_GPU_CORE_PWR_ASSERT_COUNTER_OFFSET: usize = 0xe51; +/// Native cold value before the host publishes its first flag-1 control. +pub(crate) const REGION_C_GPU_CORE_PWR_ASSERT_COUNT: u32 = 0; + +/// Whether a device-control opcode advances the host GPU-core power-assert +/// counter. The 64-entry table is exhaustive in B1; values outside it are not +/// valid device-control opcodes. +pub(crate) const fn device_control_asserts_gpu_power(opcode: u32) -> Option { + if opcode > 0x3f { + return None; + } + Some(!matches!( + opcode, + 0x00 | 0x0e | 0x15 | 0x16 | 0x1b | 0x22 | 0x2a | 0x2e | 0x3a..=0x3f + )) +} + +// --- Status blocks ----------------------------------------------------------------- + +/// Status block `+0x04` holds u32 1 pre-init. `+0x10`/`+0x14` are *post-ack +/// firmware state* and must be zero at handoff. +pub(crate) const STATUS_BLOCK_ONE_OFFSET: usize = 0x04; +pub(crate) const STATUS_BLOCK_POST_ACK_OFFSETS: [usize; 2] = [0x10, 0x14]; +pub(crate) const PRIMARY_STATUS_B_FWCTL_STATE: usize = 0x48e0; +pub(crate) const PRIMARY_STATUS_B_FWCTL_RING: usize = 0x48e8; +pub(crate) const PRIMARY_STATUS_B_CONFIG_HEADER: usize = 0xe434; +pub(crate) const PRIMARY_STATUS_B_CONFIG_OFFSET: usize = 0xe440; + +// --- Instance separation ----------------------------------------------------------- + +/// The secondary root sits exactly this far above the primary root; the +/// secondary's initialisation message carries the primary's address plus this +/// delta in the same field, so the pair is not placed freely. +pub(crate) const SECONDARY_ROOT_DELTA: u64 = 0x8000; +/// Each instance maps the ADT shared region from its own half: the secondary +/// maps it from base plus this delta, and walks a top table at the same +/// offset into its half. Mappings installed only in the primary's tables +/// leave the secondary unable to translate anything it is handed. +pub(crate) const SECONDARY_SHARED_REGION_DELTA: u64 = 0x40000; + +/// Physical firmware instance encoded at root `+0x28`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum InstanceRole { + /// `gfx-asc`: scheduler, device control, and all work channels. + Primary = 0, + /// `gfx1-asc`: control-only peer. Linux starts it after primary RTKit is ready. + Secondary = 1, +} + +pub(crate) const fn root_size(role: InstanceRole) -> usize { + match role { + InstanceRole::Primary => ROOT_SIZE_PRIMARY, + InstanceRole::Secondary => ROOT_SIZE_SECONDARY, + } +} + +/// GPU virtual addresses of the objects both instances share. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedAddresses { + /// Shared 16 KiB region A (root `+0x08`). + pub(crate) region_a: u64, + /// Shared region C (root `+0x20`), 0x1000 bytes. + pub(crate) region_c: u64, + /// Base of the contiguous hardware-data bundle. The hardware-data object + /// sits at the base; the five main-config views, the repeated region, and + /// the secondary's `+0x471` pointer are all derived offsets into it. + pub(crate) hw_data_bundle: u64, + /// Size of that one allocation; must be at least + /// [`HW_DATA_BUNDLE_MIN_ALLOC`]. + pub(crate) hw_data_bundle_alloc: usize, +} + +/// GPU virtual addresses of one instance's private objects. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct InstanceAddresses { + /// This instance's root; its address rides the endpoint-0x20 doorbell. + pub(crate) root: u64, + pub(crate) main_config: u64, + pub(crate) status_a: u64, + /// Required for the primary; must be zero for the secondary (whose root + /// still has the field, written zero). + pub(crate) status_b: u64, + /// Root `+0xb8`/`+0xc0`. Required for the secondary; the primary root is + /// only 0xb8 bytes and cannot even hold them, so they must be zero there. + pub(crate) secondary_extras: [u64; 2], +} + +/// One channel-table entry: three state-counter addresses, then the ring base. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ChannelEntry { + pub(crate) states: [u64; 3], + pub(crate) ring: u64, +} + +impl ChannelEntry { + pub(crate) const EMPTY: ChannelEntry = ChannelEntry { + states: [0; 3], + ring: 0, + }; +} + +/// The primary's global region views at main `+0x2d0`: a globally retained, +/// initially zero HWDSID accounting table (256 pairs of u32), followed by +/// the high halves of two low/high alias pairs (the low halves +/// are the fixed context-0 addresses in +/// [`REGION_VIEW_LOW_ADDRS`]). Each pair must map one physical page from both +/// address spaces — an invariant of the UAT mapping layer that cannot be +/// checked from the addresses alone. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrimaryRegionViews { + pub(crate) sentinel_high: u64, + pub(crate) alias_high: [u64; 2], +} + +/// One register-window declaration for the hardware-data mapping array. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWindow { + pub(crate) slot: usize, + pub(crate) phys: u64, + pub(crate) device_va: u64, + /// Byte count; written at both `+0x10` and `+0x14`. Not always a granule + /// multiple. + pub(crate) size: u32, + /// Low 24 bits of the physical address for accelerator-internal windows, + /// else zero. + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +/// M5 25G76 descriptor fields +0x10 and +0x14 are independent. In slot 39 +/// native publishes 0x10000 and 0x8000 respectively. The older RegisterWindow +/// encoder duplicates one size and must not serialize this target's table. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RegisterWindow { + pub(crate) slot: usize, + pub(crate) phys: u64, + pub(crate) device_va: u64, + pub(crate) size_10: u32, + pub(crate) size_14: u32, + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +impl M5RegisterWindow { + /// Actual M5 PTE walks cover size_10 rounded to 16 KiB, including the + /// four pages of slot 39. size_14 is independently serialized firmware + /// metadata and must not truncate that mapping to two pages. + pub(crate) fn mapped_pages(&self) -> Result<(u64, u64, u64), InitdataError> { + let mask = UAT_PAGE_SIZE as u64 - 1; + if self.size_10 == 0 || self.phys & mask != self.device_va & mask { + return Err(InitdataError::RegisterWindow); + } + let physical = self.phys & !mask; + let virtual_address = self.device_va & !mask; + let end = self.device_va.checked_add(self.size_10 as u64) + .and_then(|value| value.checked_add(mask)) + .ok_or(InitdataError::AddressOutOfRange)? & !mask; + let span = end.checked_sub(virtual_address).ok_or(InitdataError::AddressOutOfRange)?; + physical.checked_add(span).ok_or(InitdataError::AddressOutOfRange)?; + Ok((physical, virtual_address, span)) + } +} + +/// Source-constructed register descriptors from both identical pre-handoff +/// J813/25G76 captures. These are firmware virtual mappings, not a request to +/// access or power the devices. No runtime currently consumes this profile. +pub(crate) const M5_25G76_REGISTER_WINDOWS: [M5RegisterWindow; 22] = { + const fn w(slot: usize, phys: u64, va: u64, size_10: u32, size_14: u32, + unk_18: u64, flag: u32) -> M5RegisterWindow { + M5RegisterWindow { slot, phys, device_va: 0xffff_fc21_8000_0000 + va, + size_10, size_14, unk_18, flag } + } + [ + w(0, 0x381014000, 0x000000, 0x04000, 0x04000, 0, 2), + w(3, 0x220104000, 0x008000, 0x18000, 0x18000, 0, 2), + w(8, 0x3a1350000, 0x028000, 0x04000, 0x04000, 0, 2), + w(9, 0x3803d0000, 0x030000, 0x04000, 0x04000, 0, 2), + w(10, 0x3803c0000, 0x038000, 0x04000, 0x04000, 0, 0), + w(12, 0x42165c000, 0x040000, 0x04000, 0x04000, 0, 2), + w(14, 0x380280000, 0x048000, 0x08000, 0x08000, 0, 0), + w(15, 0x38c840000, 0x058000, 0x24000, 0x24000, 0, 2), + w(17, 0x580000000, 0x080000, 0x21400, 0x21400, 0, 2), + w(22, 0x581000000, 0x0a8000, 0x08000, 0x08000, 0, 2), + w(24, 0x580ec0000, 0x0b8000, 0x10000, 0x10000, 0, 2), + w(26, 0x580d04000, 0x0d0000, 0x08000, 0x08000, 0xd04000, 2), + w(27, 0x580d0d000, 0x0e1000, 0x01000, 0x01000, 0xd0d000, 2), + w(28, 0x580d50000, 0x0e8000, 0x10000, 0x10000, 0xd50000, 2), + w(29, 0x580d10000, 0x100000, 0x04000, 0x04000, 0xd10000, 2), + w(31, 0x580d40000, 0x108000, 0x04000, 0x04000, 0xd40000, 2), + w(32, 0x580d60000, 0x110000, 0x08000, 0x08000, 0xd60000, 2), + w(35, 0x580e00000, 0x120000, 0x04000, 0x04000, 0xe00000, 2), + w(39, 0x580e08000, 0x128000, 0x10000, 0x08000, 0, 2), + w(41, 0x580e33800, 0x143800, 0x04000, 0x04000, 0, 2), + w(42, 0x580e35000, 0x151000, 0x04000, 0x04000, 0, 2), + w(50, 0x580014400, 0x160400, 0x00400, 0x00400, 0x14400, 0), + ] +}; + +pub(crate) fn encode_m5_25g76_register_map(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for window in M5_25G76_REGISTER_WINDOWS { + let base = window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, window.phys); + put_u64(out, base + 8, window.device_va); + put_u32(out, base + 0x10, window.size_10); + put_u32(out, base + 0x14, window.size_14); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + for slot in [2, 5, 6, 7, 30, 33, 34, 37, 38, 40, 43, 46, 47, 48, 49, 52] { + put_u32(out, slot * HW_REGISTER_MAP_ENTRY_SIZE + 0x20, 2); + } + Ok(()) +} + +pub(crate) const M5_25G76_HW_DATA_SIZE: usize = 0x2708; + +/// Native sharedDataDescriptor sizes loaded at AGXFirmware::allocFirmwareData +/// +0x8b8 onwards from 25G76 __const 0xfffffe0007139820. The allocator rounds +/// each next object to 64 bytes. These are independent allocations, not the +/// overlapping views used while investigating T8140. +pub(crate) const M5_25G76_SHARED_ALLOCATIONS: [(usize, usize); 7] = [ + (0x0000, 0x2708), (0x2740, 0x0c18), (0x3380, 0x1048), + (0x4400, 0x0e10), (0x5240, 0x6a30), (0xbc80, 0x0068), + (0xbd00, 0x0800), +]; +pub(crate) const M5_25G76_SHARED_PREFIX_SIZE: usize = 0xc500; +pub(crate) const M5_25G76_AON_PTD_SIZE: usize = 0xe4; + +/// J813/25G76 AON PTD backing, captured twice before each role's handoff and +/// unchanged after a passing Metal workload. The three recorded words are +/// scoped to this platform's telemetry description; they are not generic +/// M5 constants. The allocation and GPU binding must remain caller-owned. +pub(crate) fn encode_m5_25g76_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_25G76_AON_PTD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, 0x6730_5452); + put_u32(out, 4, 0x001d_0001); + put_u32(out, 8, 0x0000_1401); + Ok(()) +} + +/// Native independent primary status/config allocation sizes, from the +/// 25G76 descriptors at 0xfffffe0007139880 / 0xfffffe0007139888. +pub(crate) const M5_25G76_STATUS_CONFIG_SIZE: usize = 0x6a8; +const M5_25G76_STATUS_CONFIG_WORDS: &[(usize, u32)] = &[ + (0x010, 0x0000001f), (0x014, 0x3e0a3d71), (0x018, 0x401ae148), + (0x01c, 0x00000028), (0x020, 0x00000001), (0x0c8, 0x00002af8), + (0x0cc, 0x00001f40), (0x0d8, 0x41980000), (0x0dc, 0x3d0f5c28), + (0x0f0, 0x000026ac), (0x0f4, 0x000000c8), (0x108, 0x42c80000), + (0x10c, 0x43c80000), (0x110, 0x000000c8), (0x128, 0x00000001), + (0x12c, 0x00005eb0), (0x130, 0x3e800000), (0x134, 0x3f99999a), + (0x158, 0x0000000e), (0x1c0, 0x00000001), (0x1c4, 0x00000001), + (0x1c8, 0x00000002), (0x1cc, 0x00000001), (0x1d0, 0x00000001), + (0x1d4, 0x00000001), (0x1d8, 0x00000001), (0x588, 0x00000001), + (0x58c, 0x0000000a), (0x590, 0x00000dac), (0x594, 0x000003e8), + (0x598, 0x00000bb8), (0x59c, 0x000002bc), (0x5cc, 0x00000001), + (0x5d0, 0x00000004), (0x5dc, 0x00001f40), (0x5e0, 0x000000c8), + (0x5e4, 0x00000fa0), (0x5e8, 0x0000012c), (0x5ec, 0x000007d0), + (0x5f0, 0x0000012c), (0x5f4, 0x000003e8), (0x5f8, 0x0000012c), + (0x600, 0x00000001), (0x604, 0x41700000), (0x608, 0x40a00000), + (0x60c, 0x00000020), (0x610, 0x000004b0), (0x620, 0x0000044c), +]; + +/// Build the primary status object before either firmware role is started. +/// Secondary root+0xb8 aliases status+0xe440. GFX1 copies that pointer to +/// firmware global 0x176248 at text+0x211d4 and later writes config+0x670, +/// +0x678 and +0x680 at text+0x1af94..0x1afc4. Those are firmware-owned +/// runtime state, not platform constants, and must start at zero even when +/// a primary pre-handoff capture (taken after secondary startup) has values. +pub(crate) fn encode_m5_25g76_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != PRIMARY_STATUS_B_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, PRIMARY_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, PRIMARY_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, PRIMARY_STATUS_B_CONFIG_HEADER, 1); + for &(offset, value) in M5_25G76_STATUS_CONFIG_WORDS { + put_u32(out, PRIMARY_STATUS_B_CONFIG_OFFSET + offset, value); + } + Ok(()) +} + + +/// Shared private arena observed before primary initialization on J813. +/// The status and secondary-grid pointers name overlapping views of this +/// one backing. This extent is a measured mapped arena, not a claim about +/// the sizes of the native host's individual suballocations. +pub(crate) const M5_25G76_PRIVATE_CLUSTER_SIZE: usize = 0x178000; + +pub(crate) fn encode_m5_25g76_private_cluster( + fwctl_state: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_PRIVATE_CLUSTER_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + let fwctl_ring = fwctl_state.checked_add(0x40).ok_or(InitdataError::AddressOutOfRange)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_m5_25g76_primary_status_b(fwctl_state, fwctl_ring, &mut out[0x340..0xee40])?; + for offset in [0xee40, 0xc3100] { + encode_status_block(&mut out[offset..offset + STATUS_BLOCK_SIZE])?; + } + encode_secondary_root_extra_1_for_profile(RootProfile::J813G17G25G76, + &mut out[0x177400..0x177480])?; + // The main-config encoders carry one measured opening record (0x16 / + // 0x2a). Publish exactly one producer entry for each control channel. + // All work-channel and report counters remain zero. + put_u32(out, 0x320, 1); + put_u32(out, 0x1773e0, 1); + Ok(()) +} + +/// Caller-owned bindings in the power/performance policy allocation. +/// The host stores mach_absolute_time() at policy+0x651c. Supply the live +/// platform counter in its native timebase, never a captured timestamp. +#[derive(Clone, Copy)] +pub(crate) struct M5PolicyBindings { + pub state: u64, + pub aon_ptd: u64, + pub boot_timestamp: u64, +} + +/// Pre-handoff J813/25G76 scalars, identical across two cold boots after +/// removing owned pointers and the native boot timestamp. The repeated +/// 512-entry u16 table is encoded separately. Calibration fields whose +/// meanings remain unknown are scoped to this measured ten-core target. +const M5_25G76_POLICY_WORDS: &[(usize, u32)] = &[ + (0x3d98, 0xffffffff), (0x3db0, 0xffffffff), (0x5244, 0x00030d40), + (0x5248, 0x00030d40), (0x5250, 0x00000004), (0x5254, 0x3f800000), + (0x526c, 0x00000001), (0x5270, 0x00000001), (0x5280, 0x00000064), + (0x5284, 0x00000001), (0x5288, 0x000004b0), (0x5290, 0x00000064), + (0x52a0, 0x00000004), (0x52a4, 0x3f800000), (0x52ac, 0x00000001), + (0x52b0, 0x00000001), (0x52c0, 0x00000064), (0x52c4, 0x00000001), + (0x52c8, 0x000004b0), (0x52d0, 0x00000064), (0x5300, 0x3f828f5c), + (0x5304, 0x3f828f5c), (0x5308, 0x3f828f5c), (0x530c, 0x3f828f5c), + (0x5310, 0x3f828f5c), (0x5314, 0x3f828f5c), (0x5318, 0x3f828f5c), + (0x531c, 0x3f828f5c), (0x5320, 0x3f828f5c), (0x5324, 0x3f828f5c), + (0x5328, 0x3f828f5c), (0x532c, 0x3f828f5c), (0x5330, 0x3f828f5c), + (0x5bf8, 0x00000271), (0x5c04, 0x3f7f2e9f), (0x5c0c, 0x3b516154), + (0x5c14, 0x3ca59586), (0x5c20, 0x46bd6000), (0x5c24, 0x40a90fe4), + (0x5c2c, 0xbd4ac5b4), (0x5c30, 0x00000028), (0x5c34, 0x000004b0), + (0x5c38, 0x000004b0), (0x5c44, 0x00005eb0), (0x5c78, 0x000004b0), + (0x5cb0, 0x00000008), (0x5cbc, 0x3f000000), (0x5cc4, 0x3f000000), + (0x5ccc, 0x3f99999a), (0x5cd8, 0x47800000), (0x5cdc, 0x3e800000), + (0x5ce8, 0x00000028), (0x5cec, 0x000004b0), (0x5cfc, 0x00005eb0), + (0x5d08, 0x00000010), (0x5d10, 0x0005dc00), (0x5d68, 0x0000004a), + (0x5d70, 0x0000005a), (0x5d74, 0x00000032), (0x5d78, 0x00000006), + (0x5d80, 0x00000006), (0x5d84, 0x00000001), (0x5d8c, 0x3f4ccccd), + (0x5d90, 0x3f7eb852), (0x5d94, 0x3e4ccccd), (0x5d98, 0x3ba3d70a), + (0x5d9c, 0x3f144b3c), (0x5da0, 0x3f144b3c), (0x5da8, 0x42be0000), + (0x5dac, 0x4029eb85), (0x5db0, 0x4029eb85), (0x5db4, 0x4139435e), + (0x5db8, 0x00000064), (0x5dbc, 0x000004b0), (0x5dc0, 0x00000064), + (0x5dcc, 0x0000004a), (0x5dd4, 0x00000001), (0x5e00, 0x00000064), + (0x5e20, 0x00000401), (0x5e24, 0x00006400), (0x5e2c, 0x001f4000), + (0x5e30, 0x0000c800), (0x5e34, 0x000fa000), (0x5e38, 0x00012c00), + (0x5e3c, 0x0007d000), (0x5e40, 0x00012c00), (0x5e44, 0x0003e800), + (0x5e48, 0x00012c00), (0x5e4c, 0x00000100), (0x5ea4, 0x00000001), + (0x5ed0, 0x00000001), (0x74d8, 0x47800000), (0x74dc, 0x42200000), + (0x74e0, 0x44960000), (0x74e4, 0x0000062a), (0x74f8, 0x00000028), + (0x7500, 0x42c80000), (0x7504, 0x000004b0), (0x750c, 0x3f4ccccd), + (0x7510, 0x3e4ccccd), (0x7550, 0xdcdcfbfa), (0x7558, 0x00001f40), + (0x7560, 0x00000004), (0x7574, 0x3f800000), (0x757c, 0x3d0f5c28), + (0x7588, 0x47800000), (0x758c, 0x41980000), (0x7598, 0x00000028), + (0x759c, 0x000004b0), (0x75a0, 0x000004b0), (0x75ac, 0x00002af8), + (0x75e0, 0x000004b0), (0x77f8, 0x00000002), (0x77fc, 0x42000006), + (0x7a10, 0xb8520000), (0x7a14, 0x00003f7e), (0x7a18, 0xd70a0000), + (0x7a1c, 0x00003ba3), (0x7a24, 0x000043c8), (0x7a30, 0x00004780), + (0x7a34, 0x000042c8), (0x7a3c, 0x0384bac8), (0x7a40, 0x03e80000), + (0x7a44, 0x03e80000), (0x7a4c, 0x80000000), (0x7a50, 0x26ac453b), + (0x7a84, 0x03e80000), (0x7aa0, 0x00c80000), (0x7b58, 0x000004b0), + (0x7b6c, 0x00000001), (0x7ba0, 0x00000101), (0x7ba4, 0x00000200), + (0x7bbc, 0x00023f00), (0x7bc0, 0x00000100), (0x7bc4, 0x00000100), + (0x7bdc, 0x00024000), (0x7be0, 0x00000100), (0x7be4, 0x00000100), + (0x7bfc, 0x00024100), (0x7c48, 0x00000001), (0x7c50, 0x00005eb0), + (0x7c54, 0x00005eb0), (0x7c58, 0x00005eb0), (0x7c68, 0x3f800000), + (0x7c6c, 0x00000004), (0x7c70, 0x00000008), (0x7c74, 0x0002ee00), + (0x7d44, 0x0000007c), (0x7d50, 0x3f77bdef), (0x7d58, 0x3d042108), + (0x7d60, 0x3e0a3d71), (0x7d6c, 0x47800000), (0x7d70, 0x401ae148), + (0x7d7c, 0x00000028), (0x7d80, 0x000004b0), (0x7d84, 0x000004b0), + (0x7d8c, 0x46bd6000), (0x7d90, 0x00005eb0), (0x7d9c, 0x000000f8), + (0x7da4, 0x005ad200), (0x7dc4, 0x000004b0), (0x7e04, 0x00000001), + (0x7ed4, 0x00000032), (0x7ed8, 0x00000001), (0x7ee0, 0x3f638e39), + (0x7ee4, 0x3f2aaaab), (0x7ee8, 0x3de38e39), (0x7eec, 0x3eaaaaab), + (0x7ef0, 0xbecccccd), (0x7ef4, 0xbf4ccccd), (0x7efc, 0x47800000), + (0x7f00, 0xc0a00000), (0x7f04, 0xc0a00000), (0x7f0c, 0x00000064), + (0x7f10, 0x000004b0), (0x7f14, 0x00000064), (0x7f1c, 0x45fa0000), + (0x7f20, 0x00000578), (0x7f28, 0x0000000e), (0x7f2c, 0x00000048), + (0x7f30, 0x00000018), (0x7f34, 0x001a5e00), (0x7f3c, 0x0008ca00), + (0x7f48, 0x47800000), (0x7f70, 0x00001f40), (0x7f74, 0x00000004), + (0x7f78, 0x000009c4), (0x7f7c, 0x00000258), (0x7f80, 0x00000002), + (0x7f84, 0x00000004), (0x7f88, 0x00000032), (0x7fb8, 0x47800000), + (0x7fc8, 0x00000028), (0x7fcc, 0x000004b0), (0x8098, 0x00000001), + (0x80ac, 0x0000000a), (0x80b8, 0x00000001), (0x8174, 0xac000000), + (0x8178, 0xe800000d), (0x817c, 0xb8000003), (0x8180, 0xb000000b), + (0x8184, 0xb0000004), (0x8188, 0xbc000004), (0x818c, 0x20000002), + (0x8198, 0x00000001), (0x81b4, 0x00000004), (0x81c0, 0x3f780000), + (0x81c8, 0x3d000000), (0x81d0, 0x40a00000), (0x81dc, 0x47800000), + (0x81e0, 0x41700000), (0x81ec, 0x0000044c), (0x81f0, 0x000004b0), + (0x81f4, 0x000004b0), (0x8200, 0x000004b0), (0x8234, 0x000004b0), + (0x827c, 0x00000064), (0x8290, 0xdcdcfbfa), (0x8298, 0x0000007d), + (0x829c, 0x00000001), (0x8ee0, 0xdcdcfbfb), (0x8ee8, 0x00000001), + (0xaeec, 0x41c80000), (0xaef0, 0x41c80000), (0xb6a8, 0x67305452), + (0xb6e8, 0x00000001), (0xb728, 0x5243464c), (0xb72c, 0x4758464c), + (0xb738, 0x47584c50), (0xb748, 0x00000001), (0xb7a8, 0x00000001), + (0xb7b4, 0x42b73333), (0xb7f4, 0x449b8000), (0xb834, 0x42bc0000), + (0xb8e0, 0x0000186a), +]; + +/// Initialize the six allocations following M5 hardware-data. Preserve the +/// hardware block and subsequent channel resources. No firmware is started. +pub(crate) fn encode_m5_25g76_bundle_policy( + bindings: M5PolicyBindings, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() < M5_25G76_SHARED_PREFIX_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(bindings.state)?; + required_address(bindings.aon_ptd)?; + let state_end = bindings.state.checked_add(STATUS_BLOCK_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + let aon_end = bindings.aon_ptd.checked_add(M5_25G76_AON_PTD_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(state_end - 1)?; + required_address(aon_end - 1)?; + if bindings.state < aon_end && bindings.aon_ptd < state_end { + return Err(InitdataError::PairAliasedObject); + } + out[0x2740..M5_25G76_SHARED_PREFIX_SIZE].fill(0); + for &(offset, value) in M5_25G76_POLICY_WORDS { + put_u32(out, offset, value); + } + for index in 0..512 { + put_u16(out, 0x6ed7 + index * 2, 0x7fff); + } + put_u64(out, 0x8ed8, bindings.state); + put_u64(out, 0xb6a0, bindings.aon_ptd); + put_u64(out, 0xb75c, bindings.boot_timestamp); + Ok(()) +} + + +/// Recorded platform words outside the explicit register, performance and +/// owned-pointer fields. Meanings not yet derived are deliberately kept as +/// target-scoped measured values. Never insert runtime completion counters +/// or captured allocation addresses here. Reference hardware-data SHA-256: +/// 6f9ac0c4664f898a0ddb4e5df9ad160a23c1b52d3fd750750779cbdebe446be9. +const M5_25G76_HW_PLATFORM_WORDS: &[(usize, u32)] = &[ + (0x0004, 0x0000006f), (0x0008, 0xffc00000), (0x0014, 0x00000010), + (0x001c, 0x00000010), (0x0020, 0xffff8000), (0x0024, 0x000002ff), + (0x0028, 0x81400000), (0x002c, 0xfffffc21), (0x00e0, 0x00002008), + (0x00e8, 0x20080000), (0x00f4, 0x00002008), (0x00f8, 0x000024cb), + (0x00fc, 0xc98a2cfa), (0x0100, 0xf4f624cb), (0x0104, 0x1877e917), + (0x0108, 0x38d924cb), (0x010c, 0xbdab0000), (0x02d8, 0xfff82000), + (0x02dc, 0xd32d2cdb), (0x02e0, 0xf5002000), (0x02e4, 0x21dae926), + (0x02e8, 0x38b62000), (0x02ec, 0xc7420008), (0x03e0, 0x00007fe0), + (0x03e8, 0x7fe00000), (0x03f4, 0x00007fe0), (0x03f8, 0x00008000), + (0x0400, 0x80000000), (0x040c, 0x00008000), (0x05d8, 0x4b232645), + (0x05dc, 0x00000e98), (0x05e0, 0xd5a2ea5f), (0x05e4, 0x40003fff), + (0x05e8, 0xca5e4000), (0x05ec, 0x4000f5a2), (0x0e90, 0x00008142), + (0x0e94, 0x00000002), (0x0ea0, 0x00000001), (0x0ea4, 0x00000001), + (0x0eb8, 0x00000001), (0x0ec0, 0x00000001), (0x0ec8, 0x00000001), + (0x0ed0, 0x00005dc0), (0x0ed4, 0x00000001), (0x0ed8, 0x00000008), + (0x0ee0, 0x00000001), (0x0ee4, 0x00000001), (0x0ee8, 0x00000001), + (0x0f04, 0x0000001f), (0x0f24, 0x07800004), (0x0f28, 0x00000006), + (0x0f34, 0x00000001), (0x0f38, 0x00000001), (0x0f4c, 0x00000031), + (0x0f6c, 0x00000001), (0x0f88, 0x0000000a), (0x0f8c, 0x00000001), + (0x0fac, 0x00000001), (0x0fb8, 0x00000020), (0x0fbc, 0x00000005), + (0x0fc0, 0x0000000a), (0x0fc4, 0x0000000c), (0x2544, 0x00000004), + (0x2548, 0x00000006), (0x254c, 0x00000003), (0x2550, 0x00000007), + (0x2554, 0x00000007), (0x2568, 0x0000000a), (0x2570, 0x00000001), + (0x258c, 0x00180000), (0x2594, 0x00000001), (0x2598, 0x00000001), + (0x259c, 0x00000001), (0x25a4, 0x00000001), (0x25a8, 0x00000001), + (0x25ac, 0x00000001), (0x25b4, 0xffffffff), (0x25b8, 0xffffffff), + (0x25bc, 0xffffffff), (0x25c0, 0xffffffff), (0x25c4, 0xffffffff), + (0x25c8, 0xffffffff), (0x25cc, 0xffffffff), (0x25d0, 0xffffffff), + (0x25d4, 0xffffffff), (0x25d8, 0xffffffff), (0x25dc, 0xffffffff), + (0x25e0, 0xffffffff), (0x25f4, 0x00000001), (0x2600, 0x00000001), + (0x2608, 0x00000001), (0x2610, 0x00000100), (0x26b4, 0x00000001), + (0x26b8, 0x00000001), (0x26e0, 0x000003e8), (0x26e8, 0x00000001), +]; + +/// Reconstruct the M5 reference hardware-data block with caller-owned queue +/// and QoS bindings. Performance values are measured on the ten-core J813 +/// reference, not derived from ADT and not a generic M5 power configuration. +/// Both pre-handoff firmware roles supplied identical reference bytes. +/// This encoder grants no runtime admission and performs no MMIO accesses. +pub(crate) fn encode_m5_25g76_hw_data( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + qos_high: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(qos_high)?; + let qos_end = qos_high.checked_add(0xe40).ok_or(InitdataError::AddressOutOfRange)?; + required_address(qos_end - 1)?; + let mut completion = [0; 0x80]; + encode_m5_25g76_completion_table(aliases, &mut completion)?; + for alias in aliases { + if qos_high < alias.high + M5_25G76_COMPLETION_BACKING_SIZE as u64 + && alias.high < qos_end { + return Err(InitdataError::PairAliasedObject); + } + } + + out.fill(0); + for &(offset, value) in M5_25G76_HW_PLATFORM_WORDS { + put_u32(out, offset, value); + } + encode_m5_25g76_register_map(&mut out[0x640..0xe88])?; + out[0x2614..0x2694].copy_from_slice(&completion); + put_u64(out, 0x26ac, qos_high); + + // Native includes index zero plus twelve operating points. Each voltage + // occupies all sixteen lanes of its 0x40-byte state row. + const FREQUENCY: [u32; 13] = [0, 338, 486, 636, 796, 888, 988, 1084, 1182, 1278, 1374, 1470, 1578]; + const CORE_VOLTAGE: [u32; 13] = [125, 615, 630, 660, 705, 735, 755, 800, 830, 865, 905, 945, 1010]; + const MEMORY_VOLTAGE: [u32; 13] = [770, 770, 770, 770, 770, 770, 770, 800, 830, 865, 905, 945, 1010]; + const RELATIVE_A: [u32; 13] = [0, 17, 23, 30, 42, 50, 58, 71, 82, 85, 89, 93, 100]; + const RELATIVE_B: [u32; 13] = [0, 0, 11, 24, 36, 44, 52, 60, 68, 75, 83, 91, 100]; + for index in 0..13 { + put_u32(out, 0xfc8 + index * 4, FREQUENCY[index]); + put_u32(out, 0x1848 + index * 4, 0x3f828f5c); + put_u32(out, 0x18c8 + index * 4, RELATIVE_A[index]); + put_u32(out, 0x1908 + index * 4, RELATIVE_B[index]); + for lane in 0..16 { + put_u32(out, 0x1008 + index * 0x40 + lane * 4, CORE_VOLTAGE[index]); + put_u32(out, 0x1408 + index * 0x40 + lane * 4, MEMORY_VOLTAGE[index]); + } + } + for index in 0..16 { + put_u32(out, 0x19c8 + index * 4, index as u32); + } + Ok(()) +} + +/// T8140 register windows mapped into the firmware MMIO aperture. +pub(crate) const T8140_REGISTER_WINDOWS: [RegisterWindow; 18] = [ + RegisterWindow { + slot: 0, + phys: 0x0030_1014_000, + device_va: 0xffff_fc21_8000_0000, + size: 0x004000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 3, + phys: 0x0022_0104_000, + device_va: 0xffff_fc21_8000_8000, + size: 0x018000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 9, + phys: 0x0030_03d0_000, + device_va: 0xffff_fc21_8002_8000, + size: 0x001000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 10, + phys: 0x0030_03c0_000, + device_va: 0xffff_fc21_8003_0000, + size: 0x002000, + unk_18: 0, + flag: 0, + }, + RegisterWindow { + slot: 12, + phys: 0x0040_165c_000, + device_va: 0xffff_fc21_8003_8000, + size: 0x004000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 14, + phys: 0x0030_0280_000, + device_va: 0xffff_fc21_8004_0000, + size: 0x008000, + unk_18: 0, + flag: 0, + }, + RegisterWindow { + slot: 17, + phys: 0x0048_0000_000, + device_va: 0xffff_fc21_8005_0000, + size: 0x021400, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 22, + phys: 0x0048_1000_000, + device_va: 0xffff_fc21_8007_8000, + size: 0x008000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 26, + phys: 0x0048_0d04_000, + device_va: 0xffff_fc21_8008_8000, + size: 0x008000, + unk_18: 0xd04000, + flag: 2, + }, + RegisterWindow { + slot: 27, + phys: 0x0048_0d0d_000, + device_va: 0xffff_fc21_8009_9000, + size: 0x001000, + unk_18: 0xd0d000, + flag: 2, + }, + RegisterWindow { + slot: 28, + phys: 0x0048_0d58_000, + device_va: 0xffff_fc21_800a_0000, + size: 0x008000, + unk_18: 0xd58000, + flag: 2, + }, + RegisterWindow { + slot: 29, + phys: 0x0048_0d10_000, + device_va: 0xffff_fc21_800b_0000, + size: 0x004000, + unk_18: 0xd10000, + flag: 2, + }, + RegisterWindow { + slot: 31, + phys: 0x0048_0d40_000, + device_va: 0xffff_fc21_800b_8000, + size: 0x004000, + unk_18: 0xd40000, + flag: 2, + }, + RegisterWindow { + slot: 32, + phys: 0x0048_0d60_000, + device_va: 0xffff_fc21_800c_0000, + size: 0x004000, + unk_18: 0xd60000, + flag: 2, + }, + RegisterWindow { + slot: 35, + phys: 0x0048_0e00_000, + device_va: 0xffff_fc21_800c_8000, + size: 0x004000, + unk_18: 0xe00000, + flag: 2, + }, + RegisterWindow { + slot: 39, + phys: 0x0048_0e08_000, + device_va: 0xffff_fc21_800d_0000, + size: 0x008000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 40, + phys: 0x0048_0e1c_000, + device_va: 0xffff_fc21_800e_0000, + size: 0x004000, + unk_18: 0xe1c000, + flag: 2, + }, + RegisterWindow { + slot: 41, + phys: 0x0048_0e1f_800, + device_va: 0xffff_fc21_800e_b800, + size: 0x004000, + unk_18: 0, + flag: 2, + }, +]; + +pub(crate) const T8140_REGISTER_FLAG_ONLY_SLOTS: [(usize, u32); 17] = [ + (2, 2), + (5, 2), + (6, 2), + (7, 2), + (8, 2), + (30, 2), + (33, 2), + (34, 2), + (37, 2), + (38, 2), + (42, 2), + (43, 2), + (46, 2), + (47, 2), + (48, 2), + (49, 2), + (52, 2), +]; + +/// Fail-closed validation errors for the grounded layout. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum InitdataError { + BufferSize, + AddressOutOfRange, + NullRequiredAddress, + RoleSpecificAddress, + BundleAllocationTooSmall, + ChannelTableShape, + RegisterWindow, + PairRootSeparation, + PairAliasedObject, + SharedRegionSeparation, + InterfaceMismatch, + HostMappedAllocationsDisabled, +} + +fn address_representable(address: u64) -> bool { + let high = address & !INITDATA_ADDR_MASK; + high == 0 || high == INITDATA_ADDR_PREFIX +} + +fn required_address(address: u64) -> Result<(), InitdataError> { + if address == 0 { + Err(InitdataError::NullRequiredAddress) + } else if !address_representable(address) { + Err(InitdataError::AddressOutOfRange) + } else { + Ok(()) + } +} + +fn put_u16(bytes: &mut [u8], offset: usize, value: u16) { + bytes[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn get_u32(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) +} + +fn write_runs(bytes: &mut [u8], runs: &[(usize, &[u8])]) { + for (offset, data) in runs { + bytes[*offset..*offset + data.len()].copy_from_slice(data); + } +} + +fn validate_shared(shared: &SharedAddresses) -> Result<(), InitdataError> { + required_address(shared.region_a)?; + required_address(shared.region_c)?; + required_address(shared.hw_data_bundle)?; + if shared.hw_data_bundle_alloc < HW_DATA_BUNDLE_MIN_ALLOC { + return Err(InitdataError::BundleAllocationTooSmall); + } + Ok(()) +} + +fn validate_instance(role: InstanceRole, inst: &InstanceAddresses) -> Result<(), InitdataError> { + required_address(inst.root)?; + required_address(inst.main_config)?; + required_address(inst.status_a)?; + match role { + InstanceRole::Primary => { + // The primary root has no +0xb8/+0xc0 fields at all; carrying + // secondary extras there is a role-confusion bug at the caller. + if inst.secondary_extras != [0, 0] { + return Err(InitdataError::RoleSpecificAddress); + } + required_address(inst.status_b)?; + } + InstanceRole::Secondary => { + // Every hardware-booting host hands the secondary a zero status-B; + // a nonzero value there is ungrounded, so fail closed on it. + if inst.status_b != 0 { + return Err(InitdataError::RoleSpecificAddress); + } + for extra in inst.secondary_extras { + required_address(extra)?; + } + } + } + Ok(()) +} + +/// Encode one 0x20-byte UAT level descriptor. Every field is derivable from +/// the level's index shift and entry count; the index mask +/// `(entries - 1) << shift` reproduces all three captured masks exactly. +fn encode_uat_level_descriptor(bytes: &mut [u8], offset: usize, shift: u8, entries: u16) { + bytes[offset] = 8; + bytes[offset + 1] = UAT_PAGE_BITS; + bytes[offset + 2] = UAT_PAGE_BITS; + bytes[offset + 3] = shift; + put_u16(bytes, offset + 0x04, entries); + put_u16(bytes, offset + 0x06, UAT_PAGE_SIZE); + put_u64(bytes, offset + 0x08, 1); + put_u64(bytes, offset + 0x10, UAT_LEVEL_PHYS_MASK); + put_u64(bytes, offset + 0x18, (entries as u64 - 1) << shift); +} + +/// Encode one instance's root. +/// +/// The output is zeroed first, which makes unknown bytes deterministic and, +/// critically, prevents stale host memory from becoming invented firmware +/// ABI. Root `+0x10` stays zero: it is zero at handoff on both instances and +/// its semantics are unknown. +pub(crate) fn encode_root( + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_root_for_profile(RootProfile::J700G17P, role, shared, inst, out) +} + +/// Encode the common root layout with an explicitly selected firmware build. +/// The existing runtime continues to call the J700-only wrapper above. +pub(crate) fn encode_root_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + out: &mut [u8], +) -> Result<(), InitdataError> { + // Profile-sized: identical to root_size(role) for J700/J813. + if out.len() != profile.root_size(role) { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + + out.fill(0); + put_u64(out, ROOT_VERSION, profile.version_word()); + put_u64(out, ROOT_REGION_A, shared.region_a); + // ROOT_ZERO_10 deliberately left zero. + put_u64(out, ROOT_MAIN_CONFIG, inst.main_config); + put_u64(out, ROOT_REGION_C, shared.region_c); + put_u32(out, ROOT_INSTANCE_KIND, role as u32); + put_u32(out, ROOT_CONSTANT_2C, 1); + put_u16(out, ROOT_PAGE_SIZE, UAT_PAGE_SIZE); + out[ROOT_PAGE_BITS] = UAT_PAGE_BITS; + out[ROOT_LEVEL_COUNT] = UAT_LEVEL_COUNT as u8; + for (index, (shift, entries)) in UAT_LEVELS.iter().enumerate() { + encode_uat_level_descriptor( + out, + ROOT_LEVELS + index * UAT_LEVEL_DESC_SIZE, + *shift, + *entries, + ); + } + put_u64(out, ROOT_STATUS_A, inst.status_a); + put_u64(out, ROOT_STATUS_B, inst.status_b); + if role == InstanceRole::Secondary { + put_u64(out, ROOT_SECONDARY_EXTRA_0, inst.secondary_extras[0]); + put_u64(out, ROOT_SECONDARY_EXTRA_1, inst.secondary_extras[1]); + } + Ok(()) +} + +/// Byte range of one channel-table entry inside the main configuration. +pub(crate) fn channel_table_entry_range(index: usize) -> Option> { + if index >= CHANNEL_TABLE_ENTRIES { + return None; + } + let start = MAIN_CHANNEL_TABLE + index * CHANNEL_ENTRY_SIZE; + Some(start..start + CHANNEL_ENTRY_SIZE) +} + +/// Derive the native channel table for one instance. +/// +/// `state_grid` is this instance's compact channel-state block (0x40-stride, +/// permuted per [`WORK_STATE_GRID_OFFSETS`]); `status_a` anchors the trailing +/// report/partial channels; the ring for entry 12 is `main_config + 0x4c0`; +/// work rings for the primary sit in the bundle at +/// [`WORK_RING_BUNDLE_OFFSETS`] (which requires the +/// [`NATIVE_SHARED_CLUSTER_SIZE`] cluster). The secondary leaves entries +/// 0..11 empty: it is a control-only peer with no work channels. +pub(crate) fn derive_channel_table( + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = [ChannelEntry::EMPTY; CHANNEL_TABLE_ENTRIES]; + let grid_states = |index: usize| { + let base = state_grid + WORK_STATE_GRID_OFFSETS[index] as u64; + [ + base, + base + CHANNEL_STATE_SPACING as u64, + base + 2 * CHANNEL_STATE_SPACING as u64, + ] + }; + if role == InstanceRole::Primary { + for index in 0..WORK_CHANNEL_COUNT { + table[index] = ChannelEntry { + states: grid_states(index), + ring: bundle + WORK_RING_BUNDLE_OFFSETS[index] as u64, + }; + } + } + table[CONTROL_CHANNEL_INDEX] = ChannelEntry { + states: grid_states(CONTROL_CHANNEL_INDEX), + ring: main_config + MAIN_CONTROL_RING as u64, + }; + for (slot, index) in REPORT_CHANNEL_INDICES.iter().enumerate() { + let mut states = [0u64; 3]; + for (word, offset) in states.iter_mut().zip(REPORT_STATE_STATUS_A_OFFSETS[slot]) { + *word = status_a + offset as u64; + } + table[*index] = ChannelEntry { + states, + ring: status_a + REPORT_RING_STATUS_A_OFFSETS[slot] as u64, + }; + } + table[PARTIAL_CHANNEL_INDEX] = ChannelEntry { + states: [status_a + PARTIAL_STATE_STATUS_A_OFFSET as u64, 0, 0], + ring: 0, + }; + table +} + +/// [`derive_channel_table`] with the profile's work-ring placement. For J700 +/// and J813 the result is identical to [`derive_channel_table`]. G18G keeps +/// the same channel/ring permutation and status-A-relative report layout +/// (FW_PROVEN consumers b000 0x84f0/0x27048/0x7f14/0x19984 match G17); only +/// the ring bundle offsets move (+0xf00, from the 26A428 size table). +pub(crate) fn derive_channel_table_for_profile( + profile: RootProfile, + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = derive_channel_table(role, state_grid, status_a, main_config, bundle); + if role == InstanceRole::Primary { + let rings = profile.work_ring_bundle_offsets(); + for (index, entry) in table.iter_mut().take(WORK_CHANNEL_COUNT).enumerate() { + entry.ring = bundle + rings[index] as u64; + } + } + table +} + +fn validate_populated_entry(entry: &ChannelEntry) -> Result<(), InitdataError> { + // A populated channel needs all three state-counter addresses — a null + // producer (state 2) on a populated entry makes firmware fault on address + // zero — and a real ring. + for state in entry.states { + if state == 0 { + return Err(InitdataError::ChannelTableShape); + } + if !address_representable(state) { + return Err(InitdataError::AddressOutOfRange); + } + } + required_address(entry.ring) +} + +fn validate_channel_table( + role: InstanceRole, + main_config: u64, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], +) -> Result<(), InitdataError> { + for (index, entry) in channels.iter().enumerate() { + match index { + 0..=11 => match role { + InstanceRole::Primary => validate_populated_entry(entry)?, + // The secondary has no work channels at all. + InstanceRole::Secondary => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + }, + 12 => { + validate_populated_entry(entry)?; + // Device control is embedded in the main object, at exactly + // main + 0x4c0 on both instances. + if entry.ring != main_config + MAIN_CONTROL_RING as u64 { + return Err(InitdataError::ChannelTableShape); + } + } + 13 | 14 => validate_populated_entry(entry)?, + 15 => { + // Partial: a first state address and nothing else. + required_address(entry.states[0])?; + if entry.states[1] != 0 || entry.states[2] != 0 || entry.ring != 0 { + return Err(InitdataError::ChannelTableShape); + } + } + _ => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + } + } + Ok(()) +} + +/// Encode one instance's 0x600-byte main configuration object. +/// +/// `region_views` must be `Some` for the primary and `None` for the +/// secondary: the secondary's region views carry the two low context-0 +/// values with no addresses at all. +pub(crate) fn encode_main_config( + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], + region_views: Option<&PrimaryRegionViews>, + out: &mut [u8], +) -> Result<(), InitdataError> { + encode_main_config_for_profile(RootProfile::J700G17P, role, shared, inst, + channels, region_views, out) +} + +/// The two complete M5/25G76 pre-handoff main configurations match the common +/// layout except primary +0x32c/+0x344 and secondary +0x300. Keep these +/// measured scalars build-specific; their meanings are not inferred here. +pub(crate) fn encode_main_config_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], + region_views: Option<&PrimaryRegionViews>, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != MAIN_CONFIG_SIZE { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + validate_channel_table(role, inst.main_config, channels)?; + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + required_address(views.sentinel_high)?; + for high in views.alias_high { + required_address(high)?; + } + } + (InstanceRole::Secondary, None) => {} + _ => return Err(InitdataError::RoleSpecificAddress), + } + // Every view must fit inside the one bundle allocation. (The last two end + // exactly at 0xf000, inside the 0x10000 minimum.) The view table is the + // profile's: J700/J813 keep BUNDLE_VIEW_OFFSETS/EXTENTS unchanged. + let view_offsets = profile.bundle_view_offsets(); + for (offset, extent) in view_offsets.iter().zip(profile.bundle_view_extents()) { + if offset + extent > shared.hw_data_bundle_alloc { + return Err(InitdataError::BundleAllocationTooSmall); + } + } + + out.fill(0); + // Firmware log enable. The emitter at A000 0xfffffc0000021fa0 reads this + // word and returns immediately when it is zero; every buffer the facility + // needs is already allocated and pointed at by the addresses below. The + // records are plain ASCII. Shipping macOS provisions the whole facility + // and then throws the messages away (drainFirmwareLogRing is `bti c; ret`), + // which is why the captured value is zero. + if *crate::module_parameters::g17p_fw_log.value() != 0 { + put_u32(out, MAIN_FIRMWARE_LOG_ENABLE, 1); + } + let bundle = shared.hw_data_bundle; + put_u64(out, MAIN_HW_DATA, bundle); + let repeated = bundle + profile.repeated_region_bundle_offset() as u64; + put_u64(out, MAIN_REPEATED_0, repeated); + put_u64(out, MAIN_REPEATED_1, repeated); + + for (index, entry) in channels.iter().enumerate() { + let base = MAIN_CHANNEL_TABLE + index * CHANNEL_ENTRY_SIZE; + for (word, state) in entry.states.iter().enumerate() { + put_u64(out, base + word * 8, *state); + } + put_u64(out, base + CHANNEL_ENTRY_STATE_COUNT * 8, entry.ring); + } + + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + // The five bare pointers are views into the one bundle + // allocation, at fixed offsets. Unaligned stores at +0x254. + for (index, offset) in view_offsets.iter().enumerate() { + put_u64(out, MAIN_BUNDLE_VIEWS + index * 8, bundle + *offset as u64); + } + // High-only sentinel followed by two low/high region-view pairs. + put_u64(out, MAIN_REGION_VIEWS, views.sentinel_high); + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x10, views.alias_high[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u64(out, MAIN_REGION_VIEWS + 0x20, views.alias_high[1]); + if profile == RootProfile::J700G17P { + put_u32(out, MAIN_PRIMARY_SCALAR_32C, 1); + put_u32(out, MAIN_PRIMARY_SCALAR_344, 0xabcd_abcd); + } + // KEXT_PROVEN for G18G too (0xfffffe0008c31780: main+0x3e0=0xff). + put_u32(out, MAIN_PRIMARY_MASK, MAIN_PRIMARY_MASK_VALUE); + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + (InstanceRole::Secondary, _) => { + // The secondary keeps the two low context-0 values but carries no + // view or region addresses; the values are therefore not + // properties of any region the host allocates. + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u32(out, MAIN_SECONDARY_KIND, match profile { + RootProfile::J700G17P => MAIN_SECONDARY_KIND_VALUE, + RootProfile::J813G17G25G76 => 0xc, + // INFERRED: 26A428 stores accel+0x10a74 here (25G76: + // accel+0xed30, measured 0xc on J813). Value not read on J873. + RootProfile::J873G18G26A428 => G18G_26A428_MAIN_SECONDARY_KIND, + }); + if profile == RootProfile::J873G18G26A428 { + // KEXT_PROVEN (26A428 initFirmwareSharedData, g18g_key.c:4464): + // role 1 main+0x481 = FW VA of the perf/policy allocation. + // 25G76 stored it at +0x471 and also wrote an accelerator + // pair at +0x479/+0x481, which 26A428 no longer writes. + put_u64(out, G18G_26A428_MAIN_SECONDARY_POLICY_PTR, + bundle + G18G_26A428_SHARED_LAYOUT.policy as u64); + } else { + // One additional unaligned pointer *into* primary view 2, landing + // five bytes before that view's first populated run. + let extra = bundle + + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] as u64 + + SECONDARY_EXTRA_VIEW_OFFSET as u64; + put_u64(out, MAIN_SECONDARY_EXTRA_PTR, extra); + } + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + _ => unreachable!(), + } + Ok(()) +} + +/// Encode the shared 0x3db4-byte hardware-data object (the base of the +/// bundle). Both instances reference this one object; handing the secondary a +/// private copy makes it fault. +/// +/// `completion_aliases` are the low/high GPU mappings of the two retained +/// completion-queue backings named by descriptor ordinals 0 and 2. +pub(crate) fn encode_hw_data( + windows: &[RegisterWindow], + flag_only_slots: &[(usize, u32)], + completion_aliases: [G17PKsmCompletionAliases; 2], + qos_resource_high: u64, + sksm_qid_resource_high: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + let mut used = [false; HW_REGISTER_MAP_SLOTS]; + let mut claim = |slot: usize| -> Result<(), InitdataError> { + if slot >= HW_REGISTER_MAP_SLOTS || used[slot] { + return Err(InitdataError::RegisterWindow); + } + used[slot] = true; + Ok(()) + }; + for window in windows { + claim(window.slot)?; + if window.phys == 0 || window.size == 0 { + return Err(InitdataError::RegisterWindow); + } + // Firmware reaches its registers only through this table; the device + // VAs live in one fixed aperture. + if !HW_REGISTER_APERTURE.contains(&window.device_va) { + return Err(InitdataError::RegisterWindow); + } + } + for (slot, _flag) in flag_only_slots { + claim(*slot)?; + } + for aliases in completion_aliases { + required_address(aliases.low)?; + required_address(aliases.high)?; + } + required_address(qos_resource_high)?; + required_address(sksm_qid_resource_high)?; + + out.fill(0); + // Native's `AGXFirmware::initFirmwareData` opens this object with five + // qwords that Linux has always left zero: the range-3 base and size, the + // architectural GART window base twice (Asahi's HwDataB calls the pair + // "USC start"), and the reserved page at the top of range 5. They are the + // only place the host tells the firmware side about 0x10_0000_0000, which + // is the base hardware folds into every compact TA/VDM pointer, so a zero + // here leaves USC-class units resolving bare offsets. + for (offset, value) in HW_FIRMWARE_RANGE_HEAD { + put_u64(out, *offset, *value); + } + for window in windows { + let base = HW_REGISTER_MAP_OFFSET + window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, window.phys); + put_u64(out, base + 0x08, window.device_va); + put_u32(out, base + 0x10, window.size); + put_u32(out, base + 0x14, window.size); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + for (slot, flag) in flag_only_slots { + let base = HW_REGISTER_MAP_OFFSET + slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u32(out, base + 0x20, *flag); + } + + put_u32(out, HW_CHIP_ID_OFFSET, HW_CHIP_ID_T8140); + + for index in 0..HW_PERF_LADDER_ENTRIES { + put_u32(out, HW_PERF_GROUP_BASES[0] + index * 4, HW_PERF_FREQ_A[index]); + put_u32(out, HW_PERF_FREQ_LADDER_B + index * 4, HW_PERF_FREQ_B[index]); + put_u32(out, HW_PERF_SCALE_LADDER_B + index * 4, HW_PERF_SCALE_B[index]); + put_u32( + out, + HW_PERF_RELATIVE_LADDER_A + index * 4, + HW_PERF_RELATIVE_A_VALUES[index], + ); + put_u32( + out, + HW_PERF_RELATIVE_LADDER_B + index * 4, + HW_PERF_RELATIVE_B_VALUES[index], + ); + put_u32(out, HW_PERF_INDEX_MAP_A + index * 4, HW_PERF_INDEX_A_VALUES[index]); + put_u32(out, HW_PERF_INDEX_MAP_B + index * 4, HW_PERF_INDEX_B_VALUES[index]); + put_u32(out, HW_PERF_GROUP_BASES[1] + index * 4, HW_PERF_FREQ_B[index]); + + for repeat in 0..HW_PERF_GROUP_REPEATS[0] { + put_u32( + out, + HW_PERF_GROUP_BASES[0] + + HW_PERF_VOLTAGE_DELTA + + index * HW_PERF_STATE_BLOCK_STRIDE + + repeat * 4, + HW_PERF_CORE_VOLTAGE[index], + ); + put_u32( + out, + HW_PERF_GROUP_BASES[0] + + HW_PERF_MEMORY_VOLTAGE_DELTA + + index * HW_PERF_STATE_BLOCK_STRIDE + + repeat * 4, + HW_PERF_MEMORY_VOLTAGE[index], + ); + } + put_u32( + out, + HW_PERF_GROUP_BASES[1] + + HW_PERF_VOLTAGE_DELTA + + index * HW_PERF_STATE_BLOCK_STRIDE, + HW_PERF_CORE_VOLTAGE[index], + ); + put_u32( + out, + HW_PERF_GROUP_BASES[1] + + HW_PERF_MEMORY_VOLTAGE_DELTA + + index * HW_PERF_STATE_BLOCK_STRIDE, + HW_PERF_MEMORY_VOLTAGE[index], + ); + } + + // The acceptance block: derived from the work-channel count rather than + // copied, so a constructed descriptor needs nothing from a capture here. + put_u16(out, HW_ACCEPT_CHANNEL_COUNT, WORK_CHANNEL_COUNT as u16); + for group in HW_ACCEPT_FLAG_GROUPS { + for (index, value) in HW_ACCEPT_FLAG_PATTERN.iter().enumerate() { + put_u32(out, group + index * 4, *value); + } + } + for channel in 0..WORK_CHANNEL_COUNT { + put_u32(out, HW_ACCEPT_UNASSIGNED_TABLE + channel * 4, 0xffff_ffff); + } + for offset in HW_ACCEPT_TRAILING_ONES { + // NB +0x2600 is also DAT_178648, a firmware GATE: FUN_e004's per-DM + // KSM completion-block arm runs only when (a == 0 || DAT_178648 == 0). + // Writing 0 here to let that arm run was TESTED and the driver failed + // probe with -22. I first wrote that our own initdata validation + // rejected it; that was an invention -- nothing in the runtime path + // validates this field, and the cause of the -22 was never + // established. What settles it instead is the live 25G83 capture: + // native's hardware-data block reads 1 at +0x2600 and 1 at +0x2608, + // the same as ours, so this is not a divergence. + put_u32(out, offset, 1); + } + + put_u32( + out, + G17P_KSM_COMPLETION_TABLE_PREFIX_OFFSET, + G17P_KSM_COMPLETION_TABLE_PREFIX, + ); + for (aliases, ordinal) in completion_aliases + .iter() + .zip(G17P_KSM_COMPLETION_PRESENT_ORDINALS) + { + let base = G17P_KSM_COMPLETION_TABLE_OFFSET + + ordinal as usize * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE; + put_u64(out, base, aliases.low); + put_u64(out, base + 0x08, aliases.high); + put_u32(out, base + 0x10, G17P_KSM_COMPLETION_CAPACITY); + put_u32(out, base + 0x14, G17P_KSM_COMPLETION_ENTRY_SIZE); + put_u32(out, base + 0x18, ordinal); + // +0x1c is the firmware-owned processed cursor and starts at zero. + } + + put_u64(out, G17P_QOS_RESOURCE_LIVE_OFFSET, qos_resource_high); + put_u64( + out, + G17P_SKSM_QID_CONFIG_2700_OFFSET, + G17P_SKSM_QID_CONFIG_2700, + ); + // The valid live G17P HWData object leaves +0x2708, +0x27a4 and +0x27ac + // zero. The kext writes those only in a staging/version tail and B1 has no + // consumer; the live QoS pointer is the independent +0x26ac field above. + + // Recorded platform constants. Descriptor *acceptance* survives without + // them, but the device-control phase that follows does not. +0x2630 is + // deliberately absent from the runs: it is firmware state and must start + // zero (the fill above guarantees it). + write_runs(out, HW_DATA_OPAQUE_RUNS); + // The host->firmware transport selector. + // + // `AGXFirmware::initFirmwareData` @0xfffffe0008a20940 exports + // `initdata[0x26f8] = accel[0x4f8] & 1`. That same bit decides, in + // `AGX{TA,3D,CL}ChannelSKSM::submitBuffer`, whether the host publishes the + // command record and the tag-14 AddKicks as CHANNEL COMMANDS through + // `writeChannelCommandPointer` (bit set, `accel[0x10769] & 1 == 0`) or + // instead pushes onto the host kick-trace ring and drives + // `configureHardware`/`addKicks` over MMIO (bit clear). + // + // A live SKSM capture on this target shows tag-15 and tag-14 published as + // channel commands for both render queues, so native runs with the bit + // SET and this byte at 1. This driver uses the same channel-command + // transport but left the byte at 0 -- it is outside every + // `HW_DATA_OPAQUE_RUNS` extent and nothing else wrote it -- so the + // firmware was told the host drives the MMIO bridge while the host in + // fact speaks channel commands. Declare the transport we actually use. + // MEASURED 2026-09-02, and it contradicts the reasoning above. Native's + // complete hardware-data block (hwdata-gfx.txt, captured from + // accel+0x5b0 -> +0x2b8 at submit3DChannel ENTRY, i.e. long after init) + // carries ZERO at 0x26f8. The "native runs with the bit SET" conclusion was + // inferred from seeing channel commands on the wire, not read off the + // block, and the block disagrees. Publish what native publishes. + if let Some(byte) = out.get_mut(HW_DATA_TRANSPORT_SELECTOR_OFFSET) { + *byte = 0; + } + // The measured performance tables were encoded above. Their values are + // target-scoped observations rather than a complete ADT derivation; see + // [`PERF_TABLE_VALUES_ENCODED`]. + Ok(()) +} + +/// Encode the shared 0x1000-byte region C. Both roots point at the same +/// object; giving the secondary a private copy is a difference from what a +/// working host does. +pub(crate) fn encode_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + encode_region_c_for_profile(RootProfile::J700G17P, out) +} + +/// Encode target-specific pre-handoff platform policy. M5's native policy is +/// kept separate from the experimental T8140 watchdog and idle overrides. +pub(crate) fn encode_region_c_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + if profile == RootProfile::J873G18G26A428 { + // Derived from the J813 bytes below through the proven +4 shift. + return encode_g18g_26a428_region_c(out); + } + out.fill(0); + write_runs(out, REGION_C_RUNS); + if profile == RootProfile::J813G17G25G76 { + // Scalars measured before each ASC received its root on two cold + // J813 / 25G76 boots. These are a scoped reference configuration, + // not generic M5 calibration or a power-management qualification. + // The final pair moved by 0x20; leaving the T8140 pair in place would + // initialize unrelated M5 fields. + for (offset, value) in [ + (0x084, 0x0000_5eb0), + (0x088, 0x0000_04b0), + (0x08c, 0x0000_04b0), + (0x098, 0x0000_04b0), + (0x7c4, 0x0000_04e2), + (0x7e0, 0x3f33_3333), + (0x9a8, 0x0000_0333), + (0xe38, 0), + (0xe3c, 0), + (0xe58, 0x0000_0300), + (0xe5c, 0x0000_0100), + ] { + put_u32(out, offset, value); + } + return Ok(()); + } + // regionC+0x9cc is the KSM progress-watchdog threshold for data master 2 + // (compute); the firmware copies it to 0x1778d0 and its handler starts + // `cbz w20 -> ret`, so 0 disables the watchdog. We ship 0x32 (50) here and + // leave the other data master's slot at 0 -- i.e. we arm the watchdog on + // exactly the queue we submit to. When work cannot progress for any other + // reason this turns a stall into a self-sustaining recovery loop. + if crate::g18::fw::config::PROGRESS_WATCHDOG == 0 { + put_u32(out, 0x9cc, 0); + } + // Native initializes the idle-descent selector to zero. Keep the old + // alternate predicate only as an explicit diagnostic override. + if crate::g18::fw::config::IDLE_SELECTOR != 0 { + out[0xe21] = 1; + } + for (offset, value) in [ + (0x7c, 1), + (0x80, 1), + (0x9c, 0x3000_001d), + (0xa0, 0x3100_0002), + ] { + put_u32(out, offset, value); + } + Ok(()) +} + +/// Overlay the target-scoped static third-page bytes and its one live state +/// pointer without disturbing hardware-data or overlapping view content. +pub(crate) fn encode_bundle_static( + state_ptr: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() < HW_DATA_BUNDLE_MIN_ALLOC { + return Err(InitdataError::BufferSize); + } + required_address(state_ptr)?; + write_runs(out, BUNDLE_STATIC_RUNS); + for (index, runs) in BUNDLE_VIEW_RUNS.iter().enumerate() { + let base = BUNDLE_VIEW_OFFSETS[index]; + let valid = BUNDLE_VIEW_EXTENTS[index]; + for (offset, data) in *runs { + if offset + data.len() > valid { + continue; + } + let start = base + offset; + out[start..start + data.len()].copy_from_slice(data); + } + } + for (view, offset, count, trailing_ff) in BUNDLE_VIEW_U16_REPEATS { + let base = BUNDLE_VIEW_OFFSETS[*view] + offset; + let end = base + *count * 2 + usize::from(*trailing_ff); + if end > BUNDLE_VIEW_OFFSETS[*view] + BUNDLE_VIEW_EXTENTS[*view] { + return Err(InitdataError::BundleAllocationTooSmall); + } + for index in 0..*count { + out[base + index * 2..base + index * 2 + 2].copy_from_slice(&[0xff, 0x7f]); + } + if *trailing_ff { + out[base + *count * 2] = 0xff; + } + } + put_u64(out, 0x8ed8, state_ptr); + Ok(()) +} + +/// Encode one 0x80-byte status block in its pre-init state: u32 1 at `+0x04`, +/// everything else zero. `+0x10`/`+0x14` are post-acknowledgement firmware +/// state and must be zero at handoff. +pub(crate) fn encode_status_block(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, STATUS_BLOCK_ONE_OFFSET, 1); + Ok(()) +} + +/// Encode the complete pre-init primary status/config object. +pub(crate) fn encode_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != PRIMARY_STATUS_B_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, PRIMARY_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, PRIMARY_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, PRIMARY_STATUS_B_CONFIG_HEADER, 1); + for (offset, data) in PRIMARY_STATUS_B_CONFIG_RUNS { + let start = PRIMARY_STATUS_B_CONFIG_OFFSET + offset; + out[start..start + data.len()].copy_from_slice(data); + } + Ok(()) +} + +/// Encode the secondary root's independent `+0xc0` target. +pub(crate) fn encode_secondary_root_extra_1(out: &mut [u8]) -> Result<(), InitdataError> { + encode_secondary_root_extra_1_for_profile(RootProfile::J700G17P, out) +} + +pub(crate) fn encode_secondary_root_extra_1_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + write_runs(out, SECONDARY_ROOT_EXTRA_1_RUNS); + // G18G: KEXT_PROVEN identical store geometry for this 0x60 object + // (26A428 vs 25G76 R_60 stores, all at the same offsets); the J813 values + // are carried unchanged (INFERRED, listed in G18G_26A428_UNPROVEN). + if matches!(profile, RootProfile::J813G17G25G76 | RootProfile::J873G18G26A428) { + put_u32(out, 0x10, 2); + } + Ok(()) +} + +// --- GPU firmware fast-die controller (GFX1) -------------------------------- +// +// KEXT_PROVEN (25G83 AGXG17P): +// - AGXAcceleratorG17P::populateFastDieDeviceConfigData (0x88c7af0) sets target +// 110.0 C and sensor mask 0xaa0 when accel[0x4a0] == 0x1f, else 0x82a. It +// zeroes the per-sensor array for each mask bit. It panics ("Not all MTR +// Sensors required for FastDie are enabled") only when the host MTR-enable word +// accel[0xe978] is non-zero and lacks a required bit. +// - The base populate (0x883d41c) takes optional ADT gpu-fast-die0-sensor-mask64/ +// -sensor-mask/-target (default 101.0 C, overridden above). setupConfig reads +// the optional gpu-fast-die0-release-temp/-prop-tgt-delta/-proportional-gain/ +// -integral-gain and gpu-fastdie-mrc-period. The J700 ADT carries none of +// them. +// - initPowerAndPerformanceData copies the staged values into +// - the GFX1 power object: `[[fw+0x2d8]+0x6678]`; GFX1 reaches it through +// secondary main +0x471 = bundle view 2 + 0xe40; +// - the 0x6a8-byte secondary-root +0xb8 object (`fw+0xcd8`), which Linux +// places at primary status B + 0xe440. +// FW_PROVEN (fw1 25G83 b100, 0x174d0): +// - When the +0xb8 object's +0xc4 flag is set, GFX1 copies its +0xc8..+0xe4 +// fields into the power object. Native updateLifetimeServoTempTarget and +// updateFastDieController set that flag after writing the fields. +// - GFX1 takes the lower of that target and an external target, then programs +// each masked MTR sensor's threshold registers (stride 0x200, +0x1000/+0x1004/ +// +0x1014) at (target - delta) / 100 C. +// The recorded Linux runs already carry every non-zero value native publishes +// here (see the test `fast_die_config_matches_native`). + +/// Offset of the GFX1 power object inside the hardware-data bundle. +pub(crate) const FASTDIE_OBJECT_BUNDLE_OFFSET: usize = + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] + SECONDARY_EXTRA_VIEW_OFFSET; +pub(crate) const FASTDIE_OBJ_MASK: usize = 0x2310; +pub(crate) const FASTDIE_OBJ_RELEASE_CC: usize = 0x2318; +pub(crate) const FASTDIE_OBJ_KI_DT: usize = 0x233c; +pub(crate) const FASTDIE_OBJ_KP: usize = 0x234c; +pub(crate) const FASTDIE_OBJ_DELTA_CC: usize = 0x2364; +pub(crate) const FASTDIE_OBJ_TARGET_CC: usize = 0x236c; +pub(crate) const FASTDIE_OBJ_STATE: usize = 0x25bc; +/// Offsets inside the secondary-root +0xb8 object (status B + 0xe440 here). +pub(crate) const FASTDIE_LIVE_UPDATE: usize = 0xc4; +pub(crate) const FASTDIE_LIVE_TARGET_CC: usize = 0xc8; +pub(crate) const FASTDIE_LIVE_RELEASE_CC: usize = 0xcc; +pub(crate) const FASTDIE_LIVE_KP: usize = 0xd8; +pub(crate) const FASTDIE_LIVE_KI_DT: usize = 0xdc; +/// Native G17P target, 0.01 C. +pub(crate) const FASTDIE_NATIVE_TARGET_CC: u32 = 11_000; +/// Diagnostic overrides may only lower the target, never below this, so the +/// recorded 80.00 C release stays below it. +pub(crate) const FASTDIE_MIN_TARGET_CC: u32 = 8_500; + +/// Effective fast-die configuration as encoded (raw words; floats as bits). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct FastDieConfig { + pub(crate) mask: u64, + pub(crate) target_cc: u32, + pub(crate) release_cc: u32, + pub(crate) delta_cc: u32, + pub(crate) kp_bits: u32, + pub(crate) ki_dt_bits: u32, + pub(crate) state: u32, + pub(crate) live_target_cc: u32, + pub(crate) live_release_cc: u32, + pub(crate) live_kp_bits: u32, + pub(crate) live_ki_dt_bits: u32, + pub(crate) live_update: u32, +} + +/// Read the fast-die fields from an encoded bundle and status-B object. +pub(crate) fn fast_die_config(bundle: &[u8], status_b: &[u8]) -> Result { + let o = FASTDIE_OBJECT_BUNDLE_OFFSET; + let l = PRIMARY_STATUS_B_CONFIG_OFFSET; + if bundle.len() < o + FASTDIE_OBJ_STATE + 4 || status_b.len() < l + FASTDIE_LIVE_KI_DT + 4 { + return Err(InitdataError::BufferSize); + } + Ok(FastDieConfig { + mask: get_u64(bundle, o + FASTDIE_OBJ_MASK), + target_cc: get_u32(bundle, o + FASTDIE_OBJ_TARGET_CC), + release_cc: get_u32(bundle, o + FASTDIE_OBJ_RELEASE_CC), + delta_cc: get_u32(bundle, o + FASTDIE_OBJ_DELTA_CC), + kp_bits: get_u32(bundle, o + FASTDIE_OBJ_KP), + ki_dt_bits: get_u32(bundle, o + FASTDIE_OBJ_KI_DT), + state: get_u32(bundle, o + FASTDIE_OBJ_STATE), + live_target_cc: get_u32(status_b, l + FASTDIE_LIVE_TARGET_CC), + live_release_cc: get_u32(status_b, l + FASTDIE_LIVE_RELEASE_CC), + live_kp_bits: get_u32(status_b, l + FASTDIE_LIVE_KP), + live_ki_dt_bits: get_u32(status_b, l + FASTDIE_LIVE_KI_DT), + live_update: get_u32(status_b, l + FASTDIE_LIVE_UPDATE), + }) +} + +/// Validate a diagnostic target: 0 keeps native; otherwise lower-only within +/// [`FASTDIE_MIN_TARGET_CC`]..=[`FASTDIE_NATIVE_TARGET_CC`]. +pub(crate) fn fast_die_target_override(requested_cc: u32) -> Result, InitdataError> { + match requested_cc { + 0 => Ok(None), + t if (FASTDIE_MIN_TARGET_CC..=FASTDIE_NATIVE_TARGET_CC).contains(&t) => Ok(Some(t)), + _ => Err(InitdataError::BufferSize), + } +} + +/// Apply a validated target to the power object copy (pre-boot encode only). +pub(crate) fn apply_fast_die_target_bundle(bundle: &mut [u8], target_cc: u32) -> Result<(), InitdataError> { + let at = FASTDIE_OBJECT_BUNDLE_OFFSET + FASTDIE_OBJ_TARGET_CC; + if bundle.len() < at + 4 { + return Err(InitdataError::BufferSize); + } + put_u32(bundle, at, target_cc); + Ok(()) +} + +/// Apply a validated target to the +0xb8 live object copy (pre-boot encode +/// only; no update flag: firmware starts from these words). +pub(crate) fn apply_fast_die_target_status_b(status_b: &mut [u8], target_cc: u32) -> Result<(), InitdataError> { + let at = PRIMARY_STATUS_B_CONFIG_OFFSET + FASTDIE_LIVE_TARGET_CC; + if status_b.len() < at + 4 { + return Err(InitdataError::BufferSize); + } + put_u32(status_b, at, target_cc); + Ok(()) +} + +/// Validate the cross-instance invariants established before either ASC is +/// kicked. The two roots are not placed freely: the secondary's sits exactly +/// [`SECONDARY_ROOT_DELTA`] above the primary's. +pub(crate) fn validate_pair( + primary: &InstanceAddresses, + secondary: &InstanceAddresses, +) -> Result<(), InitdataError> { + if secondary.root != primary.root + SECONDARY_ROOT_DELTA { + return Err(InitdataError::PairRootSeparation); + } + if primary.main_config == secondary.main_config || primary.status_a == secondary.status_a { + return Err(InitdataError::PairAliasedObject); + } + Ok(()) +} + +/// Validate the per-instance halves of the ADT shared region: the secondary +/// maps it from the primary's base plus [`SECONDARY_SHARED_REGION_DELTA`]. +pub(crate) fn validate_shared_region_bases( + primary_base: u64, + secondary_base: u64, +) -> Result<(), InitdataError> { + if secondary_base != primary_base + SECONDARY_SHARED_REGION_DELTA { + return Err(InitdataError::SharedRegionSeparation); + } + Ok(()) +} + +/// Reproduce the firmware's root acceptance checks. +/// +/// The version-word gate is re-derived from our own J700 image +/// (`__TEXT+0x1e8a8`); a mismatch is answered with "Host and FW Interface is +/// mismatched" on the crashlog endpoint. The `+0x2c` nonzero gate was proven +/// on the M5 parser and the field is constant 1 in every T8140 capture; its +/// T8140 parse-gate status has not been independently re-derived, but a host +/// must write 1 there regardless. Passing this does not mean the rest of the +/// descriptor is complete. +pub(crate) fn check_firmware_root_gates(root: &[u8]) -> Result<(), InitdataError> { + check_firmware_root_gates_for_profile(RootProfile::J700G17P, root) +} + +pub(crate) fn check_firmware_root_gates_for_profile( + profile: RootProfile, + root: &[u8], +) -> Result<(), InitdataError> { + if root.len() != ROOT_SIZE_PRIMARY && root.len() != ROOT_SIZE_SECONDARY { + return Err(InitdataError::BufferSize); + } + // G18G: both roles are copied as 0xc8 bytes (FW_PROVEN b000 0x21664). + if profile == RootProfile::J873G18G26A428 && root.len() != G18G_26A428_ROOT_SIZE { + return Err(InitdataError::BufferSize); + } + if get_u64(root, ROOT_VERSION) != profile.version_word() { + return Err(InitdataError::InterfaceMismatch); + } + if get_u32(root, ROOT_CONSTANT_2C) == 0 { + return Err(InitdataError::HostMappedAllocationsDisabled); + } + Ok(()) +} + +/// Recorded T8140 hardware-data platform constants (values known, meanings +/// not; derived from the hardware-validated `g17p_initdata.py` +/// `HWDATA_CONSTANTS`). 29 sparse runs, 262 bytes total. `+0x2630` is +/// deliberately absent: it is firmware state, not host configuration. +/// +/// Native fixture pointer bytes are not platform constants. The original +/// +0x26ad run included seven bytes of the captured QoS allocation address; +/// replaying it after encode_hw_data's live pointer store silently replaced +/// the owned allocation with that unrelated address. Keep only its adjacent +/// state words here and encode the complete pointer from its retained owner. +/// +/// +0x0f08 is zero in native too. That field is the firmware's ISR mask +/// selector -- FUN_1e858 copies it to DAT_1785f4, and FUN_215cc picks the +/// interrupt-enable word from it as +/// DAT_1785f4 == 2 ? 0x0e00620038 : 0x0a00620038 +/// storing it to DAT_179328, which the ISR ANDs against MMIO[0x10a10]. Writing +/// 2 here was tested and does exactly that (0x10a08 measured 0x0e00620038, +/// bit 34 enabled) and changes nothing else, so leaving it unset is right and +/// this part is not the variant-2 configuration. +/// +/// A caution for anyone diffing this table again: it has TWENTY-NINE runs, +/// several spanning many lines. A naive `\(0x..,\s*&\[(.*?)\]\)` regex +/// silently matches only 19 of them and mis-associates offsets, which +/// manufactures differences that are not there and, if used to rewrite the +/// table, corrupts it into 17 runs / 292 bytes and makes probe fail with -22. +/// Parse by matching brackets, and check the 29/262 totals before trusting a +/// comparison. +pub(crate) const HW_DATA_OPAQUE_RUNS: &[(usize, &[u8])] = &[ + (0x0004, &[0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0xff]), + ( + 0x0014, + &[ + 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x80, + 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, 0x81, 0x21, 0xfc, 0xff, 0xff, + ], + ), + (0x00e0, &[0x08, 0x20]), + (0x00ea, &[0x08, 0x20]), + ( + 0x00f4, + &[ + 0x08, 0x20, 0x00, 0x00, 0xcb, 0x24, 0x00, 0x00, 0xfa, 0x2c, 0x8a, 0xc9, 0xcb, 0x24, + 0xf6, 0xf4, 0x17, 0xe9, 0x77, 0x18, 0xcb, 0x24, 0xd9, 0x38, 0x00, 0x00, 0xab, 0xbd, + ], + ), + ( + 0x02d9, + &[ + 0x20, 0xf8, 0xff, 0xdb, 0x2c, 0x2d, 0xd3, 0x00, 0x20, 0x00, 0xf5, 0x26, 0xe9, 0xda, + 0x21, 0x00, 0x20, 0xb6, 0x38, 0x08, 0x00, 0x42, 0xc7, + ], + ), + (0x03e0, &[0xe0, 0x7f]), + (0x03ea, &[0xe0, 0x7f]), + (0x03f4, &[0xe0, 0x7f, 0x00, 0x00, 0x00, 0x80]), + (0x0403, &[0x80]), + (0x040d, &[0x80]), + ( + 0x05d8, + &[ + 0x45, 0x26, 0x23, 0x4b, 0x98, 0x0e, 0x00, 0x00, 0x5f, 0xea, 0xa2, 0xd5, 0xff, 0x3f, + 0x00, 0x40, 0x00, 0x40, 0x5e, 0xca, 0xa2, 0xf5, 0x00, 0x40, + ], + ), + ( + 0x0e94, + &[ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, + ], + ), + ( + 0x0eb8, + &[ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5d, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + ], + ), + (0x0f04, &[0x1f]), + // NOTE, do not "fix" this by adding 0x0f08 without evidence. That u32 is + // the firmware's ISR mask selector: FUN_1e858 copies hw-data +0xf08 to + // DAT_1785f4 (store at b1 0x1e9e8), and FUN_215cc picks the interrupt + // enable word from it with + // csel: DAT_1785f4 == 2 ? 0x0e00620038 : 0x0a00620038 + // storing the result to DAT_179328, which the ISR at 0x216f0 ANDs against + // MMIO[0x10a10] to decide which interrupts to service. The field is absent + // here, so it reads zero and 0x10a08 measures 0x0a00620038 -- the variant + // without bit 34. + // + // TESTED 2026-09-02: writing 2 here does exactly what the disassembly + // says -- 0x10a08 measured 0x0e00620038, bit 34 enabled -- and changes + // nothing else. The render still stalls with bit 4 clear, no KTrace 2, + // fragment status 4. Reverted because there is no oracle saying this part + // is variant 2 and it alters firmware interrupt behaviour on a guess. + // Both variants already enable bits 3, 4 and 5, so neither withholds + // new-work. + (0x0f24, &[0x04]), + (0x0f34, &[0x01, 0x00, 0x00, 0x00, 0x01]), + (0x0f4c, &[0x31]), + (0x0f6c, &[0x01]), + (0x0f88, &[0x06, 0x00, 0x00, 0x00, 0x01]), + (0x0fac, &[0x01]), + ( + 0x0fb8, + &[ + 0x1e, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0a, + ], + ), + (0x1cd8, &[0x0a]), + ( + 0x2544, + &[ + 0x04, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x07, 0x00, + 0x00, 0x00, 0x07, + ], + ), + ( + 0x2568, + &[0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01], + ), + // +0x26ac..+0x26b3 is the live QoS pointer, never captured allocation bytes. + (0x26b4, &[0x01, 0x00, 0x00, 0x00, 0x01]), + (0x26e8, &[0x01]), + (0x3d98, &[0xff, 0xff, 0xff, 0xff]), + (0x3db0, &[0xff, 0xff, 0xff, 0xff]), +]; + +/// Recorded T8140 region-C platform tuning constants (values known, meanings +/// not; generated from the hardware-validated `g17p_initdata.py` +/// `REGION_C_CONSTANTS`). 12 sparse runs, 180 bytes total. The final run +/// initializes the byte preceding the host power-assert counter and the +/// counter itself to their native cold value of zero. +pub(crate) const REGION_C_RUNS: &[(usize, &[u8])] = &[ + (0x0024, &[0xb8, 0x0b, 0x00, 0x00]), + ( + 0x0030, + &[ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x0054, + &[ + 0xff, 0xff, 0x28, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, + ], + ), + (0x0078, &[0x01, 0x00, 0x00, 0x00]), + ( + 0x0084, + &[ + 0x4e, 0x25, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, + ], + ), + (0x0098, &[0xe8, 0x03, 0x00, 0x00]), + ( + 0x07c4, + &[ + 0xd0, 0x07, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0xcd, 0xcc, 0x4c, 0x3f, 0xcd, 0xcc, + 0x4c, 0x3e, 0x66, 0x66, 0x66, 0x3f, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x80, 0x3e, + 0x9a, 0x99, 0x19, 0x3f, 0x66, 0x66, 0x66, 0x3f, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, + ], + ), + ( + 0x0998, + &[ + 0x28, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x09b8, + &[ + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + ], + ), + (0x0e1c, &[0x00, 0x01, 0x00, 0x00]), + (0x0e38, &[0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00]), + (0x0e50, &[0x00, 0x00, 0x00, 0x00]), +]; + +pub(crate) const BUNDLE_VIEW_RUNS: [&[(usize, &[u8])]; 5] = [ + &[ + (0x1658, &[0xff, 0xff, 0xff, 0xff]), + (0x1670, &[0xff, 0xff, 0xff, 0xff]), + ], + &[ + (0xa18, &[0xff, 0xff, 0xff, 0xff]), + (0xa30, &[0xff, 0xff, 0xff, 0xff]), + ], + &[ + (0xe45, &[0xdc, 0x05, 0x00, 0x00, 0xdc, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f]), + (0xe6c, &[0x01, 0x00, 0x00, 0x00, 0x01]), + (0xe80, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0xea0, &[0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01]), + (0xec0, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0xf00, &[0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f]), + (0x17f8, &[0x71, 0x02]), + (0x1804, &[0x9f, 0x2e, 0x7f, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x54, 0x61, 0x51, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x86, 0x95, 0xa5, 0x3c]), + (0x1821, &[0x38, 0x15, 0x46, 0xdb, 0x0f, 0xa9, 0x40, 0x00, 0x00, 0x00, 0x00, 0x44, 0x73, 0xd6, 0xbd, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03]), + (0x1844, &[0x4e, 0x25]), + (0x1878, &[0xe8, 0x03]), + (0x18b0, &[0x04]), + (0x18c6, &[0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0xcc, 0x40]), + (0x18da, &[0x80, 0x47, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03]), + (0x18fc, &[0x4e, 0x25]), + (0x1908, &[0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x05]), + (0x1968, &[0x5c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0x4c, 0x3f, 0xb6, 0xf3, 0x7d, 0x3f, 0xcd, 0xcc, 0x4c, 0x3e, 0x6f, 0x12, 0x03, 0x3c, 0xd8, 0xd4, 0x69, 0x3f, 0xd8, 0xd4, 0x69, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x42, 0xc3, 0xf5, 0x64, 0x40, 0xc3, 0xf5, 0x64, 0x40, 0x36, 0x94, 0x17, 0x41, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x64]), + (0x19cc, &[0x5c]), + (0x1a00, &[0x64]), + (0x1a20, &[0x01, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x1f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xa0, 0x0f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xd0, 0x07, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x01]), + (0x1aa4, &[0x01]), + (0x1ad0, &[0x01]), + (0x30da, &[0x80, 0x47, 0x00, 0x00, 0x20, 0x42, 0x00, 0x00, 0x7a, 0x44, 0xbe, 0x05]), + (0x30f8, &[0x28]), + (0x3102, &[0xc8, 0x42, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0x4c, 0x3f, 0xcd, 0xcc, 0x4c, 0x3e]), + (0x3150, &[0x2a, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04]), + (0x3176, &[0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x19, 0x04, 0x16, 0x41]), + (0x318a, &[0x80, 0x47, 0x52, 0xb8, 0xa2, 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03]), + (0x31ac, &[0xf8, 0x2a]), + (0x31e0, &[0xe8, 0x03]), + (0x33fc, &[0x06, 0x00, 0x00, 0x42]), + (0x3612, &[0x52, 0xb8, 0x7e, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x0a, 0xd7, 0xa3, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x43]), + (0x3630, &[0x80, 0x47, 0x00, 0x00, 0xc8, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0xba, 0x84, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3b, 0x45, 0xac, 0x26]), + (0x3686, &[0xe8, 0x03]), + (0x36a2, &[0xc8]), + (0x3758, &[0xe8, 0x03]), + (0x376c, &[0x01]), + (0x37a0, &[0x01, 0x01, 0x00, 0x00, 0x00, 0x04]), + (0x37bd, &[0x3f, 0x02, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01]), + (0x37dd, &[0x40, 0x02, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01]), + (0x37fd, &[0x41, 0x02]), + (0x3848, &[0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x25, 0x00, 0x00, 0x4e, 0x25, 0x00, 0x00, 0x4e, 0x25]), + (0x386a, &[0x80, 0x3f, 0x04, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x05]), + (0x3944, &[0x3c]), + (0x3950, &[0xef, 0xee, 0x6e, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x89, 0x88, 0x88, 0x3d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f]), + (0x396e, &[0x80, 0x47, 0x00, 0x00, 0x88, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0x15, 0x46, 0x4e, 0x25]), + (0x399c, &[0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x57]), + (0x39c4, &[0xe8, 0x03]), + (0x3a04, &[0x01]), + (0x3ad4, &[0x32, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x39, 0x8e, 0x63, 0x3f, 0xab, 0xaa, 0x2a, 0x3f, 0x39, 0x8e, 0xe3, 0x3d, 0xab, 0xaa, 0xaa, 0x3e, 0xcd, 0xcc, 0x4c, 0xbf, 0xcd, 0xcc, 0x4c, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x47, 0x00, 0x00, 0xa0, 0xc0, 0x00, 0x00, 0xa0, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x64]), + (0x3b1e, &[0x7a, 0x46, 0x78, 0x05]), + (0x3b2c, &[0x90, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x11]), + (0x3b4a, &[0x80, 0x47]), + (0x3b70, &[0x80, 0x3e, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0xc4, 0x09, 0x00, 0x00, 0x0d, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x32]), + (0x3bba, &[0x80, 0x47]), + (0x3bc8, &[0x28, 0x00, 0x00, 0x00, 0xe8, 0x03]), + ], + &[ + (0x15c5, &[0xdc, 0x05, 0x00, 0x00, 0xdc, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f]), + (0x15ec, &[0x01, 0x00, 0x00, 0x00, 0x01]), + (0x1600, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0x1620, &[0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01]), + (0x1640, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0x1680, &[0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f]), + (0x1f78, &[0x71, 0x02]), + (0x1f84, &[0x9f, 0x2e, 0x7f, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x54, 0x61, 0x51, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x86, 0x95, 0xa5, 0x3c]), + (0x1fa1, &[0x38, 0x15, 0x46, 0xdb, 0x0f, 0xa9, 0x40, 0x00, 0x00, 0x00, 0x00, 0x44, 0x73, 0xd6, 0xbd, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03]), + (0x1fc4, &[0x4e, 0x25]), + (0x1ff8, &[0xe8, 0x03]), + (0x2030, &[0x04]), + (0x2046, &[0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0xcc, 0x40]), + (0x205a, &[0x80, 0x47, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03]), + (0x207c, &[0x4e, 0x25]), + (0x2088, &[0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x05]), + (0x20e8, &[0x5c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0x4c, 0x3f, 0xb6, 0xf3, 0x7d, 0x3f, 0xcd, 0xcc, 0x4c, 0x3e, 0x6f, 0x12, 0x03, 0x3c, 0xd8, 0xd4, 0x69, 0x3f, 0xd8, 0xd4, 0x69, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x42, 0xc3, 0xf5, 0x64, 0x40, 0xc3, 0xf5, 0x64, 0x40, 0x36, 0x94, 0x17, 0x41, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x64]), + (0x214c, &[0x5c]), + (0x2180, &[0x64]), + (0x21a0, &[0x01, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x1f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xa0, 0x0f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xd0, 0x07, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x01]), + (0x2224, &[0x01]), + (0x2250, &[0x01]), + ], + &[ + (0x1545, &[0xdc, 0x05, 0x00, 0x00, 0xdc, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f]), + (0x156c, &[0x01, 0x00, 0x00, 0x00, 0x01]), + (0x1580, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0x15a0, &[0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01]), + (0x15c0, &[0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64]), + (0x1600, &[0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f]), + (0x1ef8, &[0x71, 0x02]), + (0x1f04, &[0x9f, 0x2e, 0x7f, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x54, 0x61, 0x51, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x86, 0x95, 0xa5, 0x3c]), + (0x1f21, &[0x38, 0x15, 0x46, 0xdb, 0x0f, 0xa9, 0x40, 0x00, 0x00, 0x00, 0x00, 0x44, 0x73, 0xd6, 0xbd, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03]), + (0x1f44, &[0x4e, 0x25]), + (0x1f78, &[0xe8, 0x03]), + (0x1fb0, &[0x04]), + (0x1fc6, &[0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0xcc, 0x40]), + (0x1fda, &[0x80, 0x47, 0x00, 0x00, 0x00, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0xe8, 0x03]), + (0x1ffc, &[0x4e, 0x25]), + (0x2008, &[0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x05]), + (0x2068, &[0x5c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0x4c, 0x3f, 0xb6, 0xf3, 0x7d, 0x3f, 0xcd, 0xcc, 0x4c, 0x3e, 0x6f, 0x12, 0x03, 0x3c, 0xd8, 0xd4, 0x69, 0x3f, 0xd8, 0xd4, 0x69, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x42, 0xc3, 0xf5, 0x64, 0x40, 0xc3, 0xf5, 0x64, 0x40, 0x36, 0x94, 0x17, 0x41, 0x64, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x64]), + (0x20cc, &[0x5c]), + (0x2100, &[0x64]), + (0x2120, &[0x01, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x1f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xa0, 0x0f, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xd0, 0x07, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x01]), + (0x21a4, &[0x01]), + (0x21d0, &[0x01]), + ], +]; +pub(crate) const BUNDLE_VIEW_U16_REPEATS: &[(usize, usize, usize, bool)] = &[ + (2, 0x2ad7, 512, false), + (3, 0x3257, 148, true), + (4, 0x31d7, 148, true), +]; + +pub(crate) const BUNDLE_STATIC_RUNS: &[(usize, &[u8])] = &[ + (0x80ac, &[0x06]), + (0x80b8, &[0x01]), + (0x8177, &[0xac, 0x0d]), + (0x817b, &[0xe8, 0x03]), + (0x817f, &[0xb8, 0x0b]), + (0x8183, &[0xe8, 0x03]), + (0x8187, &[0xe8, 0x03]), + (0x818b, &[0x64]), + (0x818f, &[0x20]), + (0x81b4, &[0x04]), + (0x81c2, &[0x78, 0x3f]), + (0x81cb, &[0x3d]), + (0x81d2, &[0xa0, 0x40]), + (0x81de, &[0x80, 0x47]), + (0x81e2, &[0x70, 0x41]), + (0x81ec, &[0x84, 0x03]), + (0x81f0, &[0xe8, 0x03]), + (0x81f4, &[0xe8, 0x03]), + (0x8200, &[0xe8, 0x03]), + (0x8234, &[0xe8, 0x03]), + (0x8274, &[0x64]), + (0x8290, &[0x2a, 0x08]), + (0x8298, &[0x7d]), + (0x829c, &[0x01]), + (0x8ee0, &[0xae, 0x18]), + (0x8ee8, &[0x01]), + (0xaeec, &[0xcd, 0xcc, 0xc8, 0x41, 0xcd, 0xcc, 0xc8, 0x41]), + (0xb5e4, &[0x01]), + (0xb5e8, &[0xf4, 0x01]), + (0xb5ec, &[0x06]), + (0xb5f0, &[0xd4, 0x30]), + (0xb5f4, &[0xd4, 0x30]), + (0xb5f8, &[0xd4, 0x30]), + (0xb5fc, &[0xd4, 0x30]), + (0xb600, &[0xd4, 0x30]), + (0xb604, &[0xd4, 0x30]), + (0xb608, &[0xa0, 0x0f]), + (0xb61c, &[0xd4, 0x30]), + (0xb620, &[0xd4, 0x30]), + (0xb624, &[0xd4, 0x30]), + (0xb628, &[0xd4, 0x30]), + (0xb62c, &[0xd4, 0x30]), + (0xb630, &[0x01]), + (0xb7a8, &[0x01]), + (0xb7b4, &[0xcd, 0xcc, 0x4c, 0x42]), + (0xb7f5, &[0x80, 0x06, 0x44]), + (0xb834, &[0x9a, 0x99, 0xc9, 0x41]), + (0xb8e0, &[0x6a, 0x18]), +]; + +pub(crate) const PRIMARY_STATUS_B_CONFIG_RUNS: &[(usize, &[u8])] = &[ + (0x010, &[0x0f]), + (0x017, &[0x3f]), + (0x01a, &[0x88, 0x40, 0x28]), + (0x020, &[0x01]), + (0x0c8, &[0xf8, 0x2a]), + (0x0cc, &[0x40, 0x1f]), + (0x0d8, &[0x52, 0xb8, 0xa2, 0x41, 0x19, 0x04, 0x16, 0x41]), + (0x0f0, &[0xac, 0x26]), + (0x0f4, &[0xc8]), + (0x10a, &[0xc8, 0x42]), + (0x10e, &[0xc8, 0x43, 0xc8]), + (0x128, &[0x01]), + (0x12c, &[0x16, 0x26]), + (0x133, &[0x3f, 0xcd, 0xcc, 0xcc, 0x40]), + (0x14c, &[0x01]), + (0x1c0, &[0x01]), + (0x1c4, &[0x01]), + (0x1c8, &[0x04]), + (0x1cc, &[0x01]), + (0x1d0, &[0x01]), + (0x1d4, &[0x01]), + (0x1d8, &[0x01]), + (0x4e4, &[0x01]), + (0x4e8, &[0xf4, 0x01]), + (0x4ec, &[0x06]), + (0x4f0, &[0xd4, 0x30]), + (0x4f4, &[0xd4, 0x30]), + (0x4f8, &[0xd4, 0x30]), + (0x4fc, &[0xd4, 0x30]), + (0x500, &[0xd4, 0x30]), + (0x504, &[0xd4, 0x30]), + (0x51c, &[0xd4, 0x30]), + (0x520, &[0xd4, 0x30]), + (0x524, &[0xd4, 0x30]), + (0x528, &[0xd4, 0x30]), + (0x52c, &[0xd4, 0x30]), + (0x530, &[0x01]), + (0x58c, &[0x06]), + (0x590, &[0xac, 0x0d]), + (0x594, &[0xe8, 0x03]), + (0x598, &[0xb8, 0x0b]), + (0x59c, &[0x64]), + (0x5cc, &[0x01]), + (0x5d0, &[0x04]), + (0x5dc, &[0x40, 0x1f]), + (0x5e0, &[0xc8]), + (0x5e4, &[0xa0, 0x0f]), + (0x5e8, &[0xc8]), + (0x5ec, &[0xd0, 0x07]), + (0x5f0, &[0xc8]), + (0x5f4, &[0xe8, 0x03]), + (0x5f8, &[0xc8]), + (0x606, &[0x70, 0x41]), + (0x60a, &[0xa0, 0x40, 0x20]), + (0x610, &[0xe8, 0x03]), + (0x620, &[0x84, 0x03]), +]; + +pub(crate) const SECONDARY_ROOT_EXTRA_1_RUNS: &[(usize, &[u8])] = &[ + (0x14, &[0x01]), + (0x2c, &[0x01]), + (0x30, &[0x01]), + (0x40, &[0x01]), + (0x4c, &[0x01]), + (0x50, &[0x6a, 0x18]), +]; + +// ===================================================================================== +// J873 / T8152 / G18G / 26A428 (RTKit_release-3514.0.15) +// ===================================================================================== +// +// Plan M1 steps 1.1-1.7. Everything below is selected by +// `RootProfile::J873G18G26A428` or carries a `G18G_26A428`/`g18g_26a428` name. +// Nothing above changes its output for J700 or J813: the M5 constants stay +// literal, and `g18_initdata_test.rs` proves that the layout functions below, +// run over the 25G76 kext tables, reproduce those constants bit for bit, and +// that every J700/J813 encoder output hash is unchanged. +// +// Evidence tags: +// FW_PROVEN read off armfw_g18g-b000 / armfw1_g18g-b010 (J873 loads the B +// images; ADT iop nub UUIDs match them) +// KEXT_PROVEN read off the 26A428 AGXG18G kext in the kernelcache +// ADT_PROVEN read off the J873 ADT (full38-live-adt.bin) +// INFERRED carried from the measured J813/25G76 reference through a +// proven relocation, or matched by analogy; the value itself is +// not proven for 26A428 +// GUESSED no evidence for the value; a placeholder +// A relocation band's tag qualifies the offset mapping, not the carried value. +// Every INFERRED/GUESSED value is also a row of [`G18G_26A428_UNPROVEN`]. + +/// Root `+0x00`: u16 {0x04c0, 0x83a0, 0x4339, 0x0c8b}. FW_PROVEN: b000 +/// 0x2164c-0x21660 and b010 0x294f0-0x29504 compare root+0 with this word and +/// abort with "Host and FW Interface is mismatched" otherwise. KEXT_PROVEN: +/// AGXArmFirmware::initFirmwareData (0xfffffe0008c31f6c) writes it to both +/// roots. +pub(crate) const G18G_26A428_INITDATA_VERSION_WORD: u64 = 0x0c8b_4339_83a0_04c0; +/// Both roots are 0xc8 bytes. FW_PROVEN: each image copies 0xc8 bytes of its +/// root (b000 0x21664-0x216a8) and requires root+0x2c != 0 (b000 0x216ac). +/// Root field offsets are unchanged from 25G76 (KEXT_PROVEN g18g_key.c:3748). +pub(crate) const G18G_26A428_ROOT_SIZE: usize = 0xc8; + +/// Provenance of one G18G relocation band or unproven field. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum Evidence { + FirmwareProven, + KextProven, + AdtProven, + Inferred, + Guessed, +} + +// --- Kext allocation tables and the packing rule ----------------------------------- + +/// `AGXFirmware::allocFirmwareData` allocation-size table, 24 u64. +/// KEXT_PROVEN: 25G76 __const 0xfffffe0007139820. Index names are the +/// `KX_*` constants below. +pub(crate) const KEXT_ALLOCATION_SIZES_25G76: [usize; 24] = [ + 0x2708, 0x0c18, 0x1048, 0x0e10, 0x6a30, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe440, 0x06a8, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e78, 0x00e4, +]; +/// KEXT_PROVEN: 26A428 __const 0xfffffe000714f0d8. Only entries 0 (hw_data), +/// 4 (perf/policy), 12 (status B), 13 (status config) and 22 (region C) +/// differ from 25G76. +pub(crate) const KEXT_ALLOCATION_SIZES_26A428: [usize; 24] = [ + 0x2848, 0x0c18, 0x1048, 0x0e10, 0x7800, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe730, 0x06d0, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e80, 0x00e4, +]; +pub(crate) const KX_HW_DATA: usize = 0; +pub(crate) const KX_BLOCK_C18: usize = 1; +pub(crate) const KX_BLOCK_1048: usize = 2; +pub(crate) const KX_BLOCK_E10: usize = 3; +pub(crate) const KX_POLICY: usize = 4; +pub(crate) const KX_BLOCK_68: usize = 5; +pub(crate) const KX_BLOCK_800: usize = 6; +pub(crate) const KX_BLOCK_88: usize = 7; +pub(crate) const KX_DM_RING: usize = 8; +pub(crate) const KX_MAIN_CONFIG: usize = 9; +pub(crate) const KX_CONTROL_RING: usize = 10; +pub(crate) const KX_RING_STATE: usize = 11; +pub(crate) const KX_STATUS_B: usize = 12; +pub(crate) const KX_STATUS_CONFIG: usize = 13; +pub(crate) const KX_STATUS_A: usize = 14; +pub(crate) const KX_LOG_STATE: usize = 15; +pub(crate) const KX_RING_4800: usize = 16; +pub(crate) const KX_BUFFER_28800: usize = 17; +pub(crate) const KX_BUFFER_79800: usize = 18; +pub(crate) const KX_RING_9000: usize = 19; +pub(crate) const KX_SECONDARY_EXTRA_1: usize = 20; +pub(crate) const KX_POLICY_STATE: usize = 21; +pub(crate) const KX_REGION_C: usize = 22; +pub(crate) const KX_AON_PTD: usize = 23; +/// `getSizeOfFWBRNTable` returns 0 in both kexts (KEXT_PROVEN +/// 0xfffffe00089ee74c / 0xfffffe0008c507e4). +pub(crate) const KEXT_BRN_TABLE_SIZE: usize = 0; +/// Both shared allocations are rounded to one 16 KiB page (allocateSharedData). +pub(crate) const KEXT_ALLOCATION_PAGE: usize = 0x4000; +pub(crate) const SHARED_ALLOCATION_ENTRIES: usize = 25; +pub(crate) const PRIVATE_ALLOCATION_ENTRIES: usize = 38; +const ROLE_SET_ENTRIES: usize = 10; + +const fn round_up_usize(value: usize, alignment: usize) -> usize { + (value + alignment - 1) & !(alignment - 1) +} + +/// The allocateSharedData packing rule (KEXT_PROVEN 0xfffffe0008c038d8): +/// each next entry starts at `(previous offset + previous size + 0x3f) & !0x3f`. +pub(crate) const fn next_packed_allocation(offset: usize, size: usize) -> usize { + round_up_usize(offset + size, 0x40) +} + +/// Offsets of `sizes` packed by [`next_packed_allocation`], the first at 0. +pub(crate) const fn packed_allocation_offsets(sizes: &[usize; N]) -> [usize; N] { + let mut out = [0usize; N]; + let mut index = 1; + while index < N { + out[index] = next_packed_allocation(out[index - 1], sizes[index - 1]); + index += 1; + } + out +} + +/// The 25 entries of shared allocation #1 in allocation order: hw_data, the +/// 0xc18/0x1048/0xe10 blocks, perf/policy, the 0x68/0x800 blocks, the BRN +/// table, the 0x88 block, twelve data-master rings, then main config and +/// device-control ring for role 0 and for role 1 (KEXT_PROVEN descriptor +/// build, 25G76 g17g_key.c:1224-1310 / 26A428 g18g_key.c:1236-1320). +pub(crate) const fn shared_allocation_sizes(table: &[usize; 24]) -> [usize; SHARED_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; SHARED_ALLOCATION_ENTRIES]; + sizes[0] = table[KX_HW_DATA]; + sizes[1] = table[KX_BLOCK_C18]; + sizes[2] = table[KX_BLOCK_1048]; + sizes[3] = table[KX_BLOCK_E10]; + sizes[4] = table[KX_POLICY]; + sizes[5] = table[KX_BLOCK_68]; + sizes[6] = table[KX_BLOCK_800]; + sizes[7] = KEXT_BRN_TABLE_SIZE; + sizes[8] = table[KX_BLOCK_88]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + sizes[9 + ring] = table[KX_DM_RING]; + ring += 1; + } + sizes[21] = table[KX_MAIN_CONFIG]; + sizes[22] = table[KX_CONTROL_RING]; + sizes[23] = table[KX_MAIN_CONFIG]; + sizes[24] = table[KX_CONTROL_RING]; + sizes +} + +/// The 38 entries of shared allocation #2 (the private cluster): twelve +/// data-master ring states and the role-0 device-control state (0x30 each), +/// status B, status config, the ten-entry per-role set for role 0 and role 1, +/// the role-1 device-control state, the 0x60 secondary-root object and the +/// 8-byte policy state (KEXT_PROVEN 25G76 g17g_key.c:1318-1450 / 26A428 +/// g18g_key.c:1330-1460). +pub(crate) const fn private_allocation_sizes(table: &[usize; 24]) -> [usize; PRIVATE_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; PRIVATE_ALLOCATION_ENTRIES]; + let mut state = 0; + while state <= WORK_CHANNEL_COUNT { + sizes[state] = table[KX_RING_STATE]; + state += 1; + } + sizes[13] = table[KX_STATUS_B]; + sizes[14] = table[KX_STATUS_CONFIG]; + let role_set = [ + table[KX_STATUS_A], table[KX_RING_STATE], table[KX_LOG_STATE], table[KX_RING_STATE], + table[KX_RING_STATE], table[KX_RING_4800], table[KX_BUFFER_28800], + table[KX_BUFFER_79800], table[KX_RING_9000], table[KX_RING_4800], + ]; + let mut role = 0; + while role < 2 { + let mut entry = 0; + while entry < ROLE_SET_ENTRIES { + sizes[15 + role * ROLE_SET_ENTRIES + entry] = role_set[entry]; + entry += 1; + } + role += 1; + } + sizes[35] = table[KX_RING_STATE]; + sizes[36] = table[KX_SECONDARY_EXTRA_1]; + sizes[37] = table[KX_POLICY_STATE]; + sizes +} + +/// Offsets (from the start of the shared allocation #1 backing) of every +/// object in it. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedClusterLayout { + pub(crate) hw_data: usize, + pub(crate) block_c18: usize, + pub(crate) block_1048: usize, + pub(crate) block_e10: usize, + pub(crate) policy: usize, + pub(crate) block_68: usize, + pub(crate) block_800: usize, + pub(crate) brn_table: usize, + pub(crate) block_88: usize, + /// Data-master rings in allocation order (see [`WORK_CHANNEL_DM_INDEX`]). + pub(crate) dm_rings: [usize; WORK_CHANNEL_COUNT], + pub(crate) main_config: [usize; 2], + pub(crate) control_ring: [usize; 2], + pub(crate) end: usize, + pub(crate) total: usize, +} + +/// One role's ten-entry set in the private cluster. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RoleSetLayout { + /// Root `+0xa8` (0x20 bytes). + pub(crate) status_a: usize, + /// Event report state (report channel 13 state 0). + pub(crate) event_state: usize, + /// 0x1b0 firmware-log control block (report channel 13 state 2). + pub(crate) log_state: usize, + /// Trace report state (report channel 14 state 0). + pub(crate) trace_state: usize, + pub(crate) state_30: usize, + /// 256 x 0x48 event ring (report channel 13 state 1). + pub(crate) event_ring: usize, + pub(crate) buffer_28800: usize, + /// 0x79800 buffer (main +0x200 / channel 15 state 0). + pub(crate) buffer_79800: usize, + /// 512 x 0x48 trace ring (report channel 14 state 1). + pub(crate) trace_ring: usize, + pub(crate) buffer_4800: usize, +} + +/// Offsets (from the start of shared allocation #2) of every object in it. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrivateClusterLayout { + pub(crate) dm_ring_states: [usize; WORK_CHANNEL_COUNT], + /// Role-0 device-control state (0x30). + pub(crate) control_state: usize, + /// Primary root `+0xb0`. + pub(crate) status_b: usize, + /// Secondary root `+0xb8`. + pub(crate) status_config: usize, + pub(crate) roles: [RoleSetLayout; 2], + /// Role-1 device-control state (0x30). + pub(crate) secondary_control_state: usize, + /// Secondary root `+0xc0` (0x60 bytes). + pub(crate) secondary_extra_1: usize, + /// 8-byte state named by the policy block. + pub(crate) policy_state: usize, + pub(crate) end: usize, + pub(crate) total: usize, +} + +pub(crate) const fn shared_cluster_layout(table: &[usize; 24]) -> SharedClusterLayout { + let sizes = shared_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_rings = [0usize; WORK_CHANNEL_COUNT]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + dm_rings[ring] = offsets[9 + ring]; + ring += 1; + } + let end = offsets[24] + sizes[24]; + SharedClusterLayout { + hw_data: offsets[0], + block_c18: offsets[1], + block_1048: offsets[2], + block_e10: offsets[3], + policy: offsets[4], + block_68: offsets[5], + block_800: offsets[6], + brn_table: offsets[7], + block_88: offsets[8], + dm_rings, + main_config: [offsets[21], offsets[23]], + control_ring: [offsets[22], offsets[24]], + end, + total: round_up_usize(end, KEXT_ALLOCATION_PAGE), + } +} + +const fn role_set_layout(offsets: &[usize; PRIVATE_ALLOCATION_ENTRIES], first: usize) -> RoleSetLayout { + RoleSetLayout { + status_a: offsets[first], + event_state: offsets[first + 1], + log_state: offsets[first + 2], + trace_state: offsets[first + 3], + state_30: offsets[first + 4], + event_ring: offsets[first + 5], + buffer_28800: offsets[first + 6], + buffer_79800: offsets[first + 7], + trace_ring: offsets[first + 8], + buffer_4800: offsets[first + 9], + } +} + +pub(crate) const fn private_cluster_layout(table: &[usize; 24]) -> PrivateClusterLayout { + let sizes = private_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_ring_states = [0usize; WORK_CHANNEL_COUNT]; + let mut state = 0; + while state < WORK_CHANNEL_COUNT { + dm_ring_states[state] = offsets[state]; + state += 1; + } + let end = offsets[37] + sizes[37]; + PrivateClusterLayout { + dm_ring_states, + control_state: offsets[12], + status_b: offsets[13], + status_config: offsets[14], + roles: [role_set_layout(&offsets, 15), role_set_layout(&offsets, 25)], + secondary_control_state: offsets[35], + secondary_extra_1: offsets[36], + policy_state: offsets[37], + end, + total: round_up_usize(end, KEXT_ALLOCATION_PAGE), + } +} + +/// Data-master allocation index of each channel-table work entry (TA_0, 3D_0, +/// CL_0, TA_1, ...). A ring and its 0x30-byte state share the index, which is +/// why the native state grid [`WORK_STATE_GRID_OFFSETS`] is `index * 0x40`. +/// The host object order of both allocations is unchanged in 26A428 (the ring +/// descriptors are this+0x448.. in 25G76 and this+0x438.. in 26A428, same +/// sequence), so the permutation is INFERRED unchanged for G18G. +pub(crate) const WORK_CHANNEL_DM_INDEX: [usize; WORK_CHANNEL_COUNT] = + [3, 7, 11, 2, 6, 10, 1, 5, 9, 0, 4, 8]; + +/// Work-ring bundle offsets in channel-table order for a shared layout. +pub(crate) const fn work_ring_offsets(layout: &SharedClusterLayout) -> [usize; WORK_CHANNEL_COUNT] { + let mut rings = [0usize; WORK_CHANNEL_COUNT]; + let mut channel = 0; + while channel < WORK_CHANNEL_COUNT { + rings[channel] = layout.dm_rings[WORK_CHANNEL_DM_INDEX[channel]]; + channel += 1; + } + rings +} + +/// The 25G76 layouts. Not used by the M5 encoders above (which keep their +/// literal constants); the regression tests prove they agree. +pub(crate) const M5_25G76_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&KEXT_ALLOCATION_SIZES_25G76); +pub(crate) const M5_25G76_PRIVATE_LAYOUT: PrivateClusterLayout = + private_cluster_layout(&KEXT_ALLOCATION_SIZES_25G76); +/// The 26A428 layouts: hw_data 0x2848 at 0, blocks at 0x2880/0x34c0/0x4540, +/// perf/policy 0x7800 at 0x5380, 0x68/0x800 at 0xcb80/0xcc00, BRN and 0x88 at +/// 0xd400, rings from 0xd4c0, main r0 0x1f4c0, control r0 0x1f980, main r1 +/// 0x23980, control r1 0x23e40; status B 0x340, status config 0xea80, status +/// A 0xf180/0xc3440, tail 0x177700/0x177740/0x1777c0. Totals 0x28000 and +/// 0x178000, as in 25G76. +pub(crate) const G18G_26A428_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&KEXT_ALLOCATION_SIZES_26A428); +pub(crate) const G18G_26A428_PRIVATE_LAYOUT: PrivateClusterLayout = + private_cluster_layout(&KEXT_ALLOCATION_SIZES_26A428); + +pub(crate) const G18G_26A428_SHARED_CLUSTER_SIZE: usize = G18G_26A428_SHARED_LAYOUT.total; +pub(crate) const G18G_26A428_PRIVATE_CLUSTER_SIZE: usize = G18G_26A428_PRIVATE_LAYOUT.total; +pub(crate) const G18G_26A428_HW_DATA_SIZE: usize = KEXT_ALLOCATION_SIZES_26A428[KX_HW_DATA]; +pub(crate) const G18G_26A428_POLICY_SIZE: usize = KEXT_ALLOCATION_SIZES_26A428[KX_POLICY]; +pub(crate) const G18G_26A428_STATUS_B_SIZE: usize = KEXT_ALLOCATION_SIZES_26A428[KX_STATUS_B]; +pub(crate) const G18G_26A428_STATUS_CONFIG_SIZE: usize = KEXT_ALLOCATION_SIZES_26A428[KX_STATUS_CONFIG]; +/// The kext allocates 0xe80 bytes of region C (25G76: 0xe78). The host maps +/// a whole page ([`REGION_C_SIZE`]) for both. +pub(crate) const G18G_26A428_REGION_C_ALLOCATION: usize = KEXT_ALLOCATION_SIZES_26A428[KX_REGION_C]; +pub(crate) const G18G_26A428_AON_PTD_SIZE: usize = KEXT_ALLOCATION_SIZES_26A428[KX_AON_PTD]; +/// Bytes of shared allocation #1 before the BRN table: hw_data plus the six +/// blocks the policy encoder owns (25G76 analog: [`M5_25G76_SHARED_PREFIX_SIZE`]). +pub(crate) const G18G_26A428_SHARED_PREFIX_SIZE: usize = G18G_26A428_SHARED_LAYOUT.brn_table; +/// Main `+0x254` views (KEXT_PROVEN 26A428 initFirmwareSharedData: the FW VAs +/// of the 0xc18, 0x1048, 0xe10, 0x68 and 0x800 blocks, this+0x310..0x338). +pub(crate) const G18G_26A428_BUNDLE_VIEW_OFFSETS: [usize; 5] = [ + G18G_26A428_SHARED_LAYOUT.block_c18, + G18G_26A428_SHARED_LAYOUT.block_1048, + G18G_26A428_SHARED_LAYOUT.block_e10, + G18G_26A428_SHARED_LAYOUT.block_68, + G18G_26A428_SHARED_LAYOUT.block_800, +]; +pub(crate) const G18G_26A428_BUNDLE_VIEW_EXTENTS: [usize; 5] = [ + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_C18], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_1048], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_E10], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_68], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_800], +]; +pub(crate) const G18G_26A428_WORK_RING_BUNDLE_OFFSETS: [usize; WORK_CHANNEL_COUNT] = + work_ring_offsets(&G18G_26A428_SHARED_LAYOUT); +pub(crate) const G18G_26A428_MAIN_BUNDLE_OFFSETS: [usize; 2] = G18G_26A428_SHARED_LAYOUT.main_config; +pub(crate) const G18G_26A428_CONTROL_RING_BUNDLE_OFFSETS: [usize; 2] = + G18G_26A428_SHARED_LAYOUT.control_ring; +/// Role-1 main `+0x481`: FW VA of the perf/policy allocation (KEXT_PROVEN). +pub(crate) const G18G_26A428_MAIN_SECONDARY_POLICY_PTR: usize = 0x481; +/// Role-1 main `+0x300`. INFERRED: the J813 value (26A428 source accel+0x10a74). +pub(crate) const G18G_26A428_MAIN_SECONDARY_KIND: u32 = 0xc; + +/// Private-cluster offsets the resources module and live boot need. +pub(crate) const G18G_26A428_PRIVATE_STATUS_A: [usize; 2] = [ + G18G_26A428_PRIVATE_LAYOUT.roles[0].status_a, + G18G_26A428_PRIVATE_LAYOUT.roles[1].status_a, +]; +/// Compact channel-state grid of each role: role 0 starts at the first ring +/// state; role 1's device-control state is the tail 0x30 object, so its +/// (control-only) grid base is that object minus the grid's control offset. +pub(crate) const G18G_26A428_PRIVATE_STATE_GRIDS: [usize; 2] = [ + G18G_26A428_PRIVATE_LAYOUT.dm_ring_states[0], + G18G_26A428_PRIVATE_LAYOUT.secondary_control_state - WORK_STATE_GRID_OFFSETS[CONTROL_CHANNEL_INDEX], +]; +/// Status config inside the status-B span (primary status B .. status A). +pub(crate) const G18G_26A428_STATUS_CONFIG_IN_STATUS_B: usize = + G18G_26A428_PRIVATE_LAYOUT.status_config - G18G_26A428_PRIVATE_LAYOUT.status_b; +/// Bytes from primary status B to primary status A (the object M5 calls +/// [`PRIMARY_STATUS_B_OBJECT_SIZE`], 0xeb00 there). +pub(crate) const G18G_26A428_STATUS_B_SPAN: usize = + G18G_26A428_PRIVATE_LAYOUT.roles[0].status_a - G18G_26A428_PRIVATE_LAYOUT.status_b; +/// The fwctl ring follows its 0x20-byte state in the ArmFirmware fwctl +/// allocation (KEXT_PROVEN {0x20, 0x1800} entries, 64-byte packing). +pub(crate) const G18G_26A428_FWCTL_RING_OFFSET: usize = 0x40; + +// --- Status B and status config (plan step 1.3) ----------------------------------- + +/// KEXT_PROVEN (25G76 0x48e0/0x48e8 stores moved +0x1e0): FW-control state +/// and ring pointers. +pub(crate) const G18G_26A428_STATUS_B_FWCTL_STATE: usize = 0x4ac0; +pub(crate) const G18G_26A428_STATUS_B_FWCTL_RING: usize = 0x4ac8; +/// FW_PROVEN u64 recovery epoch (25G76 +0x48f0). The recovery routine +/// (b000 0x6ef0) increments it as it halts (0x7fb8-0x7fc4), and posts the +/// value it had before as the type-4 event's first payload word +/// (0x79ac/0x7df0). EP 0x21 type 6 (b000 0x255b0-0x255dc) requests a host +/// recovery only when its low 44 bits equal it. Starts zero. +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_EPOCH: usize = 0x4ad0; +/// FW_PROVEN u32 recovery handshake state (25G76 +0x4900). Firmware writes 1 +/// and spins until the host writes 2 (b000 0x7fb0-0x7fe0), recovers, then +/// writes 3 and spins until the host writes 0 (0x80fc-0x8114). Starts zero. +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_STATE: usize = 0x4ae0; +/// FW_PROVEN u32: nonzero when the host requested the recovery (b000 0x7fa8 +/// stores w25 = block 0x68 +4 != 0; 25G76 +0x4910). A host recovery posts +/// no type-4 event. +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_HOST: usize = 0x4af0; +/// FW_PROVEN: 256 x 8-byte recovery-info entries (bit 0 = valid) and the +/// cause words after them. The routine clears 0x828 bytes from +0x4298 before +/// each firmware-detected recovery (b000 0x7df8-0x7e08) and writes the cause +/// at +0x4a9c..+0x4abf (0x7eb0-0x7ee8); 25G76 +0x40b8..+0x48df. +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_INFO: usize = 0x4298; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_INFO_ENTRIES: usize = 0x100; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_EMITTED: usize = 0x4a98; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_SOURCE_COUNT: usize = 0x4aa0; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_COMMAND: usize = 0x4aa8; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_REASON: usize = 0x4aac; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_DETAIL: usize = 0x4ab0; +pub(crate) const G18G_26A428_STATUS_B_RECOVERY_SLOT_MASK: usize = 0x4ab8; +/// FW_PROVEN: the 0x68-byte shared block published at main +0x26c is the +/// recovery request (b000 0x6f18-0x6f44, 0x255c8-0x255dc): +0 dispatcher +/// state, +4 host request, +0xc firmware request, +0x20..+0x4b cause. +pub(crate) const G18G_26A428_RECOVERY_REQUEST_WORDS: usize = 0x68 / 4; +/// The 0x800-byte shared block published at main +0x274: one +/// `{u32 valid, u32 pad, u64 stamp}` last-submitted entry per QID. Native +/// `restartWorkQueue` writes it before resuming a halted firmware, which +/// rebuilds queue progress from it (G17 analysis, g17_resources.rs). +pub(crate) const G18G_26A428_QUEUE_PROGRESS_ENTRIES: usize = 128; +pub(crate) const G18G_26A428_QUEUE_PROGRESS_STRIDE: usize = 0x10; +/// FW_PROVEN: the region-C loader reads it (b000 0x36a0/0x377c); KEXT_PROVEN +/// value 1 (25G76 +0xe434). +pub(crate) const G18G_26A428_STATUS_B_CONFIG_HEADER: usize = 0xe714; +/// KEXT_PROVEN new cross-link pair (g18g_key.c:813-816): u32 0 at +0xe71c and +/// the FW VA of status config +0x6c4 at +0xe720. +pub(crate) const G18G_26A428_STATUS_B_LINK_WORD: usize = 0xe71c; +pub(crate) const G18G_26A428_STATUS_B_LINK_PTR: usize = 0xe720; +/// KEXT_PROVEN: status config +0x6c4 = 1 and +0x6c8 = FW VA of status B +0xe71c. +pub(crate) const G18G_26A428_STATUS_CONFIG_LINK_WORD: usize = 0x6c4; +pub(crate) const G18G_26A428_STATUS_CONFIG_LINK_PTR: usize = 0x6c8; +/// KEXT_PROVEN submit*Channel polls (25G76 +0x670/+0x678/+0x680). Written by +/// firmware at run time; they must start zero. +pub(crate) const G18G_26A428_STATUS_CONFIG_SUBMIT_POLLS: [usize; 3] = [0x68c, 0x694, 0x69c]; + +// --- Relocation bands ------------------------------------------------------------- + +/// Bytes `[start, end)` of a 25G76 object move to `[start + delta, end + delta)` +/// of the 26A428 object. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct LayoutBand { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) delta: isize, + pub(crate) evidence: Evidence, +} + +const fn band(start: usize, end: usize, delta: isize, evidence: Evidence) -> LayoutBand { + LayoutBand { start, end, delta, evidence } +} + +/// The 26A428 offset of 25G76 byte `offset`, or None when no band carries it +/// (the byte was removed or its new place is unknown). +pub(crate) const fn relocate_by_bands(bands: &[LayoutBand], offset: usize) -> Option { + let mut index = 0; + while index < bands.len() { + let band = bands[index]; + if band.start <= offset && offset < band.end { + return Some((offset as isize + band.delta) as usize); + } + index += 1; + } + None +} + +/// hw_data 25G76 -> 26A428. The offsets are FW_PROVEN by the init copier +/// (b000 0x21708-0x21d94 vs G17 0x1fb18-0x20124: every G17 read relocates onto +/// a b000 read, see `g18_initdata_test.rs`) and KEXT_PROVEN by the +/// initFirmwareData store alignment. The carried J813 values are INFERRED. +/// 0x640..0xe88 (the 53-slot 25G76 register map) is re-encoded per G18G slot, +/// not relocated. +pub(crate) const G18G_26A428_HW_DATA_BANDS: &[LayoutBand] = &[ + // Host prefix: range head (KEXT_PROVEN same five qwords), +0x28/+0x30/+0x638 + // qwords and the two 32 x 0x18 accelerator tables at +0x38/+0x338 (same + // geometry in both kexts). Not read by the copier; J813 values carried. + band(0x000, 0x640, 0, Evidence::Inferred), + // u64 FW VA of accel+0x1d40 (26A428 writes it at +0xfa0 = "4000"). + band(0xe88, 0xe90, 0x118, Evidence::KextProven), + // Chip block: the copier's 16-byte read moved 0xe90 -> 0xfb0. + band(0xe90, 0xebc, 0x120, Evidence::FirmwareProven), + // 0xebc: 25G76 u32 0 with no 26A428 counterpart (4 bytes removed). + band(0xec0, 0xf5c, 0x11c, Evidence::FirmwareProven), + band(0xf5c, 0xf60, 0x124, Evidence::KextProven), + // 0xf60..0xf68: 25G76 u64 0 removed; 26A428 adds its own u64 0 at +0x1078. + band(0xf68, 0xf98, 0x11c, Evidence::FirmwareProven), + band(0xf98, 0xfac, 0x120, Evidence::FirmwareProven), + // 0xfac: 25G76 flag (G17 copier read 0x1ff60, J813 value 1) has neither a + // 26A428 store nor a b000 read: dropped. + band(0xfb0, 0x25f8, 0x11c, Evidence::FirmwareProven), + band(0x25f8, 0x26b4, 0x120, Evidence::FirmwareProven), + band(0x26b4, 0x26f0, 0x128, Evidence::FirmwareProven), + band(0x26f0, M5_25G76_HW_DATA_SIZE, 0x12c, Evidence::FirmwareProven), +]; + +/// Perf/policy block (relative to the block) 25G76 -> 26A428, from the +/// AGXFirmware::initPowerAndPerformanceData store alignment (KEXT_PROVEN at +/// the band ends; interior bytes without a matched store are INFERRED). +/// 0x6468..0x651c is deliberately absent: 26A428 inserts 0x680 bytes +/// somewhere between the AON PTD pointer (+0x6460 -> +0x64d8) and the boot +/// timestamp (+0x651c -> +0x6c14), and the J813 words there are dropped. +pub(crate) const G18G_26A428_POLICY_BANDS: &[LayoutBand] = &[ + band(0x0000, 0x009c, 0, Evidence::KextProven), + // The 0x160-byte host-config memmove moved +0xa0 -> +0xb0. + band(0x00a0, 0x0200, 0x10, Evidence::KextProven), + band(0x09b8, 0x295c, 0x10, Evidence::KextProven), + band(0x2960, 0x29c1, 0x29, Evidence::KextProven), + band(0x2a01, 0x2a05, 0x8d, Evidence::KextProven), + band(0x2a08, 0x2e5c, 0x90, Evidence::KextProven), + band(0x2e6c, 0x2e70, 0x9c, Evidence::KextProven), + // this+0x216c -> this+0x211c (the host object moved by -0x50). + band(0x2e78, 0x2e7c, 0x74, Evidence::Inferred), + band(0x2f37, 0x2f43, -0x2b, Evidence::KextProven), + band(0x2f43, 0x2f53, 0x71, Evidence::KextProven), + band(0x2f58, 0x3260, 0x70, Evidence::KextProven), + // Policy state pointer and its neighbours (+0x3c98 -> +0x3d30). + band(0x3c98, 0x3cac, 0x98, Evidence::KextProven), + band(0x5cac, 0x5cb4, 0x98, Evidence::KextProven), + band(0x63a4, 0x63f4, 0xa0, Evidence::KextProven), + band(0x6458, 0x645c, 0x78, Evidence::KextProven), + // AON PTD FW VA (this+0xf78 in 26A428). + band(0x6460, 0x6468, 0x78, Evidence::KextProven), + // Boot mach_absolute_time (+0x651c -> +0x6c14) and what follows it. + band(0x651c, 0x66a8, 0x6f8, Evidence::KextProven), + band(0x6a18, 0x6a1c, 0x710, Evidence::KextProven), +]; + +/// Region C 25G76 -> 26A428. FW_PROVEN: every read of the G17 region-C loader +/// (0x3068-0x330c) relocates onto a b000 loader read (0x3508-0x37c0) under +/// this map, and the only extra b000 read is the new field at +0x38. The kext +/// agrees: 26A428 stores 0x78 at +0x40 (25G76 +0x3c) and a feature bit at +0x38. +pub(crate) const G18G_26A428_REGION_C_BANDS: &[LayoutBand] = &[ + band(0x000, 0x038, 0, Evidence::FirmwareProven), + band(0x038, 0xe78, 4, Evidence::FirmwareProven), +]; + +/// Status config 25G76 -> 26A428 (KEXT_PROVEN initFirmwareSharedData store +/// alignment). The J813 words live in the first, sixth and seventh bands. +pub(crate) const G18G_26A428_STATUS_CONFIG_BANDS: &[LayoutBand] = &[ + band(0x000, 0x1dc, 0, Evidence::KextProven), + band(0x4e0, 0x518, 0xc, Evidence::KextProven), + band(0x518, 0x528, 0x14, Evidence::KextProven), + band(0x530, 0x534, 0xc, Evidence::KextProven), + band(0x554, 0x584, 0xc, Evidence::KextProven), + band(0x584, 0x5a8, 0x1c, Evidence::KextProven), + band(0x5cc, 0x628, 0x18, Evidence::KextProven), +]; + +/// J873 secondary power-control inputs from eight native 26A428 capture views. +/// The original initPowerAndPerformanceData stores at 8c30740..8c30760 +/// copy these u32 fields from accelerator+10c10+92c..948/950. GFX1 then +/// consumes them at b010 1071c onward. Do not inherit the J813 controller +/// settings here. Values and store geometry are qualified; semantic names +/// and any relationship to the outstanding compute stall remain unqualified. +const G18G_26A428_STATUS_CONFIG_POWER_WORDS: &[(usize, u32)] = &[ + (0x1c0, 0), (0x1c4, 1), (0x1c8, 1), (0x1cc, 4), (0x1d0, 1), + (0x1d4, 1), (0x1d8, 1), (0x1dc, 1), (0x1e0, 29000), +]; + +/// Remaining fixed J873/26A428 status/config defaults. These typed scalar +/// corrections and the generated tables below make the complete 0x6d0-byte +/// allocation match all eight pre-publication native capture views. Preserve +/// the exact float bit patterns; individual calibration meanings are not +/// qualified. Never use post-run firmware counters as encoder defaults. +const G18G_26A428_STATUS_CONFIG_NATIVE_WORDS: &[(usize, u32)] = &[ + (0x0d8, 0x41a770a4), (0x0dc, 0x3d20045e), (0x12c, 0x8f16), + (0x4e8, 0x00010001), (0x59c, 1), (0x5a0, 700), (0x5a4, 12), + (0x5a8, 5000), (0x5ac, 2000), (0x5b0, 5000), (0x5b4, 0), + (0x5b8, 0), (0x618, 0), (0x628, 1300), (0x638, 1200), +]; + +fn encode_g18g_26a428_status_config_defaults(out: &mut [u8]) { + for words in [G18G_26A428_STATUS_CONFIG_POWER_WORDS, G18G_26A428_STATUS_CONFIG_NATIVE_WORDS] { + for &(offset, value) in words { + put_u32(out, offset, value); + } + } + // Original host stores at 8c30790..8c30804 copy two 16x16 tables + // from accelerator+111c4/+113c4, then two u16 enables at +4e8/+4ea. + // Native J873 tables have zero first rows/columns and linear remaining + // entries. Generate the values; do not carry the different J813 table. + for row in 0..16 { + for col in 0..16 { + let active = row != 0 && col != 0; + let wide = if active { (col * 0x1000 - 1) as u16 } else { 0 }; + let narrow = if active { (col * 0x10 - 1) as u8 } else { 0 }; + put_u16(out, 0x1e8 + 2 * (row * 16 + col), wide); + out[0x3e8 + row * 16 + col] = narrow; + } + } +} + +/// Copy every non-zero byte of `words` (u32 at `offset - source_base`) to its +/// relocated place in `out`. Bytes without a band are dropped. +fn write_relocated_words( + words: &[(usize, u32)], + source_base: usize, + bands: &[LayoutBand], + out: &mut [u8], +) -> Result<(), InitdataError> { + for &(offset, value) in words { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + if byte == 0 { + continue; + } + let source = offset + .checked_sub(source_base) + .ok_or(InitdataError::BufferSize)? + + index; + if let Some(target) = relocate_by_bands(bands, source) { + *out.get_mut(target).ok_or(InitdataError::BufferSize)? = byte; + } + } + } + Ok(()) +} + +// --- hw_data (plan step 1.4) -------------------------------------------------------- + +/// 60 register slots at `+0x640` (FW_PROVEN b000 0x236d8 walks 53 named slot +/// IDs from DATA 0xfffffc0000078bb8; KEXT_PROVEN the 26A428 copy loop runs +/// over 0x3c slots, 0x648..0xfa8). +pub(crate) const G18G_26A428_HW_REGISTER_MAP_SLOTS: usize = 60; +pub(crate) const G18G_26A428_HW_REGISTER_MAP_END: usize = + HW_REGISTER_MAP_OFFSET + G18G_26A428_HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE; +/// KEXT_PROVEN: initFirmwareData forces this slot's device VA (+0xcb0) to 0 +/// (25G76: slot 36, +0xbe8; FenderRegsDFenderPowerTransG16G in both). +pub(crate) const G18G_26A428_ZEROED_REGISTER_SLOT: usize = 41; +/// FW_PROVEN: the copier's 16-byte read of the chip block moved 0xe90 -> 0xfb0. +pub(crate) const G18G_26A428_HW_CHIP_ID_OFFSET: usize = 0xfb0; +pub(crate) const G18G_26A428_HW_CHIP_ID: u32 = 0x8152; +/// KEXT_PROVEN: initPowerAndPerformanceData stores this+0xf10 (the number of +/// performance states; J813 +0xfc4 = 12 = its gpu-num-perf-states) here. +pub(crate) const G18G_26A428_HW_PERF_STATE_COUNT: usize = 0x10e0; +/// KEXT_PROVEN: 16 frequencies in MHz (accel+0x1b8f8.. / 1e6). +pub(crate) const G18G_26A428_HW_PERF_FREQUENCY: usize = 0x10e4; +/// FW_PROVEN: 16 rows of 0x40 bytes; the b000 loop reads lanes 0 and 1 of +/// each row (0x1124/0x1128 core, 0x1524/0x1528 SRAM; b000 0x218d0-0x219f0). +pub(crate) const G18G_26A428_HW_PERF_CORE_VOLTAGE: usize = 0x1124; +pub(crate) const G18G_26A428_HW_PERF_SRAM_VOLTAGE: usize = 0x1524; +pub(crate) const G18G_26A428_HW_PERF_ROWS: usize = 16; +pub(crate) const G18G_26A428_HW_PERF_ROW_STRIDE: usize = 0x40; +pub(crate) const G18G_26A428_HW_PERF_LANES: usize = 16; +pub(crate) const G18G_26A428_HW_PERF_SCALE: usize = 0x1964; +pub(crate) const G18G_26A428_HW_PERF_RELATIVE_A: usize = 0x19e4; +pub(crate) const G18G_26A428_HW_PERF_RELATIVE_B: usize = 0x1a24; +pub(crate) const G18G_26A428_HW_PERF_INDEX_MAP: usize = 0x1ae4; +/// FW_PROVEN: four 0x20-byte KSM completion descriptors (b000 0xf57c `mov +/// w9,#0x2734`), firmware cursor at +0x1c of each (b000 0x1cbc0..0x1cc48). +pub(crate) const G18G_26A428_KSM_COMPLETION_TABLE_OFFSET: usize = 0x2734; +pub(crate) const G18G_26A428_KSM_COMPLETION_CURSOR: usize = 0x1c; +/// FW_PROVEN: QoS pointer (b000 0xa0f0; firmware dereferences QoS+0xe00). +pub(crate) const G18G_26A428_QOS_POINTER_OFFSET: usize = 0x27cc; +/// KEXT_PROVEN new constant (26A428 initFirmwareData `hw+0x2714 = 0x96`). +/// FW_PROVEN numeric use: b000 0x23220 compares it with -1 and scales it by 24; +/// 0xbb90 logs it. (It is not the 25G76 +0x25f4 flag, which moves to +0x2710.) +pub(crate) const G18G_26A428_HW_2714_OFFSET: usize = 0x2714; +pub(crate) const G18G_26A428_HW_2714_VALUE: u32 = 0x96; +/// Fields that exist only in 26A428 (KEXT_PROVEN stores), with their size. +/// +0x2714 is KEXT_PROVEN. Captured scalars below override the zero defaults; +/// the native-owned pointer +0x27d4 remains unresolved and is not copied. +pub(crate) const G18G_26A428_HW_NEW_FIELDS: [(usize, usize); 11] = [ + (0x0fa8, 8), (0x1078, 8), (0x10b4, 4), (0x2714, 4), (0x27d4, 8), (0x2818, 4), + (0x281e, 4), (0x2822, 8), (0x2832, 4), (0x283a, 4), (0x283e, 4), +]; + +/// Stable scalar values from both roles in macOS 26A428 captures 051/053/073/075. +/// Replace the J813 defaults and unknown zero enables only at these offsets. +/// Their values are observed; their complete semantics remain unqualified. +/// Failure boundaries: no pointers, timing values, voltage tables or register +/// writes are copied. audit-g18-hwdata-scalars.py checks all eight snapshots, +/// the float32 ladders, and the full opening records before a hardware trial. +pub(crate) const G18G_26A428_HW_CAPTURED_WORDS: [(usize, u32); 17] = [ + (0x0fb4, 1), (0x10d4, 0x1b), (0x10d8, 4), (0x10dc, 12), + (0x283a, 1), (0x283e, 1), + (0x0fa8, 0x8000), (0x0fac, 0), (0x0fb8, 1), + (0x1044, 12), (0x10a4, 12), (0x2660, 0), (0x2684, 12), + (0x2710, 0), (0x2720, 0), (0x2724, 1), (0x281e, 1), +]; +/// The adjacent +0x281e word is unaligned; +0x281c is only a halfword. +pub(crate) const G18G_26A428_HW_CAPTURED_HALFWORDS: [(usize, u16); 1] = [(0x281c, 1)]; + +/// J873 ADT /arm-io/sgx perf-states: (MHz, core mV), 14 entries including +/// state 0. ADT_PROVEN. +pub(crate) const J873_GPU_PERF_STATES: [(u32, u32); 14] = [ + (0, 125), (502, 675), (669, 700), (834, 720), (1002, 755), (1090, 770), (1178, 795), + (1266, 810), (1354, 840), (1442, 875), (1530, 915), (1620, 960), (1680, 975), (1735, 990), +]; +/// J873 ADT perf-states-sram voltages (mV). ADT_PROVEN. +pub(crate) const J873_GPU_PERF_STATES_SRAM_MV: [u32; 14] = + [775, 775, 775, 775, 775, 775, 795, 810, 840, 875, 915, 960, 975, 990]; +/// J873 ADT gpu-num-perf-states. ADT_PROVEN. +pub(crate) const J873_GPU_NUM_PERF_STATES: u32 = 13; +/// INFERRED: the J813 per-state scale word (1.02f), one per state. +const G18G_26A428_PERF_SCALE_VALUE: u32 = 0x3f82_8f5c; +/// Native 26A428 relative-performance ladders, identical in both roles of +/// captures 051/053/073/075. These are IEEE754 float32 bit patterns, not u32 +/// percentages. Store the bits without floating-point operations in kernel +/// context. All values are finite, monotonic, and bounded by 0..100; the +/// physical calibration formula is not yet derived. Voltage/frequency tables +/// above remain the independently verified J873 ADT values. +const G18G_26A428_PERF_RELATIVE_A: [u32; 14] = + [0, 1100735714, 1103989076, 1107351648, 1109719030, 1110992894, 1112721914, + 1114199739, 1116076971, 1117492892, 1119211535, 1120006268, 1120204862, 1120403456]; +const G18G_26A428_PERF_RELATIVE_B: [u32; 14] = + [0, 0, 1096332557, 1104636123, 1109537981, 1111408920, 1113279858, + 1115150797, 1116353300, 1117288769, 1118224238, 1119180968, 1119818789, 1120403456]; + +/// The 26A428 slot of each 25G76 register slot (FW_PROVEN name tables: G17 +/// DATA 0xfffffc0000073488, 47 rows; b000 DATA 0xfffffc0000078bb8, 53 rows). +/// None: 25G76 slot 8 (StreamingCodecRegisters) has no 26A428 slot. Unnamed +/// slots 17 and 21 keep their numbers; 25 -> 28 and 44/45/46 -> 50/51/52 are +/// INFERRED by position (44 -> 50 is confirmed by the identical die-1 PIO +/// record 0xd24000/0x100 in both kexts). +pub(crate) const G18G_26A428_SLOT_FROM_25G76: [Option; HW_REGISTER_MAP_SLOTS] = [ + Some(0), Some(1), Some(2), Some(3), Some(4), Some(5), Some(6), Some(7), None, Some(9), + Some(10), Some(11), Some(12), Some(13), Some(14), Some(15), Some(16), Some(17), Some(18), + Some(19), Some(20), Some(21), Some(22), Some(23), Some(24), Some(28), Some(31), Some(32), + Some(33), Some(34), Some(35), Some(36), Some(37), Some(38), Some(39), Some(40), Some(41), + Some(42), Some(43), Some(44), Some(45), Some(46), Some(47), Some(48), Some(50), Some(51), + Some(52), Some(53), Some(54), Some(55), Some(56), Some(57), Some(58), +]; + +/// J873 moved the always-on blocks the 25G76 map names (AIC, MCC telemetry, +/// PMGR event log, AON PTD) down by this much (ADT_PROVEN); the SGX and GPU +/// PMGR banks did not move. +pub(crate) const J873_AON_REBASE: u64 = 0x8000_0000; + +/// A register window before its device VA is assigned. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWindowSpec { + pub(crate) slot: usize, + pub(crate) phys: u64, + /// Instance count x instance size (hw_data slot +0x10). + pub(crate) size_10: u32, + /// Instance size (slot +0x14). + pub(crate) size_14: u32, + /// Offset from the GPU base for PIO-table windows (slot +0x18), else 0. + pub(crate) unk_18: u64, + /// accelerator entry flags & 2 (slot +0x20). + pub(crate) flag: u32, +} + +const fn spec(slot: usize, phys: u64, size: u32, unk_18: u64, flag: u32) -> RegisterWindowSpec { + RegisterWindowSpec { slot, phys, size_10: size, size_14: size, unk_18, flag } +} + +/// Assign firmware device VAs in slot order, from the register aperture base: +/// each window starts at the next 32 KiB boundary at least one 16 KiB guard +/// page past the previous window, plus its physical page offset. This rule +/// reproduces all 22 native J813/25G76 device VAs exactly (regression test); +/// for G18G the VAs are host-chosen (INFERRED: same rule), since firmware +/// reaches its registers only through this table. +pub(crate) const fn allocate_register_window_vas( + specs: &[RegisterWindowSpec; N], +) -> [M5RegisterWindow; N] { + let mask = UAT_PAGE_SIZE as u64 - 1; + let mut out = [M5RegisterWindow { + slot: 0, phys: 0, device_va: 0, size_10: 0, size_14: 0, unk_18: 0, flag: 0, + }; N]; + let mut next: u64 = 0; + let mut index = 0; + while index < N { + let window = specs[index]; + let va = next + (window.phys & mask); + out[index] = M5RegisterWindow { + slot: window.slot, + phys: window.phys, + device_va: HW_REGISTER_APERTURE.start + va, + size_10: window.size_10, + size_14: window.size_14, + unk_18: window.unk_18, + flag: window.flag, + }; + let end = (va + window.size_10 as u64 + mask) & !mask; + next = (end + UAT_PAGE_SIZE as u64 + 0x7fff) & !0x7fff; + index += 1; + } + out +} + +/// The 22 mapped G18G windows, in slot order. Sources (26A428 kext): +/// - AcceleratorG::configureDevice (0xfffffe0008c490ac): slot 3 MCache +/// 0x220104000 (0x140fb rounded, count = popcount(active-mcc-bit-vector), +/// absent on J873 -> 1), 9/10 telemetry at AON+0x3d0000/+0x3c0000, 14 PMS at +/// AON+0x280000 size 0x14f54 (25G76: 0x8000), 15 AONPTDSpace AON+0xc840000 +/// 0x24000, 29 DPEP 0x580001800/0x27f8 and 30 ADCLK 0x580ec2200/0xfc (both +/// new), 46 ThrtlStatCntrs GPU+0xe33800. Slots 10/14/29/30 get flags 0. +/// - PI_400 Accelerator::configureDevice (0xfffffe0008c45428): slot 22 +/// GPU+0x1000000/0x8000 and slot 44 MTR GPU+0xde0000, one 0x15000 instance +/// (25G76: GPU+0xe08000, two 0x8000 instances). +/// - AGFPIOGFXHALRelativeOffsetTableG18G (0xfffffe0007152228, 12 records): +/// slot 17 GPU+0 size 0x36000 (25G76 0x21400), Fender slots 31..37/40 at the +/// 25G76 offsets, new 35 (KSMFE, 0xd28000) and 49 (FenderRegsHWL, 0xe18000); +/// +0x18 is the table offset. 25G76 slot 50 (DpwrMXU, 0x14400) is gone. +/// - AGXAccelerator::configureDevice (base): flags 2 for the named slots, +/// 0 for 49; slot 0 {flags 2, count 1} and the AIC SW-interrupt page (the +/// ADT meta-sw-interrupt, 0x301014048 on J873: ADT_PROVEN). +/// - Slot 24 (DVFMMgpu) is absent. Both roles in native 26A428 initialization +/// captures 051/053/073 leave all 0x28 bytes zero. Do not carry the J813 +/// 0x580ec0000 window into this profile or allocate firmware VA space for it. +/// The GPU base is the SGX reg 0x580000000 (unchanged, ADT_PROVEN); the AON +/// base moved 0x380000000 -> 0x300000000 ([`J873_AON_REBASE`]). +pub(crate) const G18G_26A428_REGISTER_WINDOW_SPECS: [RegisterWindowSpec; 21] = [ + spec(0, 0x3_0101_4000, 0x4000, 0, 2), + spec(3, 0x2_2010_4000, 0x18000, 0, 2), + spec(9, 0x3_003d_0000, 0x4000, 0, 2), + spec(10, 0x3_003c_0000, 0x4000, 0, 0), + spec(14, 0x3_0028_0000, 0x14f54, 0, 0), + spec(15, 0x3_0c84_0000, 0x24000, 0, 2), + spec(17, 0x5_8000_0000, 0x36000, 0, 2), + spec(22, 0x5_8100_0000, 0x8000, 0, 2), + spec(29, 0x5_8000_1800, 0x27f8, 0, 0), + spec(30, 0x5_80ec_2200, 0xfc, 0, 0), + spec(31, 0x5_80d0_4000, 0x8000, 0xd0_4000, 2), + spec(32, 0x5_80d0_d000, 0x1000, 0xd0_d000, 2), + spec(33, 0x5_80d5_0000, 0x10000, 0xd5_0000, 2), + spec(34, 0x5_80d1_0000, 0x4000, 0xd1_0000, 2), + spec(35, 0x5_80d2_8000, 0x4000, 0xd2_8000, 2), + spec(36, 0x5_80d4_0000, 0x4000, 0xd4_0000, 2), + spec(37, 0x5_80d6_0000, 0x8000, 0xd6_0000, 2), + spec(40, 0x5_80e0_0000, 0x4000, 0xe0_0000, 2), + spec(44, 0x5_80de_0000, 0x15000, 0, 2), + spec(46, 0x5_80e3_3800, 0x4000, 0, 2), + spec(49, 0x5_80e1_8000, 0x4000, 0xe1_8000, 0), +]; +pub(crate) const G18G_26A428_REGISTER_WINDOWS: [M5RegisterWindow; 21] = + allocate_register_window_vas(&G18G_26A428_REGISTER_WINDOW_SPECS); +/// Slots whose base-configured flags are 2 but which receive no physical +/// window (KEXT_PROVEN by decoding AGXAccelerator::configureDevice; the same +/// decode of the 25G76 function reproduces the J813 list exactly). Slot 47 +/// (AVMR) lost its G-variant window in 26A428; slot 8 keeps a flag although +/// firmware no longer walks it. +pub(crate) const G18G_26A428_REGISTER_FLAG_ONLY_SLOTS: [usize; 20] = + [2, 5, 6, 7, 8, 25, 26, 27, 38, 39, 42, 43, 45, 47, 48, 52, 53, 54, 55, 58]; + +/// Encode the 60-slot, 0x960-byte G18G register map. Construction only: +/// page tables for these windows are the caller's (see g18_resources.rs). +pub(crate) fn encode_g18g_26a428_register_map(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != G18G_26A428_HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for window in G18G_26A428_REGISTER_WINDOWS { + if window.slot >= G18G_26A428_HW_REGISTER_MAP_SLOTS + || window.slot == G18G_26A428_ZEROED_REGISTER_SLOT + || !HW_REGISTER_APERTURE.contains(&window.device_va) + { + return Err(InitdataError::RegisterWindow); + } + let base = window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, window.phys); + put_u64(out, base + 0x08, window.device_va); + put_u32(out, base + 0x10, window.size_10); + put_u32(out, base + 0x14, window.size_14); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + for slot in G18G_26A428_REGISTER_FLAG_ONLY_SLOTS { + put_u32(out, slot * HW_REGISTER_MAP_ENTRY_SIZE + 0x20, 2); + } + Ok(()) +} + +/// Reconstruct the G18G hw_data object (0x2848 bytes) with caller-owned +/// completion-queue and QoS bindings. The J813 platform words are relocated by +/// [`G18G_26A428_HW_DATA_BANDS`]; the register map, chip id, perf tables, +/// completion table and QoS pointer are G18G encodings. No MMIO access. +pub(crate) fn encode_g18g_26a428_hw_data( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + qos_high: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != G18G_26A428_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(qos_high)?; + let qos_end = qos_high + .checked_add(G17P_QOS_RESOURCE_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(qos_end - 1)?; + let mut completion = [0; M5_25G76_COMPLETION_QUEUE_COUNT * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE]; + // INFERRED descriptor format: the J813 one (26A428 fills it through an + // accelerator virtual that was not decoded; the location is FW_PROVEN). + encode_m5_25g76_completion_table(aliases, &mut completion)?; + for alias in aliases { + if qos_high < alias.high + M5_25G76_COMPLETION_BACKING_SIZE as u64 && alias.high < qos_end { + return Err(InitdataError::PairAliasedObject); + } + } + + out.fill(0); + write_relocated_words(M5_25G76_HW_PLATFORM_WORDS, 0, G18G_26A428_HW_DATA_BANDS, out)?; + encode_g18g_26a428_register_map(&mut out[HW_REGISTER_MAP_OFFSET..G18G_26A428_HW_REGISTER_MAP_END])?; + put_u32(out, G18G_26A428_HW_CHIP_ID_OFFSET, G18G_26A428_HW_CHIP_ID); + // INFERRED: this+0xf10 is the number of performance states (J813 12 = + // its gpu-num-perf-states); J873 ADT gpu-num-perf-states is 13. + put_u32(out, G18G_26A428_HW_PERF_STATE_COUNT, J873_GPU_NUM_PERF_STATES); + put_u32(out, G18G_26A428_HW_2714_OFFSET, G18G_26A428_HW_2714_VALUE); + for (offset, value) in G18G_26A428_HW_CAPTURED_WORDS { + put_u32(out, offset, value); + } + for (offset, value) in G18G_26A428_HW_CAPTURED_HALFWORDS { + put_u16(out, offset, value); + } + + // Performance ladder from the J873 ADT (values ADT_PROVEN, locations + // FW_PROVEN/KEXT_PROVEN). Each voltage fills all 16 lanes of its row as on + // J813 (INFERRED); b000 reads lanes 0 and 1. Rows 14 and 15 stay zero. + for (state, &(frequency, core)) in J873_GPU_PERF_STATES.iter().enumerate() { + put_u32(out, G18G_26A428_HW_PERF_FREQUENCY + state * 4, frequency); + put_u32(out, G18G_26A428_HW_PERF_SCALE + state * 4, G18G_26A428_PERF_SCALE_VALUE); + put_u32(out, G18G_26A428_HW_PERF_RELATIVE_A + state * 4, G18G_26A428_PERF_RELATIVE_A[state]); + put_u32(out, G18G_26A428_HW_PERF_RELATIVE_B + state * 4, G18G_26A428_PERF_RELATIVE_B[state]); + for lane in 0..G18G_26A428_HW_PERF_LANES { + let row = state * G18G_26A428_HW_PERF_ROW_STRIDE + lane * 4; + put_u32(out, G18G_26A428_HW_PERF_CORE_VOLTAGE + row, core); + put_u32(out, G18G_26A428_HW_PERF_SRAM_VOLTAGE + row, J873_GPU_PERF_STATES_SRAM_MV[state]); + } + } + // INFERRED: the J813 identity index map over all 16 rows. + for index in 0..G18G_26A428_HW_PERF_ROWS { + put_u32(out, G18G_26A428_HW_PERF_INDEX_MAP + index * 4, index as u32); + } + let table = G18G_26A428_KSM_COMPLETION_TABLE_OFFSET; + out[table..table + completion.len()].copy_from_slice(&completion); + put_u64(out, G18G_26A428_QOS_POINTER_OFFSET, qos_high); + Ok(()) +} + +// --- Perf/policy block and the blocks around it (plan step 1.6) ------------------- + +/// KEXT_PROVEN (26A428 initPowerAndPerformanceData 0xfffffe0008c0807c): the +/// host bzeros 0x7800 bytes, memmoves a 0x160-byte host config to +0xb0 and +/// stores mach_absolute_time() at +0x6c14 (25G76: +0x651c). +pub(crate) const G18G_26A428_POLICY_BOOT_TIMESTAMP: usize = 0x6c14; +/// KEXT_PROVEN: FW VA of the 8-byte private-cluster tail (this+0x3a0) at +/// +0x3d30 (25G76 +0x3c98). +pub(crate) const G18G_26A428_POLICY_STATE_PTR: usize = 0x3d30; +/// KEXT_PROVEN: FW VA of the AON PTD (this+0xf78) at +0x64d8 (25G76 +0x6460). +pub(crate) const G18G_26A428_POLICY_AON_PTD_PTR: usize = 0x64d8; +/// Policy-relative place of the J813 512-entry u16 0x7fff table that +/// [`encode_m5_25g76_bundle_policy`] writes at bundle +0x6ed7. +const M5_25G76_POLICY_U16_TABLE: usize = 0x1c97; +const M5_25G76_POLICY_U16_TABLE_ENTRIES: usize = 512; +/// ADT tunables at the places the 26A428 kext stores them. KEXT_PROVEN +/// locations: +0xb38 = accel+0x11080 (gpu-perf-tgt-utilization), +0xb6c/+0xb70 +/// = getPerformanceControllerIntegralGain(0/1) = accel+0x110a8/0x110ac +/// (gpu-perf-integral-gain/2), +0xb7c/+0xb80 = ProportionalGain(0/1) = +/// accel+0x110b8/0x110bc; +0x22d4/+0x2928/+0x2e1c = this+0xf10 * 100. +/// Values ADT_PROVEN (J873 sgx). +pub(crate) const G18G_26A428_POLICY_ADT_WORDS: [(usize, u32); 8] = [ + (0x0b38, 75), + (0x0b6c, 0x3f92_e148), + (0x0b70, 0x3f92_e148), + (0x0b7c, 0x4092_e148), + (0x0b80, 0x4092_e148), + (0x22d4, J873_GPU_NUM_PERF_STATES * 100), + (0x2928, J873_GPU_NUM_PERF_STATES * 100), + (0x2e1c, J873_GPU_NUM_PERF_STATES * 100), +]; +/// Region-C ADT tunables. KEXT_PROVEN: +0x7c8 = accel+0x106b0 +/// (gpu-idleoff-standby-timer; J813 +0x7c4 carried its ADT 1250) and +0x7e4 = +/// accel+0x106cc (gpu-idleoff-min_conf); ADT_PROVEN values 1500 and 0.8f. +/// +0x9c = this+0xf14 * 100 (25G76 +0x98 = this+0xf40 * 100 = 1200 on J813): +/// INFERRED to be the perf-state count again. +pub(crate) const G18G_26A428_REGION_C_ADT_WORDS: [(usize, u32); 3] = [ + (0x7c8, 1500), + (0x7e4, 0x3f4c_cccd), + (0x09c, J873_GPU_NUM_PERF_STATES * 100), +]; + +/// J873/26A428 pre-handoff policy corrections relative to the relocated +/// J813 baseline. Both roles in four cold captures (051/053/073/075) agree +/// byte-for-byte except the separately encoded boot timestamp. Values are +/// captured calibration/configuration, not inferred field types or MHz. +/// No owned address or timestamp is part of this table. The complete +/// production encoder is compared by a host startup +/// comparison; allocation and execution require live proof. +pub(crate) const G18G_26A428_POLICY_CAPTURED_WORDS: [(usize, u32); 65] = [ + (0x0048, 0x00000514), (0x0088, 0x00000514), + (0x009c, 1), (0x00a0, 1), (0x00a4, 1), (0x00a8, 1), + (0x0104, 0x3f828f5c), (0x09f0, 0x470f1600), + (0x09fc, 0xbd115e00), (0x0a04, 0x00000514), + (0x0a08, 0x00000514), (0x0a14, 0x00008f16), + (0x0a48, 0x00000514), (0x0abc, 0x00000514), + (0x0acc, 0x00008f16), (0x0b4c, 1), + (0x0b84, 0x414a1af3), (0x0b8c, 0x00000514), (0x0ca0, 0), + (0x22ac, 0x42c80000), (0x22b0, 0x44a28000), + (0x22b4, 0x000006c7), (0x22c8, 0x00000064), + (0x2320, 0xff003f80), (0x234c, 0x3d20045e), + (0x235c, 0x41a770a4), (0x236c, 0x00000514), + (0x2370, 0x00000514), (0x23b0, 0x00000514), + (0x25c8, 0x00000020), (0x2810, 0x05140000), + (0x2854, 0x05140000), (0x298c, 0x00040000), + (0x2a88, 0x71480000), (0x2a90, 0x00640000), + (0x2aa0, 0x00008f16), (0x2aa4, 0x00008f16), + (0x2aa8, 0x00008f16), (0x2bd0, 0x00000514), + (0x2bd4, 0x00000514), (0x2bdc, 0x470f1600), + (0x2be0, 0x00008f16), (0x2c14, 0x00000514), + (0x2d60, 0x00000514), (0x2f08, 0x0000000c), + (0x2f0c, 0x00001388), (0x2f10, 0x000007d0), + (0x2f14, 0x00001388), (0x2fb4, 0x00000514), + (0x2fb8, 0x00000514), (0x2fc8, 0), + (0x301c, 0x000004b0), (0x3020, 0x00000514), + (0x3024, 0x00000514), (0x3030, 0x00000514), + (0x3064, 0x00000514), (0x30ac, 0), + (0x30b0, 0x42c80000), (0x30c0, 0xff003f80), + (0x3d38, 0xff003f80), (0x64e0, 0x67305452), + (0x6520, 1), (0x6c6c, 0x43366667), + (0x6cac, 0x44e5c000), (0x6cec, 0x42f00000), +]; + +/// Stable region-C inputs from the same eight initialization snapshots. +/// +0x38 is the new 26A428 feature bit, no longer a guessed zero. +pub(crate) const G18G_26A428_REGION_C_CAPTURED_WORDS: [(usize, u32); 7] = [ + (0x0038, 1), (0x0088, 0x00008f16), (0x008c, 0x00000514), + (0x0090, 0x00000514), (0x09b0, 0x00000200), + (0x09dc, 0x00000032), (0x0e0c, 0x000000fa), +]; + +/// Initialize the six allocations following G18G hw_data (0x2880..0xd400) +/// from the J813 policy words relocated by [`G18G_26A428_POLICY_BANDS`], plus +/// the owned pointers, the boot timestamp and the J873 ADT tunables. Preserves +/// hw_data and everything from the BRN table on. No firmware is started. +pub(crate) fn encode_g18g_26a428_bundle_policy( + bindings: M5PolicyBindings, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = G18G_26A428_SHARED_LAYOUT; + if out.len() < G18G_26A428_SHARED_PREFIX_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(bindings.state)?; + required_address(bindings.aon_ptd)?; + let state_end = bindings.state.checked_add(STATUS_BLOCK_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + let aon_end = bindings.aon_ptd.checked_add(G18G_26A428_AON_PTD_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(state_end - 1)?; + required_address(aon_end - 1)?; + if bindings.state < aon_end && bindings.aon_ptd < state_end { + return Err(InitdataError::PairAliasedObject); + } + out[layout.block_c18..G18G_26A428_SHARED_PREFIX_SIZE].fill(0); + let m5 = M5_25G76_SHARED_LAYOUT; + let m5_policy_end = m5.policy + KEXT_ALLOCATION_SIZES_25G76[KX_POLICY]; + for &(offset, value) in M5_25G76_POLICY_WORDS { + if (m5.block_1048..m5.block_e10).contains(&offset) { + // KEXT_PROVEN (R_1048 stores identical in both kexts): the two + // 0xffffffff words at +0xa18/+0xa30 of the 0x1048 block. + put_u32(out, layout.block_1048 + (offset - m5.block_1048), value); + } else if !(m5.policy..m5_policy_end).contains(&offset) { + return Err(InitdataError::BufferSize); + } + } + let policy = &mut out[layout.policy..layout.policy + G18G_26A428_POLICY_SIZE]; + for &(offset, value) in M5_25G76_POLICY_WORDS { + if !(m5.policy..m5_policy_end).contains(&offset) { + continue; + } + write_relocated_words(&[(offset, value)], m5.policy, G18G_26A428_POLICY_BANDS, policy)?; + } + // The 512 x 0x7fff u16 table (J813 bundle +0x6ed7 = policy +0x1c97) + // moves with its band (INFERRED: no store is matched inside it). + for index in 0..M5_25G76_POLICY_U16_TABLE_ENTRIES { + for (byte_index, byte) in 0x7fffu16.to_le_bytes().into_iter().enumerate() { + let source = M5_25G76_POLICY_U16_TABLE + index * 2 + byte_index; + let target = relocate_by_bands(G18G_26A428_POLICY_BANDS, source) + .ok_or(InitdataError::BufferSize)?; + *policy.get_mut(target).ok_or(InitdataError::BufferSize)? = byte; + } + } + put_u64(policy, G18G_26A428_POLICY_STATE_PTR, bindings.state); + put_u64(policy, G18G_26A428_POLICY_AON_PTD_PTR, bindings.aon_ptd); + put_u64(policy, G18G_26A428_POLICY_BOOT_TIMESTAMP, bindings.boot_timestamp); + for (offset, value) in G18G_26A428_POLICY_ADT_WORDS { + put_u32(policy, offset, value); + } + for (offset, value) in G18G_26A428_POLICY_CAPTURED_WORDS { + put_u32(policy, offset, value); + } + Ok(()) +} + +// --- Region C (plan step 1.5) ------------------------------------------------------- + +/// Encode the G18G region C: the J813 bytes, the 25G76 fields from +0x38 on +/// moved up by 4 (FW_PROVEN), the J873 ADT tunables and stable native J873 +/// inputs. The new +0x38 feature bit (accel+0x6b9 bit 2) is captured as one. +pub(crate) fn encode_g18g_26a428_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + encode_region_c_for_profile(RootProfile::J813G17G25G76, out)?; + // The J813 object never reaches its 0xe78 end; shifting in place keeps the + // tail zero and needs no second 4 KiB buffer on the kernel stack. + let shifted = KEXT_ALLOCATION_SIZES_25G76[KX_REGION_C]; + if out[shifted..].iter().any(|byte| *byte != 0) { + return Err(InitdataError::BufferSize); + } + out.copy_within(0x38..shifted, 0x3c); + out[0x38..0x3c].fill(0); + for (offset, value) in G18G_26A428_REGION_C_ADT_WORDS { + put_u32(out, offset, value); + } + for (offset, value) in G18G_26A428_REGION_C_CAPTURED_WORDS { + put_u32(out, offset, value); + } + // Idle power-off policy word (Region C +0x30). The firmware loads + // it on every cold wake, which undoes the IdlePowerOff(0) sent once at boot. + // 0 keeps the scheduler idle event from powering the cores down between + // jobs. Measured on the M6: 8.6 percent less time per op. + put_u32(out, 0x30, 0); + Ok(()) +} + +// --- Private cluster: status B, status config, status A (plan step 1.3) ---------- + +/// Encode the G18G primary status-B span (status B, status config and the gap +/// to status A; [`G18G_26A428_STATUS_B_SPAN`] bytes) before either role is +/// started. `status_b` is the FW VA of the span, for the new cross-links. +pub(crate) fn encode_g18g_26a428_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + status_b: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != G18G_26A428_STATUS_B_SPAN { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + required_address(status_b)?; + let status_b_end = status_b + .checked_add(G18G_26A428_STATUS_B_SPAN as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(status_b_end - 1)?; + let config = G18G_26A428_STATUS_CONFIG_IN_STATUS_B; + out.fill(0); + // +0x04 = 1: INFERRED from J813 (26A428 stores bit 2 of accel+0x6b0 there). + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, G18G_26A428_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, G18G_26A428_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, G18G_26A428_STATUS_B_CONFIG_HEADER, 1); + put_u32(out, G18G_26A428_STATUS_B_LINK_WORD, 0); + put_u64(out, G18G_26A428_STATUS_B_LINK_PTR, + status_b + (config + G18G_26A428_STATUS_CONFIG_LINK_WORD) as u64); + write_relocated_words(M5_25G76_STATUS_CONFIG_WORDS, 0, G18G_26A428_STATUS_CONFIG_BANDS, + &mut out[config..config + G18G_26A428_STATUS_CONFIG_SIZE])?; + encode_g18g_26a428_status_config_defaults( + &mut out[config..config + G18G_26A428_STATUS_CONFIG_SIZE]); + put_u32(out, config + G18G_26A428_STATUS_CONFIG_LINK_WORD, 1); + put_u64(out, config + G18G_26A428_STATUS_CONFIG_LINK_PTR, + status_b + G18G_26A428_STATUS_B_LINK_WORD as u64); + Ok(()) +} + +/// Encode the whole 0x178000-byte G18G private cluster mapped at +/// `private_base`: status-B span, both status A blocks, the secondary root +/// `+0xc0` object and exactly one opening-producer entry per control channel +/// (plan 1.3: role 0 +0x320, role 1 +0x177720 = the J813 +0x1773e0 moved with +/// the tail). Work and report counters stay zero. +pub(crate) fn encode_g18g_26a428_private_cluster( + private_base: u64, + fwctl_state: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + let layout = G18G_26A428_PRIVATE_LAYOUT; + if out.len() != layout.total { + return Err(InitdataError::BufferSize); + } + required_address(private_base)?; + let private_end = private_base + .checked_add(layout.total as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(private_end - 1)?; + required_address(fwctl_state)?; + let fwctl_ring = fwctl_state + .checked_add(G18G_26A428_FWCTL_RING_OFFSET as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(fwctl_ring)?; + if fwctl_state < private_end && private_base < fwctl_ring + CONTROL_RECORD_SIZE as u64 { + return Err(InitdataError::PairAliasedObject); + } + out.fill(0); + let status_b = layout.status_b; + let status_a = layout.roles[0].status_a; + encode_g18g_26a428_primary_status_b(fwctl_state, fwctl_ring, private_base + status_b as u64, + &mut out[status_b..status_a])?; + for role in layout.roles { + encode_status_block(&mut out[role.status_a..role.status_a + STATUS_BLOCK_SIZE])?; + } + let extra = layout.secondary_extra_1; + encode_secondary_root_extra_1_for_profile(RootProfile::J873G18G26A428, + &mut out[extra..extra + STATUS_BLOCK_SIZE])?; + put_u32(out, layout.control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_PRIMARY_PRODUCER); + put_u32(out, layout.secondary_control_state + CHANNEL_STATE_PRODUCER * CHANNEL_STATE_SPACING, + CONTROL_OPENING_SECONDARY_PRODUCER); + Ok(()) +} + +/// AON PTD (0xe4, KEXT_PROVEN size). INFERRED: the J813 header words. +pub(crate) fn encode_g18g_26a428_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { + encode_m5_25g76_aon_ptd(out) +} + +// --- Device-control and endpoint vocabulary (plan step 1.7) ----------------------- + +/// FW_PROVEN (jump table 0xfffffc0000006c60, 56 entries from opcode 4): the +/// opening/region-C reload handler (b000 0x3ed0) is reached by 0x15, 0x2a +/// and 0x30. +pub(crate) const G18G_26A428_CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x15; +pub(crate) const G18G_26A428_CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; +/// FW_PROVEN: IdlePowerOff keeps its opcode and behaviour (b000 0x4ef0). +pub(crate) const G18G_26A428_DEVICE_CONTROL_OPCODE_IDLE_POWER_OFF: u32 = 0x0a; +pub(crate) const G18G_26A428_DEVICE_CONTROL_OPCODE_MIN: u32 = 4; +pub(crate) const G18G_26A428_DEVICE_CONTROL_OPCODE_MAX: u32 = 0x3b; +/// FW_PROVEN: jump-table entries routed to the default (invalid) handler. +pub(crate) const G18G_26A428_DEVICE_CONTROL_INVALID_OPCODES: [u32; 9] = + [0x19, 0x1c, 0x20, 0x23, 0x26, 0x28, 0x31, 0x39, 0x3a]; + +pub(crate) const fn g18g_26a428_device_control_opcode_valid(opcode: u32) -> bool { + if opcode < G18G_26A428_DEVICE_CONTROL_OPCODE_MIN || opcode > G18G_26A428_DEVICE_CONTROL_OPCODE_MAX { + return false; + } + let mut index = 0; + while index < G18G_26A428_DEVICE_CONTROL_INVALID_OPCODES.len() { + if G18G_26A428_DEVICE_CONTROL_INVALID_OPCODES[index] == opcode { + return false; + } + index += 1; + } + true +} + +// --- Unproven values ------------------------------------------------------------------ + +/// One G18G value that is not proven for 26A428. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UnprovenField { + /// "hw_data", "register_map" (offset = slot), "policy", "region_c", + /// "status_b", "status_config", "main_config", "aon_ptd", + /// "secondary_extra_1". + pub(crate) object: &'static str, + pub(crate) offset: usize, + pub(crate) len: usize, + pub(crate) evidence: Evidence, + pub(crate) note: &'static str, +} + +const fn unproven(object: &'static str, offset: usize, len: usize, evidence: Evidence, + note: &'static str) -> UnprovenField { + UnprovenField { object, offset, len, evidence, note } +} + +/// Every INFERRED or GUESSED G18G value, for review and for the hardware +/// bring-up log. A crashlog assertion should be matched against this list +/// first. +pub(crate) const G18G_26A428_UNPROVEN: &[UnprovenField] = &[ + unproven("hw_data", 0x0000, 0x640, Evidence::Inferred, + "host prefix (+0x28 FW VA, two 32x0x18 accelerator tables) carried from J813"), + unproven("hw_data", 0x0fa0, 8, Evidence::Inferred, "u64 FW VA of accel+0x1d40; J813 0"), + unproven("hw_data", 0x0fa8, 8, Evidence::Inferred, "captured 0x8000; complete semantics unqualified"), + unproven("hw_data", 0x0fb4, 4, Evidence::Inferred, "captured J873 value 1; meaning unknown"), + unproven("hw_data", 0x0fc0, 0x110, Evidence::Inferred, + "J813 relocated words with captured J873 scalar corrections; complete semantics unqualified"), + unproven("hw_data", 0x10b4, 4, Evidence::Guessed, "new u32 (accel byte & 1); left 0"), + unproven("hw_data", 0x10d4, 0xc, Evidence::Inferred, + "captured J873 0x1b/4/12; complete field semantics unqualified"), + unproven("hw_data", 0x10e0, 4, Evidence::Inferred, "13 = gpu-num-perf-states (J813 12 matched by value)"), + unproven("hw_data", 0x1124, 0x800, Evidence::Inferred, + "ADT voltages filled into all 16 lanes of each row; b000 reads lanes 0 and 1"), + unproven("hw_data", 0x1964, 0x38, Evidence::Inferred, "1.02f scale word per state (J813 value)"), + unproven("hw_data", 0x19e4, 0x38, Evidence::Inferred, "native float32 ladder A; calibration formula unqualified"), + unproven("hw_data", 0x1a24, 0x38, Evidence::Inferred, "native float32 ladder B; calibration formula unqualified"), + unproven("hw_data", 0x1ae4, 0x40, Evidence::Inferred, "identity index map 0..15 (J813)"), + unproven("hw_data", 0x2660, 0xd4, Evidence::Inferred, + "J813 relocated words with captured J873 scalar corrections; complete semantics unqualified"), + unproven("hw_data", 0x2734, 0x80, Evidence::Inferred, + "KSM completion descriptor format (capacity 0x800, entry 0x40) carried from J813"), + unproven("hw_data", 0x27d4, 8, Evidence::Guessed, "new u64 (accel+0x12338); left 0"), + unproven("hw_data", 0x27dc, 0x3c, Evidence::Inferred, "J813 +0x26b4..+0x26ec words relocated"), + unproven("hw_data", 0x2818, 4, Evidence::Guessed, "new u32 (accel+0x6c8); read as part of a u64 by b000"), + unproven("hw_data", 0x281e, 0xc, Evidence::Inferred, "captured u32=1/u64=0; complete semantics unqualified"), + unproven("hw_data", 0x2832, 4, Evidence::Guessed, "new mode word, compared with 3 at ~10 b000 sites; left 0"), + unproven("hw_data", 0x283a, 8, Evidence::Inferred, "new enables (accel+0x1165c/+0x1166c bit 0); captured as 1/1"), + unproven("register_map", 0, 1, Evidence::Inferred, "AIC SW-int page from ADT meta-sw-interrupt"), + unproven("register_map", 12, 1, Evidence::Inferred, "ANE0 doorbell left unmapped: 26A428 G variant no longer sets it"), + unproven("register_map", 29, 2, Evidence::Inferred, "DPEP/ADCLK +0x18 left 0; flags 0 mapped read-only like J813 slots 10/14"), + unproven("register_map", 49, 1, Evidence::Inferred, "FenderRegsHWL flags 0 (base sets none) mapped read-only"), + unproven("register_map", 0, 60, Evidence::Inferred, "device VAs assigned by the J813 allocation rule"), + unproven("policy", 0x0000, 0x7800, Evidence::Inferred, + "matches eight native J873 snapshots with owned pointers/timestamp normalized; calibration semantics unqualified"), + unproven("policy", 0x00b0, 0x160, Evidence::Inferred, "captured host config matches; individual field semantics unqualified"), + unproven("policy", 0x1ca7, 0x400, Evidence::Inferred, "512 x 0x7fff u16 table moved with its +0x10 band"), + unproven("policy", 0x0b5c, 0x10, Evidence::Inferred, + "filter-time-constant terms: J813 defaults (J873 ADT has none)"), + unproven("policy", 0x0b9c, 0xc, Evidence::Inferred, + "tgt - dz and dz with dz = 1 carried from J813 (J873 ADT has no gpu-perf-dz)"), + unproven("policy", 0x64e0, 0x9c, Evidence::Inferred, + "native RT0g at +0x64e0 and one at +0x6520; other words zero, complete semantics unqualified"), + unproven("region_c", 0x0000, 0xe80, Evidence::Inferred, + "matches eight native J873 snapshots; individual field semantics unqualified"), + unproven("region_c", 0x0038, 4, Evidence::Inferred, "new feature bit captured as one; runtime effect unqualified"), + unproven("region_c", 0x008c, 8, Evidence::Inferred, "native 1300/1300; field units unqualified"), + unproven("region_c", 0x009c, 4, Evidence::Inferred, "1300 = this+0xf14 x 100 assumed equal to the state count"), + unproven("status_b", 0x0004, 4, Evidence::Inferred, "1 carried (26A428 stores accel+0x6b0 bit 2)"), + unproven("status_config", 0x0000, 0x640, Evidence::Inferred, + "complete 0x6d0 allocation matches eight native J873 pre-boot views; individual calibration semantics unqualified"), + unproven("status_config", 0x01dc, 8, Evidence::Inferred, + "native J873 1/29000 and original host stores verified; semantic names unqualified"), + unproven("status_config", 0x03e8, 0x104, Evidence::Inferred, + "native generated 16x16 u8 table and two u16 enables; original host copy geometry verified"), + unproven("main_config", 0x0300, 4, Evidence::Inferred, "secondary kind 0xc carried (accel+0x10a74)"), + unproven("main_config", 0x02f8, 8, Evidence::Guessed, "third descriptor table (accel[0x235e]) absent: 0"), + unproven("aon_ptd", 0x0000, 0xc, Evidence::Inferred, "J813 PTD header words"), + unproven("secondary_extra_1", 0x0000, 0x60, Evidence::Inferred, "T8140/J813 content (same store geometry)"), +]; + +#[cfg(test)] +#[path = "g18_initdata_test.rs"] +mod g18_tests; + +#[cfg(test)] +mod tests { + #[test] + fn m5_register_extent_uses_size_10_and_rejects_invalid_pages() { + let native = super::M5_25G76_REGISTER_WINDOWS.iter().find(|w| w.slot == 39).unwrap(); + assert_eq!(native.mapped_pages(), Ok((0x580e08000, 0xfffffc2180128000, 0x10000))); + let mut invalid = *native; + invalid.phys += 1; + assert_eq!(invalid.mapped_pages(), Err(super::InitdataError::RegisterWindow)); + invalid = *native; + invalid.size_10 = 0; + assert_eq!(invalid.mapped_pages(), Err(super::InitdataError::RegisterWindow)); + invalid = *native; + invalid.device_va = u64::MAX & !0x3fff; + assert_eq!(invalid.mapped_pages(), Err(super::InitdataError::AddressOutOfRange)); + invalid = *native; + invalid.phys = u64::MAX & !0x3fff; + assert_eq!(invalid.mapped_pages(), Err(super::InitdataError::AddressOutOfRange)); + } + + #[test] + fn m5_private_arena_rejects_invalid_bindings_before_writing() { + let mut arena = vec![0xa5; super::M5_25G76_PRIVATE_CLUSTER_SIZE]; + for pointer in [0, u64::MAX - 8, 1 << 48] { + assert!(super::encode_m5_25g76_private_cluster(pointer, &mut arena).is_err()); + assert!(arena.iter().all(|&byte| byte == 0xa5)); + } + } + + use super::*; + + const KERN: u64 = 0xffff_fc20_0000_0000; + + fn encoded_fast_die() -> FastDieConfig { + let mut bundle = vec![0u8; HW_DATA_BUNDLE_MIN_ALLOC]; + encode_bundle_static(KERN + 0x1000, &mut bundle).unwrap(); + let mut status_b = vec![0u8; PRIMARY_STATUS_B_OBJECT_SIZE]; + encode_primary_status_b(KERN + 0x2000, KERN + 0x2040, &mut status_b).unwrap(); + fast_die_config(&bundle, &status_b).unwrap() + } + + #[test] + fn fast_die_config_matches_native() { + let c = encoded_fast_die(); + assert_eq!(FASTDIE_OBJECT_BUNDLE_OFFSET, 0x5240); + assert_eq!(c.mask, 0x82a); // G17P mask for accel[0x4a0] != 0x1f + assert_eq!(c.target_cc, FASTDIE_NATIVE_TARGET_CC); // 110.00 C + assert_eq!(c.release_cc, 8000); + assert_eq!(c.delta_cc, 0); + assert_eq!(c.kp_bits, 0x41a2_b852); + assert_eq!(c.ki_dt_bits, 0x4116_0419); + assert_eq!(c.state, 0x4200_0006); // fw1 0x174fc constant + assert_eq!(c.live_target_cc, c.target_cc); + assert_eq!(c.live_release_cc, c.release_cc); + assert_eq!(c.live_kp_bits, c.kp_bits); + assert_eq!(c.live_ki_dt_bits, c.ki_dt_bits); + assert_eq!(c.live_update, 0); + // Same object g17_dvfs_policy::GFX1_STATS_OFFSET (primary state + 0xe780) reads. + assert_eq!(PRIMARY_STATUS_B_STATE_GRID_OFFSET + PRIMARY_STATUS_B_CONFIG_OFFSET, 0xe780); + } + + #[test] + fn fast_die_target_override_is_lower_only() { + assert_eq!(fast_die_target_override(0).unwrap(), None); + assert_eq!(fast_die_target_override(10_000).unwrap(), Some(10_000)); + assert_eq!(fast_die_target_override(8_500).unwrap(), Some(8_500)); + assert!(fast_die_target_override(11_001).is_err()); + assert!(fast_die_target_override(8_499).is_err()); + let mut bundle = vec![0u8; HW_DATA_BUNDLE_MIN_ALLOC]; + encode_bundle_static(KERN + 0x1000, &mut bundle).unwrap(); + let mut status_b = vec![0u8; PRIMARY_STATUS_B_OBJECT_SIZE]; + encode_primary_status_b(KERN + 0x2000, KERN + 0x2040, &mut status_b).unwrap(); + apply_fast_die_target_bundle(&mut bundle, 10_000).unwrap(); + apply_fast_die_target_status_b(&mut status_b, 10_000).unwrap(); + let c = fast_die_config(&bundle, &status_b).unwrap(); + assert_eq!((c.target_cc, c.live_target_cc, c.release_cc, c.mask), (10_000, 10_000, 8000, 0x82a)); + } + + + fn native_compute_bytes(bytes: &[u8]) -> Vec { + let mut native = bytes.to_vec(); + for offset in 0..bytes.len().saturating_sub(7) { + let value = get_u64(bytes, offset); + let address = value & !CONTROL_OPERAND_ENTRY_FLAG; + if (0x70_0200_0000..0x70_03ef_8000).contains(&address) { + put_u64(&mut native, offset, value - COMPUTE_FLIST_RELOCATION); + } + } + native + } + #[test] + fn compute_aperture_ranges_do_not_overlap_each_other_or_render() { + let mut ranges = vec![ + (COMPUTE_FLIST_PAGE_LIST_ADDRESS, 0x20_0000), + (COMPUTE_FLIST_RUN_TABLE_ADDRESS, 0x1_0000), + (COMPUTE_RUNTIME_PAGE_LIST_ADDRESS, 0xc000), + (COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS, 0x4000), + ]; + for i in 0..CONTROL_OPERAND_ENTRY_COUNT { + ranges.push((COMPUTE_FLIST_BUFFER_BASE + i as u64 * CONTROL_OPERAND_BUFFER_STRIDE, + CONTROL_OPERAND_BUFFER_SIZE)); + } + for base in G17P_KSM_COMPLETION_LOW_VAS { + ranges.push((base, G17P_KSM_COMPLETION_BACKING_SIZE as u64)); + } + // Native mature kernel-low ZERO0/ZERO1 intentionally alias that + // mode's main backing, whereas ordinary client mappings retain its + // separate page-list. They are not additional overlapping mappings. + assert_eq!(COMPUTE_RUNTIME_ZERO_BUFFER_0_ADDRESS, COMPUTE_FLIST_PAGE_LIST_ADDRESS); + assert_eq!(COMPUTE_RUNTIME_ZERO_BUFFER_1_ADDRESS, COMPUTE_FLIST_PAGE_LIST_ADDRESS + 0x10_8000); + for (i, &(base, size)) in ranges.iter().enumerate() { + assert_eq!(base & 0x3fff, 0); + assert!(base >= 0x70_0200_0000 && base + size <= 0x70_0400_0000); + for &(other, other_size) in ranges.iter().skip(i + 1) { + assert!(base + size <= other || other + other_size <= base, + "overlap: {base:#x}+{size:#x} with {other:#x}+{other_size:#x}"); + } + } + assert_eq!(CONTROL_OPERAND_TABLE_ADDRESS, 0x70_0020_8000); + assert_eq!(CONTROL_OPERAND_BUFFER_BASE, 0x70_0022_0000); + } + + #[test] + fn compute_support_uses_full_unaligned_page_pointer() { + for encode in [encode_compute_readiness_class1_support, + encode_compute_readiness_class3_support, + encode_compute_runtime_class2_support, + encode_compute_partial_opening_control_shared] { + let mut bytes = vec![0xa5; 0x4000]; + encode(&mut bytes).unwrap(); + assert_eq!(get_u64(&bytes, 0x14), 0x70_0200_0000); + assert_eq!(get_u32(&bytes, 0x1c), 0x4_0000); + assert_eq!(get_u64(&bytes, 0x30), 0x70_0220_8000); + } + } + + fn shared() -> SharedAddresses { + SharedAddresses { + region_a: KERN + 0x1b_8000, + region_c: KERN + 0x1a_0000, + hw_data_bundle: KERN + 0xc078_8000, + hw_data_bundle_alloc: NATIVE_SHARED_CLUSTER_SIZE, + } + } + + fn primary() -> InstanceAddresses { + InstanceAddresses { + root: KERN + 0x1a_8000, + main_config: KERN + 0xc078_8000 + NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET as u64, + status_a: KERN + 0x2_ee40, + status_b: KERN + 0x2_0000 + PRIMARY_STATUS_B_STATE_GRID_OFFSET as u64, + secondary_extras: [0, 0], + } + } + + fn secondary() -> InstanceAddresses { + InstanceAddresses { + root: KERN + 0x1a_8000 + SECONDARY_ROOT_DELTA, + main_config: KERN + 0xc078_8000 + NATIVE_SECONDARY_MAIN_BUNDLE_OFFSET as u64, + status_a: KERN + 0xe_3100, + status_b: 0, + secondary_extras: [KERN + 0x2_e780, KERN + 0x19_7400], + } + } + + fn state_grid(role: InstanceRole) -> u64 { + match role { + InstanceRole::Primary => KERN + 0x2_0000, + InstanceRole::Secondary => KERN + 0x19_70c0, + } + } + + fn channels(role: InstanceRole) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let inst = match role { + InstanceRole::Primary => primary(), + InstanceRole::Secondary => secondary(), + }; + derive_channel_table( + role, + state_grid(role), + inst.status_a, + inst.main_config, + shared().hw_data_bundle, + ) + } + + fn region_views() -> PrimaryRegionViews { + PrimaryRegionViews { + sentinel_high: KERN + 0xc07c_8000, + alias_high: [KERN + 0x15e_0000, KERN + 0x15e_8000], + } + } + + fn encode_root_bytes(role: InstanceRole) -> Vec { + let inst = match role { + InstanceRole::Primary => primary(), + InstanceRole::Secondary => secondary(), + }; + let mut out = vec![0xa5; root_size(role)]; + encode_root(role, &shared(), &inst, &mut out).unwrap(); + out + } + + fn encode_main_bytes(role: InstanceRole) -> Vec { + let inst = match role { + InstanceRole::Primary => primary(), + InstanceRole::Secondary => secondary(), + }; + let views = region_views(); + let views = match role { + InstanceRole::Primary => Some(&views), + InstanceRole::Secondary => None, + }; + let mut out = vec![0xa5; MAIN_CONFIG_SIZE]; + encode_main_config(role, &shared(), &inst, &channels(role), views, &mut out).unwrap(); + out + } + + #[test] + fn descriptor_table_aliases_are_published_in_both_main_configs() { + let views = region_views(); + let primary = encode_main_bytes(InstanceRole::Primary); + assert_eq!(get_u64(&primary, 0x2d8), REGION_VIEW_LOW_ADDRS[0]); + assert_eq!(get_u64(&primary, 0x2e0), views.alias_high[0]); + assert_eq!(get_u64(&primary, 0x2e8), REGION_VIEW_LOW_ADDRS[1]); + assert_eq!(get_u64(&primary, 0x2f0), views.alias_high[1]); + + let secondary = encode_main_bytes(InstanceRole::Secondary); + assert_eq!(get_u64(&secondary, 0x2d8), REGION_VIEW_LOW_ADDRS[0]); + assert_eq!(get_u64(&secondary, 0x2e0), 0); + assert_eq!(get_u64(&secondary, 0x2e8), REGION_VIEW_LOW_ADDRS[1]); + assert_eq!(get_u64(&secondary, 0x2f0), 0); + } + + #[test] + fn per_role_root_sizes_version_tuple_and_handoff_zeroes() { + assert_eq!(ROOT_SIZE_PRIMARY, 0xb8); + assert_eq!(ROOT_SIZE_SECONDARY, 0xc8); + assert_eq!(root_size(InstanceRole::Primary), 0xb8); + assert_eq!(root_size(InstanceRole::Secondary), 0xc8); + // The exact version tuple, both as words and as the u64 the firmware + // compares. The M5 value 0x0c8d_e339_0728_04c0 must never reappear. + assert_eq!(INITDATA_VERSION, [0x04c0, 0x0396, 0xa322, 0x0c8a]); + assert_eq!(INITDATA_VERSION_WORD, 0x0c8a_a322_0396_04c0); + + let root = encode_root_bytes(InstanceRole::Primary); + assert_eq!(root.len(), 0xb8); + assert_eq!(get_u64(&root, 0x00), INITDATA_VERSION_WORD); + assert_eq!(get_u64(&root, 0x08), shared().region_a); + // +0x10 is zero at handoff even though the buffer started dirty. + assert_eq!(get_u64(&root, 0x10), 0); + assert_eq!(get_u64(&root, 0x18), primary().main_config); + assert_eq!(get_u64(&root, 0x20), shared().region_c); + assert_eq!(get_u32(&root, 0x28), 0); + assert_eq!(get_u32(&root, 0x2c), 1); + assert_eq!(get_u64(&root, 0xa8), primary().status_a); + assert_eq!(get_u64(&root, 0xb0), primary().status_b); + assert!(check_firmware_root_gates(&root).is_ok()); + + let root = encode_root_bytes(InstanceRole::Secondary); + assert_eq!(root.len(), 0xc8); + assert_eq!(get_u32(&root, 0x28), 1); + assert_eq!(get_u64(&root, 0xa8), secondary().status_a); + // Status B is real only on the primary. + assert_eq!(get_u64(&root, 0xb0), 0); + assert_eq!(get_u64(&root, 0xb8), secondary().secondary_extras[0]); + assert_eq!(get_u64(&root, 0xc0), secondary().secondary_extras[1]); + assert!(check_firmware_root_gates(&root).is_ok()); + } + + #[test] + fn uat_geometry_block_and_level_descriptors() { + let root = encode_root_bytes(InstanceRole::Primary); + // Page size 0x4000 (u16), page bits 14, level count 3. + assert_eq!( + u16::from_le_bytes(root[0x30..0x32].try_into().unwrap()), + 0x4000 + ); + assert_eq!(root[0x32], 14); + assert_eq!(root[0x33], 3); + // Three 0x20-byte descriptors: shifts 36/25/14, entries 64/2048/2048. + let expected = [(36u8, 64u16), (25, 2048), (14, 2048)]; + for (index, (shift, entries)) in expected.iter().enumerate() { + let base = 0x34 + index * 0x20; + assert_eq!(root[base], 8); + assert_eq!(root[base + 1], 14); + assert_eq!(root[base + 2], 14); + assert_eq!(root[base + 3], *shift); + assert_eq!( + u16::from_le_bytes(root[base + 4..base + 6].try_into().unwrap()), + *entries + ); + assert_eq!( + u16::from_le_bytes(root[base + 6..base + 8].try_into().unwrap()), + 0x4000 + ); + assert_eq!(get_u64(&root, base + 0x08), 1); + assert_eq!(get_u64(&root, base + 0x10), 0x0000_03ff_ffff_c000); + assert_eq!(get_u64(&root, base + 0x18), (*entries as u64 - 1) << *shift); + } + // Spot-check the derived index masks against their absolute values. + assert_eq!(get_u64(&root, 0x34 + 0x18), 0x3f_0000_0000_0 >> 4 << 4); // 63 << 36 + assert_eq!(get_u64(&root, 0x34 + 0x18), 63u64 << 36); + assert_eq!(get_u64(&root, 0x54 + 0x18), 2047u64 << 25); + assert_eq!(get_u64(&root, 0x74 + 0x18), 2047u64 << 14); + // The descriptor block ends at 0x94; the gap to status A stays zero. + assert!(root[0x94..0xa8].iter().all(|byte| *byte == 0)); + } + + #[test] + fn main_config_bundle_views_are_views_into_one_allocation() { + assert_eq!(MAIN_CONFIG_SIZE, 0x600); + let main = encode_main_bytes(InstanceRole::Primary); + assert_eq!(main.len(), 0x600); + let bundle = shared().hw_data_bundle; + assert_eq!(get_u64(&main, 0x00), bundle); + // The repeated writable region is bundle + 0xc500, at +0x08 and + // +0x10 — past the 0xc000 principal bundle, which is why the + // allocation must reach 0x10000. + assert_eq!(get_u64(&main, 0x08), bundle + 0xc500); + assert_eq!(get_u64(&main, 0x10), bundle + 0xc500); + assert!(HW_DATA_BUNDLE_MIN_ALLOC >= 0xc500); + // Five unaligned qwords at +0x254: views into the ONE bundle + // allocation at the fixed offsets, not independent objects. + let offsets = [0x2740u64, 0x3380, 0x4400, 0xbc80, 0xbd00]; + let extents = [0x18c0usize, 0x0c80, 0x3c00, 0x3380, 0x3300]; + for (index, (offset, extent)) in offsets.iter().zip(extents).enumerate() { + let view = get_u64(&main, 0x254 + index * 8); + assert_eq!(view, bundle + offset); + // Each view lies wholly inside the single allocation. + assert!((*offset as usize) + extent <= shared().hw_data_bundle_alloc); + } + assert_eq!(BUNDLE_VIEW_OFFSETS.map(|o| o as u64), offsets); + assert_eq!(BUNDLE_VIEW_EXTENTS, extents); + // Views 3 and 4 overlap by all but 0x80 bytes, and view 0 overlaps + // the tail of the hardware-data object — the relations independent + // allocations cannot reproduce. + assert_eq!(0xbc80 + 0x3380, 0xbd00 + 0x3300); + assert!(BUNDLE_VIEW_OFFSETS[0] < HW_DATA_SIZE); + } + + #[test] + fn bundle_allocation_below_minimum_fails_closed() { + let mut small = shared(); + small.hw_data_bundle_alloc = 0xc000; + let mut out = vec![0; ROOT_SIZE_PRIMARY]; + assert_eq!( + encode_root(InstanceRole::Primary, &small, &primary(), &mut out), + Err(InitdataError::BundleAllocationTooSmall) + ); + } + + #[test] + fn channel_table_has_seventeen_entries_with_the_control_grid_at_12() { + assert_eq!(CHANNEL_TABLE_ENTRIES, 17); + assert_eq!(channel_table_entry_range(0), Some(0x20..0x40)); + assert_eq!(channel_table_entry_range(11), Some(0x180..0x1a0)); + // Entry 12 exists. The M5-derived model refused it, misreading its + // state-counter addresses at +0x1a0/+0x1a8 as an unrelated cursor. + assert_eq!(channel_table_entry_range(12), Some(0x1a0..0x1c0)); + assert_eq!(channel_table_entry_range(16), Some(0x220..0x240)); + assert_eq!(channel_table_entry_range(17), None); + + let main = encode_main_bytes(InstanceRole::Primary); + let grid = state_grid(InstanceRole::Primary); + let bundle = shared().hw_data_bundle; + // Work entry 0 (TA_0): three separate state-counter addresses at 0x10 + // spacing; consumer = state 0, producer = state 2. Its ring is a work + // ring in the bundle. + assert_eq!(get_u64(&main, 0x20), grid + 0x0c0); // consumer + assert_eq!(get_u64(&main, 0x28), grid + 0x0d0); + assert_eq!(get_u64(&main, 0x30), grid + 0x0e0); // producer + assert_eq!( + get_u64(&main, 0x38), + bundle + WORK_RING_BUNDLE_OFFSETS[0] as u64 + ); + + // Entry 12 (device control): three counters in the compact grid, and + // the ring embedded at main + 0x4c0, where the opening opcode is staged. + let e12 = channel_table_entry_range(12).unwrap().start; + assert_eq!( + get_u64(&main, e12 + CHANNEL_STATE_CONSUMER * 8), + grid + 0x300 + ); + assert_eq!( + get_u64(&main, e12 + CHANNEL_STATE_PRODUCER * 8), + grid + 0x320 + ); + assert_eq!(get_u64(&main, e12 + 0x18), primary().main_config + 0x4c0); + assert_eq!(get_u32(&main, 0x4c0), CONTROL_OPENING_OPCODE_PRIMARY); + + // Entry 15 is partial: a first state address, no producer, no ring. + let e15 = channel_table_entry_range(15).unwrap().start; + assert_ne!(get_u64(&main, e15), 0); + assert_eq!(get_u64(&main, e15 + 0x08), 0); + assert_eq!(get_u64(&main, e15 + 0x10), 0); + assert_eq!(get_u64(&main, e15 + 0x18), 0); + // Entry 16 is entirely empty. + let e16 = channel_table_entry_range(16).unwrap().start; + assert!(main[e16..e16 + CHANNEL_ENTRY_SIZE] + .iter() + .all(|byte| *byte == 0)); + // The primary carries the +0x3e0 mask; +0x300 (secondary kind) stays 0. + assert_eq!(get_u32(&main, 0x3e0), MAIN_PRIMARY_MASK_VALUE); + assert_eq!(get_u32(&main, 0x32c), 1); + assert_eq!(get_u32(&main, 0x344), 0xabcd_abcd); + assert_eq!(get_u32(&main, 0x300), 0); + } + + #[test] + fn secondary_leaves_work_channels_empty_but_keeps_control() { + let main = encode_main_bytes(InstanceRole::Secondary); + // Entries 0..11 are empty on the control-only peer. + for index in 0..WORK_CHANNEL_COUNT { + let range = channel_table_entry_range(index).unwrap(); + assert!(main[range].iter().all(|byte| *byte == 0)); + } + // Entry 12 control ring is still main + 0x4c0, with the secondary opcode. + let e12 = channel_table_entry_range(12).unwrap().start; + assert_eq!(get_u64(&main, e12 + 0x18), secondary().main_config + 0x4c0); + assert_eq!(get_u32(&main, 0x4c0), CONTROL_OPENING_OPCODE_SECONDARY); + // Secondary-only fields: +0x300 = 4, and the unaligned +0x471 pointer + // into primary view 2. The primary's +0x3e0 mask is absent here. + assert_eq!(get_u32(&main, 0x300), MAIN_SECONDARY_KIND_VALUE); + let extra = shared().hw_data_bundle + + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] as u64 + + SECONDARY_EXTRA_VIEW_OFFSET as u64; + assert_eq!(get_u64(&main, 0x471), extra); + assert_eq!(get_u32(&main, 0x3e0), 0); + } + + #[test] + fn primary_control_opening_is_the_single_native_record() { + let mut opening = vec![0; control_opening_size(InstanceRole::Primary)]; + encode_control_opening(InstanceRole::Primary, &mut opening).unwrap(); + + assert_eq!(control_opening_producer(InstanceRole::Primary), 1); + for slot in 0..1 { + let base = slot * CONTROL_RECORD_SIZE; + assert_eq!(get_u32(&opening, base), CONTROL_OPENING_OPCODE_PRIMARY); + assert!(opening[base + 4..base + CONTROL_RECORD_SIZE] + .iter() + .all(|byte| *byte == 0)); + } + assert_eq!(opening.len(), CONTROL_RECORD_SIZE); + } + + #[test] + fn live_compute_control_tail_matches_the_output_positive_sequence() { + fn fnv1a64(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) + } + + let mut tail = vec![0u8; COMPUTE_RUNTIME_CONTROL_RECORDS * CONTROL_RECORD_SIZE]; + for index in 0..COMPUTE_RUNTIME_CONTROL_RECORDS { + let start = index * CONTROL_RECORD_SIZE; + encode_compute_runtime_control_record( + index, + &mut tail[start..start + CONTROL_RECORD_SIZE], + ) + .unwrap(); + } + assert_eq!(tail.len(), 65 * 0x40); + assert_eq!(fnv1a64(&native_compute_bytes(&tail)), 0x544f_a4b7_bc23_e65a); + assert_eq!(COMPUTE_RUNTIME_CONTROL_FINAL_COUNTER, 66); + + for (index, sequence) in [(0, 0), (53, 53), (55, 54), (56, 55), (58, 56), (64, 62)] { + let base = index * CONTROL_RECORD_SIZE; + assert_eq!(get_u32(&tail, base), 0x2e); + assert_eq!(get_u32(&tail, base + 0x04), sequence); + assert_eq!(get_u32(&tail, base + 0x0c), 0); + } + for (index, class, sequence, operand, slot_offset, count) in [ + ( + COMPUTE_RUNTIME_CLASS1_RECORD_INDEX, + 1, + 54, + COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS, + 0x480, + 0x18, + ), + ( + COMPUTE_RUNTIME_CLASS2_RECORD_INDEX, + 2, + 56, + COMPUTE_FLIST_RUN_TABLE_ADDRESS, + 0x5c0, + 0x28, + ), + ] { + let base = index * CONTROL_RECORD_SIZE; + assert_eq!(get_u32(&tail, base), 0x20); + assert_eq!(get_u32(&tail, base + 0x04), class); + assert_eq!(get_u32(&tail, base + 0x08), 0x3f); + assert_eq!(get_u32(&tail, base + 0x0c), sequence); + assert_eq!(get_u64(&tail, base + 0x14), COMPUTE_RUNTIME_SUPPORT_ADDRESS); + assert_eq!(get_u64(&tail, base + 0x1c), operand); + assert_eq!(get_u64(&tail, base + 0x24), operand + slot_offset); + assert_eq!(get_u32(&tail, base + 0x2c), count); + assert_eq!(get_u32(&tail, base + 0x30), 0); + assert_eq!(get_u32(&tail, base + 0x34), 1); + } + } + + #[test] + fn scheduler_state_release_record_is_exact() { + let mut record = [0xa5; CONTROL_RECORD_SIZE]; + encode_scheduler_state_release_record( + 0xffff_fc20_0123_4000, + [0x11, 0x22, 0x33, 0x44], + &mut record, + ) + .unwrap(); + assert_eq!(get_u32(&record, 0), 0x14); + assert_eq!(get_u32(&record, 4), 0); + assert_eq!(record[0x08], 0x44); + assert_eq!(record[0x09], 0x11); + assert_eq!(record[0x0a], 0x22); + assert_eq!(record[0x0b], 0x33); + assert_eq!(get_u64(&record, 0x0c), 0xffff_fc20_0123_4000); + assert!(record[0x14..].iter().all(|byte| *byte == 0)); + assert!(device_control_asserts_gpu_power(0x14) == Some(true)); + } + + #[test] + fn bootstrap_uma_release_record_is_exact() { + let mut record = [0xa5; CONTROL_RECORD_SIZE]; + encode_bootstrap_uma_release_record(0, 0, &mut record).unwrap(); + assert_eq!(get_u32(&record, 0), CONTROL_BOOTSTRAP_UMA_RELEASE_OPCODE); + assert!(record[4..].iter().all(|byte| *byte == 0)); + + encode_bootstrap_uma_release_record(0x1122_3344_5566_7788, 0xaabb_ccdd, &mut record) + .unwrap(); + assert_eq!(get_u64(&record, 0x04), 0x1122_3344_5566_7788); + assert_eq!(get_u32(&record, 0x0c), 0xaabb_ccdd); + assert!(record[0x10..].iter().all(|byte| *byte == 0)); + } + + #[test] + fn first_cl_readiness_suffix_has_exact_sequences_contexts_and_bytes() { + assert_eq!( + COMPUTE_READINESS_PUBLICATION_ORDER, + [ + ComputeReadinessPublicationStep::ControlRecord(0), + ComputeReadinessPublicationStep::ControlRecord(1), + ComputeReadinessPublicationStep::ControlRecord(2), + ComputeReadinessPublicationStep::InitializeContext2Activation, + ComputeReadinessPublicationStep::ControlRecord(3), + ComputeReadinessPublicationStep::ControlRecord(4), + ComputeReadinessPublicationStep::ControlRecord(5), + ] + ); + let mut records = [[0u8; CONTROL_RECORD_SIZE]; COMPUTE_READINESS_RECORDS]; + for (index, record) in records.iter_mut().enumerate() { + encode_compute_readiness_record(index, record).unwrap(); + } + + for (index, boundary) in COMPUTE_READINESS_BOUNDARIES.iter().enumerate() { + assert_eq!(boundary.index as usize, index); + assert_eq!( + boundary.producer_before, + CONTROL_OPENING_PRIMARY_PRODUCER + index as u32 + ); + assert_eq!( + boundary.producer_after, + CONTROL_OPENING_PRIMARY_PRODUCER + 1 + index as u32 + ); + assert_eq!(get_u32(&records[index], 0x00), boundary.opcode); + let (sequence_offset, context_offset) = if boundary.opcode == 0x20 { + (0x0c, 0x30) + } else { + (0x04, 0x0c) + }; + assert_eq!(get_u32(&records[index], sequence_offset), boundary.sequence); + assert_eq!(get_u32(&records[index], context_offset), boundary.context); + assert_eq!(compute_readiness_boundary(index), Some(*boundary)); + } + assert_eq!(compute_readiness_boundary(COMPUTE_READINESS_RECORDS), None); + + let mut progress = ComputeReadinessProgress::new(); + assert_eq!(progress.retire(5), None); + for boundary in COMPUTE_READINESS_BOUNDARIES { + assert_eq!(progress.next_boundary(), Some(boundary)); + let pending = progress.begin_publication(boundary).unwrap(); + assert_eq!(pending.pending_target(), Some(boundary.producer_after)); + assert_eq!(pending.next_boundary(), None); + assert_eq!(pending.begin_publication(boundary), None); + assert_eq!(pending.retire(boundary.producer_before), None); + progress = pending.retire(boundary.producer_after).unwrap(); + assert_eq!(progress.retired_records(), boundary.index as usize + 1); + } + assert!(progress.complete()); + assert_eq!(progress.next_boundary(), None); + + for (index, class, sequence, support, slot, count, context) in [ + ( + 0, + 1, + 35, + COMPUTE_READINESS_CLASS1_SUPPORT_ADDRESS, + 0x440, + 0x20, + 1, + ), + ( + 3, + 3, + 37, + COMPUTE_READINESS_CLASS3_SUPPORT_ADDRESS, + 0x5c0, + 0x28, + 2, + ), + ] { + let record = &records[index]; + assert_eq!(get_u32(record, 0x00), 0x20); + assert_eq!(get_u32(record, 0x04), class); + assert_eq!(get_u32(record, 0x08), 0x3f); + assert_eq!(get_u32(record, 0x0c), sequence); + assert_eq!(get_u64(record, 0x14), support); + assert_eq!(get_u64(record, 0x1c), COMPUTE_FLIST_RUN_TABLE_ADDRESS); + assert_eq!(get_u64(record, 0x24), COMPUTE_FLIST_RUN_TABLE_ADDRESS + slot); + assert_eq!(get_u32(record, 0x2c), count); + assert_eq!(get_u32(record, 0x30), context); + assert_eq!(get_u32(record, 0x34), 1); + } + + for (index, sequence, context) in [(1, 35, 1), (2, 36, 1), (4, 37, 2), (5, 38, 2)] { + let record = &records[index]; + assert_eq!(get_u32(record, 0x00), 0x2e); + assert_eq!(get_u32(record, 0x04), sequence); + assert_eq!(get_u32(record, 0x0c), context); + assert!(record[0x10..].iter().all(|byte| *byte == 0)); + } + assert_eq!( + COMPUTE_READINESS_FINAL_COUNTER, + CONTROL_OPENING_PRIMARY_PRODUCER + COMPUTE_READINESS_RECORDS as u32 + ); + assert!(encode_compute_readiness_record(COMPUTE_READINESS_RECORDS, &mut records[0]) + .is_err()); + } + + #[test] + fn first_cl_readiness_support_state_and_activation_pages_are_exact() { + fn expected_support( + class: u32, + active: u32, + resource_class: u64, + cursor: u32, + state: u64, + final_kind: u32, + ) -> [u8; COMPUTE_READINESS_PAGE_SIZE] { + let mut expected = [0u8; COMPUTE_READINESS_PAGE_SIZE]; + put_u32(&mut expected, 0x00, class); + put_u32(&mut expected, 0x08, active); + put_u32(&mut expected, 0x10, class); + put_u64(&mut expected, 0x14, COMPUTE_FLIST_PAGE_LIST_ADDRESS); + put_u32(&mut expected, 0x1c, 0x4_0000); + put_u64(&mut expected, 0x20, resource_class << 40); + put_u64(&mut expected, 0x28, resource_class << 40); + put_u64(&mut expected, 0x30, COMPUTE_FLIST_RUN_TABLE_ADDRESS); + put_u32(&mut expected, 0x40, 4); + put_u32(&mut expected, 0x48, cursor); + put_u64(&mut expected, 0x4c, state); + put_u32(&mut expected, 0x60, final_kind); + expected + } + + let mut class1 = [0u8; COMPUTE_READINESS_PAGE_SIZE]; + let mut class3 = [0u8; COMPUTE_READINESS_PAGE_SIZE]; + let mut state = [0xffu8; COMPUTE_READINESS_PAGE_SIZE]; + encode_compute_readiness_class1_support(&mut class1).unwrap(); + encode_compute_readiness_class3_support(&mut class3).unwrap(); + encode_compute_readiness_state(&mut state).unwrap(); + assert_eq!( + class1, + expected_support( + 1, + 1, + 0x11, + 0x88, + COMPUTE_READINESS_CLASS1_STATE_ADDRESS, + 2, + ) + ); + assert_eq!( + class3, + expected_support( + 3, + 2, + 0x17, + 0xb8, + COMPUTE_READINESS_CLASS3_STATE_ADDRESS, + 3, + ) + ); + assert_eq!(get_u32(&state, 0), 1); + assert!(state[4..].iter().all(|byte| *byte == 0)); + + let mut activation = [0xffu8; CONTROL_RECORD_SIZE]; + encode_compute_readiness_activation_record(&mut activation).unwrap(); + let mut expected = [0u8; CONTROL_RECORD_SIZE]; + put_u64(&mut expected, 0x00, 0x0000_0100_0000_ffff); + put_u64(&mut expected, 0x20, 0x0002_0000_0000_0000); + put_u64(&mut expected, 0x30, 0x0000_0000_ff00_0000); + assert_eq!(activation, expected); + assert_eq!(COMPUTE_READINESS_ACTIVATION_RECORD_OFFSET, 0x80); + + let mut operand_table = [0xffu8; CONTROL_OPERAND_TABLE_SIZE]; + encode_compute_readiness_operand_table(&mut operand_table).unwrap(); + for index in 0..COMPUTE_READINESS_OPERAND_ENTRY_COUNT { + assert_eq!( + get_u64(&operand_table, index * CONTROL_OPERAND_ENTRY_STRIDE), + (COMPUTE_FLIST_BUFFER_BASE + + index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + | CONTROL_OPERAND_ENTRY_FLAG, + ); + } + assert!(operand_table + [COMPUTE_READINESS_OPERAND_ENTRY_COUNT * CONTROL_OPERAND_ENTRY_STRIDE..] + .iter() + .all(|byte| *byte == 0)); + } + + #[test] + fn live_compute_control_closure_matches_source_builders() { + fn fnv1a64(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) + } + + let mut class1 = vec![0; COMPUTE_RUNTIME_SUPPORT_SIZE]; + encode_compute_runtime_class1_support(&mut class1).unwrap(); + assert_eq!(fnv1a64(&native_compute_bytes(&class1)), 0xb40b_a63a_16c1_937d); + assert_eq!(get_u32(&class1, 0x00), 1); + assert_eq!(get_u64(&class1, 0x14), COMPUTE_RUNTIME_PAGE_LIST_ADDRESS); + assert_eq!(get_u64(&class1, 0x30), COMPUTE_RUNTIME_CLASS1_TABLE_ADDRESS); + assert_eq!(get_u64(&class1, 0x4c), COMPUTE_RUNTIME_STATE_ADDRESS); + + let mut class2 = vec![0; COMPUTE_RUNTIME_SUPPORT_SIZE]; + encode_compute_runtime_class2_support(&mut class2).unwrap(); + assert_eq!(fnv1a64(&native_compute_bytes(&class2)), 0xdd12_cbe2_2d7c_3d50); + assert_eq!(get_u32(&class2, 0x00), 2); + assert_eq!(get_u64(&class2, 0x30), COMPUTE_FLIST_RUN_TABLE_ADDRESS); + + let mut table = vec![0; COMPUTE_RUNTIME_CLASS1_TABLE_SIZE]; + encode_compute_runtime_class1_table(&mut table).unwrap(); + assert_eq!(fnv1a64(&native_compute_bytes(&table)), 0xc820_66ed_9e50_6c41); + for (index, address) in COMPUTE_RUNTIME_BUFFER_ADDRESSES.into_iter().enumerate() { + assert_eq!(get_u64(&table, index * 0x40), address | (1 << 60)); + } + + let mut inventory = vec![0; COMPUTE_RUNTIME_PAGE_LIST_SIZE]; + encode_compute_runtime_page_inventory(&mut inventory).unwrap(); + assert_eq!(fnv1a64(&native_compute_bytes(&inventory)), 0x9f78_e915_163a_cea5); + assert_eq!(get_u64(&inventory, 0), COMPUTE_RUNTIME_BUFFER_ADDRESSES[0]); + assert_eq!( + get_u64(&inventory, inventory.len() - 8), + COMPUTE_RUNTIME_BUFFER_ADDRESSES[20] + 0xff000 + ); + + let mut dispatch = [0u8; COMPUTE_DISPATCH_RECORD_SIZE]; + encode_compute_dispatch_record(&mut dispatch).unwrap(); + let packed = get_u64(&dispatch, 0); + assert_eq!((packed & ((1 << 41) - 1)) << 7, COMPUTE_FLIST_PAGE_LIST_ADDRESS); + // Compare the capture only after undoing the packed VA relocation. + put_u64(&mut dispatch, 0, packed - (COMPUTE_FLIST_RELOCATION >> 7)); + assert_eq!(fnv1a64(&dispatch), 0x12a0_cf17_a88d_a788); + } + + #[test] + fn secondary_control_opening_is_the_single_native_record() { + let mut opening = vec![0; control_opening_size(InstanceRole::Secondary)]; + encode_control_opening(InstanceRole::Secondary, &mut opening).unwrap(); + + assert_eq!(control_opening_producer(InstanceRole::Secondary), 1); + for slot in 0..CONTROL_OPENING_SECONDARY_RECORDS { + let base = slot * CONTROL_RECORD_SIZE; + let expected = if slot < core::cmp::min(3, CONTROL_OPENING_SECONDARY_RECORDS) { + CONTROL_OPENING_OPCODE_SECONDARY + } else { + CONTROL_OPENING_SECONDARY_CONTINUE_OPCODE + }; + assert_eq!(get_u32(&opening, base), expected); + assert!(opening[base + 4..base + CONTROL_RECORD_SIZE] + .iter() + .all(|byte| *byte == 0)); + } + } + + #[test] + fn generic_control_support_keeps_descriptor_and_runtime_identity() { + let mut shared = vec![0; CONTROL_SHARED_OBJECT_SIZE]; + encode_control_shared(&mut shared).unwrap(); + + assert_eq!(get_u64(&shared, 0x00), 1); + assert_eq!(get_u64(&shared, 0x10), 1); + assert_eq!(get_u64(&shared, 0x18), 0x0004_0000_0000_0070); + assert_eq!(get_u64(&shared, 0x20), 0x0000_1100_0000_0000); + assert_eq!(get_u64(&shared, 0x28), 0x0000_1100_0000_0000); + assert_eq!(get_u64(&shared, 0x30), CONTROL_OPERAND_TABLE_ADDRESS); + assert_eq!( + get_u32(&shared, CONTROL_SHARED_CURSOR_OFFSET), + CONTROL_SHARED_CURSOR_BEFORE + ); + assert_eq!(get_u64(&shared, 0x4c), CONTROL_SHARED_INNER_ADDRESS); + assert_eq!(get_u64(&shared, 0x60), 2); + assert_eq!(CONTROL_SHARED_INNER_PRESENTED, 2); + } + + #[test] + fn compute_partial_support_preserves_hardware_proven_compact_state() { + let mut shared = vec![0; CONTROL_SHARED_OBJECT_SIZE]; + encode_compute_partial_opening_control_shared(&mut shared).unwrap(); + + assert_eq!(get_u64(&shared, 0x00), 1); + assert_eq!(get_u32(&shared, 0x08), u32::MAX); + assert_eq!(get_u32(&shared, 0x10), 2); + assert_eq!(get_u64(&shared, 0x20), 0x0000_1900_0000_0000); + assert_eq!(get_u64(&shared, 0x28), 0x0000_1900_0000_0000); + assert_eq!(get_u64(&shared, 0x30), COMPUTE_FLIST_RUN_TABLE_ADDRESS); + assert_eq!( + get_u32(&shared, CONTROL_SHARED_CURSOR_OFFSET), + COMPUTE_PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE + ); + assert_eq!(COMPUTE_PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE, 0x18); + assert_eq!( + get_u64(&shared, 0x4c), + PARTIAL_OPENING_CONTROL_SHARED_INNER_ADDRESS + ); + assert_eq!(get_u64(&shared, 0x60), 3); + } + + #[test] + fn first_partial_support_has_distinct_address_cursor_and_inner_page() { + let mut shared = vec![0; CONTROL_SHARED_OBJECT_SIZE]; + encode_partial_opening_control_shared(&mut shared).unwrap(); + + assert_eq!(PARTIAL_OPENING_CONTROL_SHARED_ADDRESS, 0xffff_fc20_c082_8000); + assert_eq!(get_u64(&shared, 0x00), 1); + assert_eq!(get_u32(&shared, 0x08), 0); + assert_eq!(get_u32(&shared, 0x0c), 0); + assert_eq!(get_u64(&shared, 0x10), 2); + assert_eq!(get_u64(&shared, 0x20), 0x0000_1100_0000_0000); + assert_eq!(get_u64(&shared, 0x28), 0x0000_1100_0000_0000); + assert_eq!(get_u64(&shared, 0x30), CONTROL_OPERAND_TABLE_ADDRESS); + assert_eq!( + get_u32(&shared, CONTROL_SHARED_CURSOR_OFFSET), + PARTIAL_OPENING_CONTROL_SHARED_CURSOR_BEFORE + ); + assert_eq!( + get_u64(&shared, 0x4c), + PARTIAL_OPENING_CONTROL_SHARED_INNER_ADDRESS + ); + assert_eq!(get_u64(&shared, 0x60), 3); + assert_eq!(PARTIAL_OPENING_CONTROL_SHARED_INNER_PRESENTED, 0); + + encode_render_control_shared(&mut shared).unwrap(); + assert_eq!(get_u64(&shared, 0x00), CONTROL_SHARED_INNER_ADDRESS); + assert_eq!(get_u32(&shared, 0x08), 0); + assert_eq!(get_u64(&shared, 0x10), 2); + assert_eq!(get_u64(&shared, 0x14), 0x70_0000_0000); + assert_eq!(get_u64(&shared, 0x30), CONTROL_OPERAND_TABLE_ADDRESS); + assert_eq!(get_u64(&shared, 0x4c), CONTROL_SHARED_INNER_ADDRESS); + assert_eq!(get_u64(&shared, 0x60), 3); + } + + #[test] + fn presized_render_control_shared_matches_apple_initial_3d_pool() { + let mut shared = [0u8; CONTROL_SHARED_OBJECT_SIZE]; + encode_presized_render_control_shared(&mut shared, 22).unwrap(); + // 25G83 populateInitialFirmwareState for the 3D pool: 0x1600 pages + // at +0x24 and +0x2c, run cursor 0xb0. + assert_eq!(get_u64(&shared, 0x20), 0x0000_1600_0000_0000); + assert_eq!(get_u64(&shared, 0x28), 0x0000_1600_0000_0000); + assert_eq!(get_u32(&shared, CONTROL_SHARED_CURSOR_OFFSET), 0xb0); + assert_eq!(get_u64(&shared, 0x00), CONTROL_SHARED_INNER_ADDRESS); + assert_eq!(get_u64(&shared, 0x4c), CONTROL_SHARED_INNER_ADDRESS); + assert_eq!(get_u64(&shared, 0x60), 3); + } + + #[test] + fn first_partial_operand_table_is_published_between_control_and_work() { + let mut table = vec![0xff; CONTROL_OPERAND_TABLE_SIZE]; + encode_partial_opening_operand_table_pre_control(&mut table).unwrap(); + assert!(table.iter().all(|byte| *byte == 0)); + + encode_partial_opening_operand_table_post_control(&mut table).unwrap(); + for index in 0..PARTIAL_OPENING_CONTROL_OPERAND_ENTRY_COUNT { + assert_eq!( + get_u64(&table, index * CONTROL_OPERAND_ENTRY_STRIDE), + CONTROL_OPERAND_ENTRY_FLAG + | (CONTROL_OPERAND_BUFFER_BASE + + index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + ); + } + assert!(table[PARTIAL_OPENING_CONTROL_OPERAND_ENTRY_COUNT + * CONTROL_OPERAND_ENTRY_STRIDE..] + .iter() + .all(|byte| *byte == 0)); + + encode_control_operand_table(&mut table).unwrap(); + + for index in 0..CONTROL_OPERAND_ENTRY_COUNT { + assert_eq!( + get_u64(&table, index * CONTROL_OPERAND_ENTRY_STRIDE), + CONTROL_OPERAND_ENTRY_FLAG + | (COMPUTE_FLIST_BUFFER_BASE + + index as u64 * CONTROL_OPERAND_BUFFER_STRIDE) + ); + } + assert!(table[CONTROL_OPERAND_ENTRY_COUNT * CONTROL_OPERAND_ENTRY_STRIDE..] + .iter() + .all(|byte| *byte == 0)); + } + + #[test] + fn a_null_producer_on_a_populated_channel_fails_closed() { + // A populated work channel with a null producer (state 2) makes the + // firmware fault on address zero; encode must refuse it. + let mut table = channels(InstanceRole::Primary); + table[0].states[CHANNEL_STATE_PRODUCER] = 0; + let views = region_views(); + let mut out = vec![0; MAIN_CONFIG_SIZE]; + assert_eq!( + encode_main_config( + InstanceRole::Primary, + &shared(), + &primary(), + &table, + Some(&views), + &mut out, + ), + Err(InitdataError::ChannelTableShape) + ); + } + + #[test] + fn role_specific_root_fields_fail_closed() { + // A primary must not carry the secondary-only extras (its root is only + // 0xb8 bytes and cannot even hold them). + let mut inst = primary(); + inst.secondary_extras = [KERN + 0x1000, 0]; + let mut out = vec![0; ROOT_SIZE_PRIMARY]; + assert_eq!( + encode_root(InstanceRole::Primary, &shared(), &inst, &mut out), + Err(InitdataError::RoleSpecificAddress) + ); + // A secondary must not carry a status-B pointer. + let mut inst = secondary(); + inst.status_b = KERN + 0x1000; + let mut out = vec![0; ROOT_SIZE_SECONDARY]; + assert_eq!( + encode_root(InstanceRole::Secondary, &shared(), &inst, &mut out), + Err(InitdataError::RoleSpecificAddress) + ); + } + + #[test] + fn wrong_root_buffer_size_fails_closed() { + // The primary root is 0xb8, not 0xc8; a secondary-sized buffer is wrong. + let mut out = vec![0; ROOT_SIZE_SECONDARY]; + assert_eq!( + encode_root(InstanceRole::Primary, &shared(), &primary(), &mut out), + Err(InitdataError::BufferSize) + ); + } + + fn register_windows() -> Vec { + vec![ + RegisterWindow { + slot: 0, + phys: 0x2_0000_0000, + device_va: HW_REGISTER_APERTURE.start, + size: 0x4000, + unk_18: 0, + flag: 2, + }, + RegisterWindow { + slot: 5, + phys: 0x2_1000_0000, + device_va: HW_REGISTER_APERTURE.start + 0x1_0000, + size: 0x8000, + unk_18: 0, + flag: 2, + }, + ] + } + + fn completion_aliases() -> [G17PKsmCompletionAliases; 2] { + [ + G17PKsmCompletionAliases { + low: G17P_KSM_COMPLETION_LOW_VAS[0], + high: shared().hw_data_bundle + + G17P_KSM_COMPLETION_HIGH_BUNDLE_OFFSETS[0] as u64, + }, + G17PKsmCompletionAliases { + low: G17P_KSM_COMPLETION_LOW_VAS[1], + high: shared().hw_data_bundle + + G17P_KSM_COMPLETION_HIGH_BUNDLE_OFFSETS[1] as u64, + }, + ] + } + + fn sksm_qid_resource_high() -> u64 { + shared().hw_data_bundle + NATIVE_SHARED_CLUSTER_SIZE as u64 + 0x4000 + } + + fn qos_resource_high() -> u64 { + shared().hw_data_bundle + NATIVE_SHARED_CLUSTER_SIZE as u64 + 0x8000 + } + + #[test] + fn hw_data_live_pointers_survive_opaque_constants() { + for qos_resource in [ + 0xffff_fc20_001f_8000, + 0xffff_fc20_013a_c000, + 0xffff_fc20_cabc_4000, + ] { + let aliases = [ + G17PKsmCompletionAliases { + low: 0x70_0240_0000, + high: 0xffff_fc20_c240_0000, + }, + G17PKsmCompletionAliases { + low: 0x70_0280_0000, + high: 0xffff_fc20_c280_0000, + }, + ]; + let windows: Vec<_> = (0..HW_REGISTER_MAP_SLOTS) + .map(|slot| RegisterWindow { + slot, + phys: 0x4_0000_0000 + (slot as u64) * 0x4000, + device_va: HW_REGISTER_APERTURE.start + (slot as u64) * 0x4000, + size: 0x4000, + unk_18: 0, + flag: 2, + }) + .collect(); + let mut hw = vec![0xa5; HW_DATA_SIZE]; + encode_hw_data(&windows, &[], aliases, qos_resource, + sksm_qid_resource_high(), &mut hw).unwrap(); + assert_eq!(get_u64(&hw, G17P_QOS_RESOURCE_LIVE_OFFSET), qos_resource); + assert_eq!(get_u32(&hw, 0x26b4), 1); + assert_eq!(get_u32(&hw, 0x26b8), 1); + for (aliases, ordinal) in aliases.iter().zip(G17P_KSM_COMPLETION_PRESENT_ORDINALS) { + let base = G17P_KSM_COMPLETION_TABLE_OFFSET + + ordinal as usize * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE; + assert_eq!(get_u64(&hw, base), aliases.low); + assert_eq!(get_u64(&hw, base + 8), aliases.high); + } + for window in windows { + let base = HW_REGISTER_MAP_OFFSET + window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + assert_eq!(get_u64(&hw, base), window.phys); + assert_eq!(get_u64(&hw, base + 8), window.device_va); + } + } + } + + #[test] + fn hw_data_opaque_runs_do_not_overlap_live_fields() { + let live = [ + HW_REGISTER_MAP_OFFSET + ..HW_REGISTER_MAP_OFFSET + HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE, + G17P_KSM_COMPLETION_TABLE_OFFSET + ..G17P_KSM_COMPLETION_TABLE_OFFSET + 4 * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE, + G17P_QOS_RESOURCE_LIVE_OFFSET..G17P_QOS_RESOURCE_LIVE_OFFSET + 8, + ]; + for (offset, data) in HW_DATA_OPAQUE_RUNS { + for range in &live { + assert!(*offset + data.len() <= range.start || *offset >= range.end, + "opaque range {offset:#x}..{:#x} overlaps live {range:#x?}", + offset + data.len()); + } + } + } + + #[test] + fn hw_data_size_acceptance_completion_table_and_firmware_state() { + assert_eq!(HW_DATA_SIZE, 0x3db4); + let windows = register_windows(); + let flag_only = [(3usize, 2u32), (7, 2)]; + let aliases = completion_aliases(); + let mut hw = vec![0xa5; HW_DATA_SIZE]; + let qos_resource = qos_resource_high(); + let sksm_qid_resource = sksm_qid_resource_high(); + encode_hw_data( + &windows, + &flag_only, + aliases, + qos_resource, + sksm_qid_resource, + &mut hw, + ) + .unwrap(); + assert_eq!(hw.len(), 0x3db4); + + // Chip id at +0xe90. + assert_eq!(get_u32(&hw, 0xe90), 0x8140); + + // Register-mapping array: 0x28 stride at +0x640. + let base0 = HW_REGISTER_MAP_OFFSET; + assert_eq!(get_u64(&hw, base0), 0x2_0000_0000); + assert_eq!(get_u64(&hw, base0 + 0x08), HW_REGISTER_APERTURE.start); + assert_eq!(get_u32(&hw, base0 + 0x10), 0x4000); + assert_eq!(get_u32(&hw, base0 + 0x14), 0x4000); + assert_eq!(get_u32(&hw, base0 + 0x20), 2); + // A declared-but-unmapped slot carries only its flag. + let base3 = HW_REGISTER_MAP_OFFSET + 3 * HW_REGISTER_MAP_ENTRY_SIZE; + assert_eq!(get_u64(&hw, base3), 0); + assert_eq!(get_u32(&hw, base3 + 0x20), 2); + + // The +0x258e acceptance block — the one block firmware requires. + assert_eq!( + u16::from_le_bytes(hw[0x258e..0x2590].try_into().unwrap()), + 12 + ); + for group in [0x2590usize, 0x25a0] { + for (index, value) in [0u32, 1, 1, 1].iter().enumerate() { + assert_eq!(get_u32(&hw, group + index * 4), *value); + } + } + for channel in 0..12 { + assert_eq!(get_u32(&hw, 0x25b4 + channel * 4), 0xffff_ffff); + } + for offset in [0x25f4usize, 0x2600, 0x2608] { + assert_eq!(get_u32(&hw, offset), 1); + } + // +0x2630 is firmware state and must start zero even though the buffer + // began dirty (0xa5). + assert_eq!(hw[0x2630], 0); + + // Four inline descriptors begin at +0x2614; only ordinals 0/2 exist. + assert_eq!(get_u32(&hw, G17P_KSM_COMPLETION_TABLE_PREFIX_OFFSET), 0x100); + for ordinal in [1usize, 3] { + let base = G17P_KSM_COMPLETION_TABLE_OFFSET + + ordinal * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE; + assert_eq!(&hw[base..base + 0x20], &[0; 0x20]); + } + for (aliases, ordinal) in aliases.iter().zip(G17P_KSM_COMPLETION_PRESENT_ORDINALS) { + let base = G17P_KSM_COMPLETION_TABLE_OFFSET + + ordinal as usize * G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE; + assert_eq!(get_u64(&hw, base), aliases.low); + assert_eq!(get_u64(&hw, base + 0x08), aliases.high); + assert_eq!(get_u32(&hw, base + 0x10), 0x800); + assert_eq!(get_u32(&hw, base + 0x14), 0x40); + assert_eq!(get_u32(&hw, base + 0x18), ordinal); + assert_eq!(get_u32(&hw, base + 0x1c), 0); + } + assert_eq!(get_u64(&hw, G17P_SKSM_QID_CONFIG_2700_OFFSET), 0); + assert_eq!(get_u64(&hw, G17P_QOS_RESOURCE_LIVE_OFFSET), qos_resource); + assert_eq!(get_u32(&hw, G17P_SKSM_QID_CONFIG_2708_OFFSET), 0); + assert_eq!(get_u64(&hw, G17P_QOS_RESOURCE_OFFSET), 0); + assert_eq!(get_u64(&hw, G17P_SKSM_QID_RESOURCE_OFFSET), 0); + + // The recorded opaque runs landed (spot check the two 0xffffffff tails). + assert_eq!(get_u32(&hw, 0x3d98), 0xffff_ffff); + assert_eq!(get_u32(&hw, 0x3db0), 0xffff_ffff); + + // Both measured ladders and their voltage columns are carried. + assert!(PERF_TABLE_VALUES_ENCODED); + assert_eq!(get_u32(&hw, HW_PERF_GROUP_BASES[0]), 0); + assert_eq!(get_u32(&hw, HW_PERF_GROUP_BASES[0] + 10 * 4), 1470); + assert_eq!(get_u32(&hw, HW_PERF_GROUP_BASES[1] + 10 * 4), 1326); + assert_eq!(get_u32(&hw, HW_PERF_GROUP_BASES[0] + 0x40), 125); + assert_eq!(get_u32(&hw, HW_PERF_GROUP_BASES[1] + 0x40), 125); + } + + #[test] + fn hw_data_rejects_out_of_aperture_and_duplicate_slots() { + let aliases = completion_aliases(); + let mut hw = vec![0; HW_DATA_SIZE]; + + // A device VA outside the fixed register aperture fails closed. + let mut window = register_windows(); + window[0].device_va = 0x1000; + assert_eq!( + encode_hw_data( + &window, + &[], + aliases, + qos_resource_high(), + sksm_qid_resource_high(), + &mut hw, + ), + Err(InitdataError::RegisterWindow) + ); + + // Two declarations for the same slot fail closed. + assert_eq!( + encode_hw_data( + ®ister_windows(), + &[(0usize, 2u32)], + aliases, + qos_resource_high(), + sksm_qid_resource_high(), + &mut hw, + ), + Err(InitdataError::RegisterWindow) + ); + } + + #[test] + fn region_c_size_and_required_word() { + assert_eq!(REGION_C_SIZE, 0x1000); + let mut region_c = vec![0xa5; REGION_C_SIZE]; + encode_region_c(&mut region_c).unwrap(); + assert_eq!(region_c.len(), 0x1000); + assert_eq!(REGION_C_REQUIRED_VALUE, 0); + assert_eq!(get_u32(®ion_c, 0xe50), REGION_C_REQUIRED_VALUE); + assert_eq!(®ion_c[0xe50..0xe55], &[0; 5]); + // Those same four bytes, framed the way b1 actually reads them: the + // byte at +0xe50 is zero and the GPU-core power-assert counter is the + // UNALIGNED u32 at +0xe51. Nothing in the image reads +0xe50 or +0xe55. + assert_eq!(region_c[REGION_C_REQUIRED_OFFSET], 0); + assert_eq!( + get_u32(®ion_c, REGION_C_GPU_CORE_PWR_ASSERT_COUNTER_OFFSET), + REGION_C_GPU_CORE_PWR_ASSERT_COUNT + ); + assert_eq!(REGION_C_GPU_CORE_PWR_ASSERT_COUNTER_OFFSET, 0xe51); + assert_eq!(get_u32(®ion_c, 0x9b8), 2); + assert_eq!(get_u32(®ion_c, 0x9bc), 40); + assert_eq!(region_c[0xe21], 0); + // A recorded run near the front landed; the tail past the last run is 0. + assert_eq!(get_u32(®ion_c, 0x24), 0x0000_0bb8); + assert_eq!(get_u32(®ion_c, 0x7c), 1); + assert_eq!(get_u32(®ion_c, 0x80), 1); + assert_eq!(get_u32(®ion_c, 0x9c), 0x3000_001d); + assert_eq!(get_u32(®ion_c, 0xa0), 0x3100_0002); + assert_eq!(region_c[REGION_C_SIZE - 1], 0); + } + + #[test] + fn device_control_power_assert_flags_match_b1() { + const CLEAR: [u32; 15] = [ + 0x00, 0x0e, 0x15, 0x16, 0x1b, 0x22, 0x2a, 0x2e, 0x3a, 0x3b, 0x3c, 0x3d, + 0x3e, 0x3f, 0x40, + ]; + for opcode in 0..=0x3f { + assert_eq!( + device_control_asserts_gpu_power(opcode), + Some(!CLEAR[..CLEAR.len() - 1].contains(&opcode)), + "opcode {opcode:#x}", + ); + } + assert_eq!(device_control_asserts_gpu_power(CLEAR[CLEAR.len() - 1]), None); + + // Native host-tally lifecycle for two updateSubmitInfo records, one + // HardwareBufferID release, and one IdlePowerOff record. + let mut tally = REGION_C_GPU_CORE_PWR_ASSERT_COUNT; + for (opcode, expected) in [(0x20, 1), (0x20, 2), (0x2e, 2), (0x0a, 3)] { + if device_control_asserts_gpu_power(opcode).unwrap() { + tally = tally.wrapping_add(1); + } + assert_eq!(tally, expected); + } + } + + #[test] + fn status_block_pre_init_shape() { + assert_eq!(STATUS_BLOCK_SIZE, 0x80); + let mut status = vec![0xa5; STATUS_BLOCK_SIZE]; + encode_status_block(&mut status).unwrap(); + assert_eq!(get_u32(&status, 0x00), 0); + assert_eq!(get_u32(&status, 0x04), 1); + // +0x10 / +0x14 are post-ack firmware state and must be zero at handoff. + assert_eq!(get_u32(&status, 0x10), 0); + assert_eq!(get_u32(&status, 0x14), 0); + } + + #[test] + fn full_status_objects_and_bundle_views_are_carried() { + let mut primary_status_b = vec![0xa5; PRIMARY_STATUS_B_OBJECT_SIZE]; + let fwctl = KERN + 0x5000_0000; + encode_primary_status_b(fwctl, fwctl + CONTROL_RECORD_SIZE as u64, &mut primary_status_b) + .unwrap(); + assert_eq!(get_u32(&primary_status_b, 0x04), 1); + assert_eq!(get_u64(&primary_status_b, PRIMARY_STATUS_B_FWCTL_STATE), fwctl); + assert_eq!( + get_u64(&primary_status_b, PRIMARY_STATUS_B_FWCTL_RING), + fwctl + CONTROL_RECORD_SIZE as u64 + ); + assert_eq!(get_u32(&primary_status_b, PRIMARY_STATUS_B_CONFIG_HEADER), 1); + assert_eq!(primary_status_b[PRIMARY_STATUS_B_CONFIG_OFFSET + 0x10], 0x0f); + + let mut secondary_extra = [0xa5; STATUS_BLOCK_SIZE]; + encode_secondary_root_extra_1(&mut secondary_extra).unwrap(); + assert_eq!(secondary_extra[0x14], 1); + assert_eq!(&secondary_extra[0x50..0x52], &[0x6a, 0x18]); + + let mut bundle = vec![0; NATIVE_SHARED_CLUSTER_SIZE]; + let state = KERN + 0x6000_0000; + encode_bundle_static(state, &mut bundle).unwrap(); + assert_eq!(get_u64(&bundle, 0x8ed8), state); + assert_eq!(bundle[0x80ac], 0x06); + assert_eq!(bundle[BUNDLE_VIEW_OFFSETS[2] + 0xe45], 0xdc); + assert_eq!( + &bundle[BUNDLE_VIEW_OFFSETS[2] + 0x2ad7..BUNDLE_VIEW_OFFSETS[2] + 0x2adb], + &[0xff, 0x7f, 0xff, 0x7f] + ); + } + + #[test] + fn instance_separation_invariants() { + assert!(validate_pair(&primary(), &secondary()).is_ok()); + // The two roots are exactly 0x8000 apart. + assert_eq!(SECONDARY_ROOT_DELTA, 0x8000); + let mut bad = secondary(); + bad.root += 0x4000; + assert_eq!( + validate_pair(&primary(), &bad), + Err(InitdataError::PairRootSeparation) + ); + // The two instances' shared-region halves are 0x40000 apart. + assert_eq!(SECONDARY_SHARED_REGION_DELTA, 0x40000); + assert!(validate_shared_region_bases(KERN, KERN + 0x40000).is_ok()); + assert_eq!( + validate_shared_region_bases(KERN, KERN + 0x8000), + Err(InitdataError::SharedRegionSeparation) + ); + } + + #[test] + fn root_gate_rejects_the_m5_version_word() { + let mut root = encode_root_bytes(InstanceRole::Primary); + assert!(check_firmware_root_gates(&root).is_ok()); + // The M5/G17S word is answered with "Host and FW Interface is mismatched". + put_u64(&mut root, 0x00, 0x0c8d_e339_0728_04c0); + assert_eq!( + check_firmware_root_gates(&root), + Err(InitdataError::InterfaceMismatch) + ); + put_u64(&mut root, 0x00, INITDATA_VERSION_WORD); + put_u32(&mut root, 0x2c, 0); + assert_eq!( + check_firmware_root_gates(&root), + Err(InitdataError::HostMappedAllocationsDisabled) + ); + } + + #[test] + fn evidence_flags_are_honest() { + // The graph the firmware accepts and boots with is grounded. + assert!(GROUNDED_LAYOUT_AVAILABLE); + assert!(PARSE_ACCEPTED_GRAPH_AVAILABLE); + // The measured performance values are carried, while ADT derivation, + // The target-scoped bundle and status-B bytes are carried too. + assert!(!ALL_HW_GLOBALS_VALUES_GROUNDED); + assert!(PERF_TABLE_VALUES_ENCODED); + assert!(BUNDLE_VIEW_CONTENT_CARRIED); + assert!(PRIMARY_STATUS_B_FULL_OBJECT_CARRIED); + } + + #[test] + fn recorded_run_tables_match_the_serializer_totals() { + // HWData excludes the seven captured QoS pointer bytes; the adjacent + // state words and the remaining platform runs are preserved. + let hw: usize = HW_DATA_OPAQUE_RUNS.iter().map(|(_, data)| data.len()).sum(); + assert_eq!(HW_DATA_OPAQUE_RUNS.len(), 29); + assert_eq!(hw, 262); + let region_c: usize = REGION_C_RUNS.iter().map(|(_, data)| data.len()).sum(); + assert_eq!(region_c, 180); + // Every run fits its object. + assert!(HW_DATA_OPAQUE_RUNS + .iter() + .all(|(off, data)| off + data.len() <= HW_DATA_SIZE)); + assert!(REGION_C_RUNS + .iter() + .all(|(off, data)| off + data.len() <= REGION_C_SIZE)); + // No opaque run touches firmware-state +0x2630. + assert!(HW_DATA_OPAQUE_RUNS.iter().all(|(off, data)| { + !(*off <= HW_FIRMWARE_STATE_OFFSET && HW_FIRMWARE_STATE_OFFSET < *off + data.len()) + })); + } + + /// Pins the remaining documented ABI/geometry constants to their grounded + /// values. They describe parts of the graph this offline module records but + /// does not yet encode (the control-record framing and the full primary + /// status-B object); asserting them keeps the + /// recorded facts under test even before the encoders reach them. + #[test] + fn documented_constants_have_their_grounded_values() { + assert_eq!(ROOT_ZERO_10, 0x10); + assert_eq!(REGION_A_SIZE, 0x4000); + assert_eq!(HW_DATA_BUNDLE_SIZE, 0xc000); + assert_eq!(REGION_C_REQUIRED_OFFSET, 0xe50); + assert_eq!(STATUS_BLOCK_POST_ACK_OFFSETS, [0x10, 0x14]); + assert_eq!(PRIMARY_STATUS_B_OBJECT_SIZE, 0xeb00); + // Device-control framing and the ownerless-UMA-free opening prefixes. + assert_eq!(CONTROL_RECORD_SIZE, 0x40); + assert_eq!(CONTROL_OPENING_PRIMARY_PRODUCER, 1); + assert_eq!(COMPUTE_RUNTIME_CONTROL_RECORDS, 65); + assert_eq!(COMPUTE_RUNTIME_CONTROL_FINAL_COUNTER, 66); + assert_eq!(CONTROL_OPENING_SECONDARY_PRODUCER, 1); + // Channel state-block stride within the compact grid. + assert_eq!(CHANNEL_STATE_STRIDE, 0x40); + // Performance-table geometry and measured values. + assert_eq!(HW_PERF_LADDER_ENTRIES, 11); + assert_eq!(HW_PERF_GROUP_REPEATS, [16, 1]); + assert_eq!(HW_PERF_VOLTAGE_DELTA, 0x40); + assert_eq!(HW_PERF_MEMORY_VOLTAGE_DELTA, 0x440); + assert_eq!(HW_PERF_STATE_BLOCK_STRIDE, 0x40); + assert_eq!(HW_PERF_FREQ_LADDER_B, 0x1808); + assert_eq!(HW_PERF_SCALE_LADDER_B, 0x1848); + assert_eq!(HW_PERF_RELATIVE_LADDER_A, 0x18c8); + assert_eq!(HW_PERF_RELATIVE_LADDER_B, 0x1908); + assert_eq!(HW_PERF_INDEX_MAP_A, 0x19c8); + assert_eq!(HW_PERF_INDEX_MAP_B, 0x1a08); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/m5_render.rs b/drivers/gpu/drm/asahi/g18/fw/m5_render.rs new file mode 100644 index 0000000000000..0bb6f2b149c52 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/m5_render.rs @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: MIT +//! M5/G17G graphics register construction from owned render resources. +//! +//! Common resource lowering is shared with G17P. Register order, tile packing +//! and per-stage controls follow the native 25G76 M5 reference. This codec is +//! not a claim that Linux TA/3D submission or presentation is qualified. + +use crate::g18::fw::render_params::{ + self as g17_render, G17pRenderParameters, RegisterWrite, RenderBuildError, RenderDescriptorMetadata, + RenderDescriptorObjects, +}; + +#[cfg(test)] +#[path = "m5_render_test.rs"] +mod tests; + +pub(crate) const TA_REGISTER_COUNT: usize = 82; +pub(crate) const FRAGMENT_REGISTER_COUNT: usize = 90; + +/// Encode the TA parameter-manager metrics slot in the compact GART window. +/// The native 25G76 slot at 0x1000080040 produces register 1c910 = 0x80041: +/// the low bit enables the record; the remaining bits address its 64 bytes. +pub(crate) fn parameter_buffer_metrics_record( + scene_metrics: u64, +) -> Result { + let offset = scene_metrics.checked_sub(0x10_0000_0000) + .ok_or(M5RenderBuildError::OwnershipMismatch)?; + if offset >= (1u64 << 32) || offset & 0x3f != 0 { + return Err(M5RenderBuildError::OwnershipMismatch); + } + Ok(offset | 1) +} + +/// Initial M5 PB slot with fresh ring cursors. The owning job chooses the +/// hardware slot separately and retains the page-list mapping. +pub(crate) fn initial_parameter_buffer_descriptor( + page_list: u64, + page_count: u32, +) -> Result<[u32; 4], M5RenderBuildError> { + if page_list == 0 || page_list >= (1u64 << 42) || page_list & 0x7f != 0 + || page_count == 0 || page_count >= (1 << 22) { + return Err(M5RenderBuildError::OwnershipMismatch); + } + // 25G76 halLoadPBDesc, 89de844..89de8dc. The descriptor has four + // dwords; +0's low flags and both cursors start at zero. Native's + // untagged user address is translated into its PB segment encoding. + let segment = ((page_list + 0x70_0000_0000) & 0x70_0000_0000) >> 8; + Ok([((page_list >> 4) as u32) & !7, + (segment as u32 & 0xf000_0000) | page_count, 0, 0]) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5RenderBuildError { + Common(RenderBuildError), + SharedRegisterLayout, + TileCountOutOfRange, + OverlappingTileFlags, + DispatchBitAlreadySet, + InvalidContext, + OwnershipMismatch, + InvalidUscState, +} + +impl From for M5RenderBuildError { + fn from(error: RenderBuildError) -> Self { + Self::Common(error) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderStageControls { + /// Native host descriptor +0x1120 (TA) / +0x7e8 (fragment). + pub(crate) resource_flags: u64, + /// Explicit per-stage d419/d429 input, independent of the work key. + pub(crate) resource_control: u64, + /// Native +0x1128 / +0x7f0; a zero source selects the native 0x60 fallback. + pub(crate) ptc_max: u32, + pub(crate) work_key: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderControls { + pub(crate) ta: M5RenderStageControls, + pub(crate) fragment: M5RenderStageControls, + pub(crate) ta_tile_control: u64, + /// Base value for the two 1c898 writes. Bit zero is supplied by the codec. + pub(crate) ta_dispatch_control: u64, + pub(crate) fragment_raster_control: u64, + /// Non-geometry bits for 16068. Never includes tile counts, utile or layers. + pub(crate) fragment_tile_flags: u64, + /// M5 adds this second pair of store-program bindings at 153b1/153b9. + pub(crate) store_pipeline_2_bind: u64, + pub(crate) store_pipeline_2: u64, + pub(crate) partial_store_pipeline_2_bind: u64, + pub(crate) partial_store_pipeline_2: u64, + pub(crate) partial_tile_flags: u64, +} + +/// Per-submission USC private-memory accounting, separate from user status +/// registers and queue completion stamps. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderUscState { + pub(crate) control: u64, + /// AGXUMADataRec +0x50. Native allocatePagesEnd derives this from pool + /// sizes and 4-KiB allocation units; it is a byte count, not a GPU VA. + pub(crate) required_bytes: u64, + /// Capacity actually backed by this job's retained USC allocation pages. + pub(crate) allocated_bytes: u64, + /// A 64-byte slot in the firmware-visible statistics ring. + pub(crate) statistics: u64, +} + +impl M5RenderUscState { + fn valid(self) -> bool { + let firmware_range = |address: u64, size: u64| { + address >= 0xffff_fc00_0000_0000 + && address & 0x3f == 0 + && address.checked_add(size).is_some() + }; + firmware_range(self.control, 0x80) + && firmware_range(self.statistics, 0x40) + && self.required_bytes != 0 + && self.required_bytes <= self.allocated_bytes + && self.allocated_bytes & 0xfff == 0 + } +} + +/// Fill the native USC fields only after the caller owns the pool and the +/// statistics slot. Other runtime fields still need queue/link publication. +/// Validate both stages before modifying either descriptor. +pub(crate) fn apply_usc_submission_state( + ta: &mut [u8], + fragment: &mut [u8], + ta_state: M5RenderUscState, + fragment_state: M5RenderUscState, +) -> Result<(), M5RenderBuildError> { + if ta.len() < g17_render::TA_DESCRIPTOR_SIZE + || fragment.len() < g17_render::FRAGMENT_DESCRIPTOR_SIZE + { + return Err(RenderBuildError::BufferTooSmall.into()); + } + if !ta_state.valid() || !fragment_state.valid() { + return Err(M5RenderBuildError::InvalidUscState); + } + // 25G76 AGXTAChannelSKSM::submitBuffer: 8a244c4..8a244ec. + // AGX3DChannelSKSM::submitBuffer takes the corresponding value from + // host +0xba0 = AGXUMADataRec +0x50 at 895b70c..895b718. + for (raw, state, control, required, statistics) in [ + (ta, ta_state, 0x934, 0x93d, 0x945), + (fragment, fragment_state, 0x21ce, 0x21d7, 0x21df), + ] { + put_u64(raw, control, state.control); + put_u64(raw, required, state.required_bytes); + put_u64(raw, statistics, state.statistics); + } + Ok(()) +} + +const EMPTY_REGISTER: RegisterWrite = RegisterWrite { + number: 0, + value: 0, +}; + +struct RegisterList { + values: [RegisterWrite; N], + count: usize, +} + +impl RegisterList { + fn new() -> Self { + Self { + values: [EMPTY_REGISTER; N], + count: 0, + } + } + + fn push(&mut self, number: u32, value: u64) -> Result<(), M5RenderBuildError> { + if self.count == N { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + self.values[self.count] = RegisterWrite { number, value }; + self.count += 1; + Ok(()) + } + + fn finish(self) -> Result<[RegisterWrite; N], M5RenderBuildError> { + if self.count != N { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + Ok(self.values) + } +} + +fn value(registers: &[RegisterWrite], number: u32) -> Result { + registers + .iter() + .find(|r| r.number == number) + .map(|r| r.value) + .ok_or(M5RenderBuildError::SharedRegisterLayout) +} + +const fn ptc_control(maximum: u32) -> u64 { + let maximum = if maximum == 0 { 0x60 } else { maximum }; + ((maximum as u64) << 32) | 0x400020 +} + +fn tile_state(p: &G17pRenderParameters, flags: u64) -> Result { + let tiles_x = (p.width as u64 + 31) / 32; + let tiles_y = (p.height as u64 + 31) / 32; + if !(1..=1024).contains(&tiles_x) || !(1..=1024).contains(&tiles_y) { + return Err(M5RenderBuildError::TileCountOutOfRange); + } + // 25G76 fragment generator: bfi at 89dcb98, 89dcbb8 and 89dcbbc. + let geometry_mask = (0xfffff_u64 << 42) | (0xf_u64 << 38) | (1_u64 << 32); + if flags & geometry_mask != 0 { + return Err(M5RenderBuildError::OverlappingTileFlags); + } + Ok(flags + | ((tiles_x - 1) << 42) + | ((tiles_y - 1) << 52) + | ((p.utile_config & 0xf000) << 26) + | if p.layers > 1 { 1_u64 << 32 } else { 0 }) +} + +fn tile_blocks(pixels: u32) -> u64 { + let tiles = (pixels as u64 + 31) / 32; + let first = (((tiles + 3) / 4) + 3) & !3; + // Native TA host +1208/+120c supplies 0x40200c for the reference: + // boundaries 12, 8, 4 in ten-bit fields, versus G17P's nine-bit fields. + (3 * first) | ((2 * first) << 10) | (first << 20) +} + +pub(crate) fn build_ta_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], M5RenderBuildError> { + // M5's captured register program addresses heap metadata directly. + // G17P's layered path now uses a separate layer table in these slots. + let mut common_parameters = *p; + common_parameters.layermeta = p.heapmeta; + let common = g17_render::build_ta_registers(&common_parameters)?; + if controls.ta_dispatch_control & 1 != 0 { + return Err(M5RenderBuildError::DispatchBitAlreadySet); + } + // Validate the M5 tile fields even when only constructing the TA half. + tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterList::new(); + for mut register in common { + match register.number { + 0x0a309 | 0x0a5a1 | 0x0d419 => continue, + 0x1c8f8 => { + out.push(0x0d499, controls.ta.resource_flags)?; + out.push(0x0d419, controls.ta.resource_control)?; + out.push(0x0a5a1, ptc_control(controls.ta.ptc_max))?; + register.value = controls.ta_tile_control; + } + 0x10171 => { + for (number, source) in [(0x1ca50, 0x101b9), (0x1ca58, 0x10149), (0x1ca60, 0x10141)] + { + out.push(number, value(&common, source)?)?; + } + out.push(0x1ca70, tile_blocks(p.width))?; + out.push(0x1ca78, tile_blocks(p.height))?; + out.push(0x1ca68, value(&common, 0x1c081)?)?; + } + 0x1c069 => register.value = tile_blocks(p.width), + 0x1c071 => register.value = tile_blocks(p.height), + 0x1c898 => { + if register.value > 1 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + register.value |= controls.ta_dispatch_control; + } + 0x10799 => out.push(0x1a149, 0)?, + 0x10209 => { + out.push(0x0a739, 0)?; + register.value = controls.ta.work_key; + } + 0x1c9f0 | 0x14320 => register.value = controls.ta.work_key, + 0x14308 => register.value = p.usc_flist_hardware_buffer_id as i32 as i64 as u64, + _ => {} + } + out.push(register.number, register.value)?; + if register.number == 0x01740 { + out.push(0x0a739, 0)?; + } + } + out.finish() +} + +pub(crate) fn build_fragment_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], M5RenderBuildError> { + // M5's captured register program addresses heap metadata directly. + // G17P's layered path now uses a separate layer table in these slots. + let mut common_parameters = *p; + common_parameters.layermeta = p.heapmeta; + let common = g17_render::build_fragment_registers(&common_parameters)?; + let geometry = tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterList::new(); + let mut dispatch_writes = 0; + let mut tile_writes = 0; + for mut register in common { + match register.number { + 0x0a301 | 0x15109 | 0x15101 | 0x160a8 | 0x0a5a9 | 0x0d429 => continue, + 0x100b1 => register.number = 0x16469, + 0x100d9 => register.number = 0x16471, + 0x100b8 => register.number = 0x16138, + 0x10791 => register.value = controls.fragment_raster_control, + 0x15321 => { + out.push(0x0d491, controls.fragment.resource_flags)?; + out.push(0x0d429, controls.fragment.resource_control)?; + out.push(0x0a5a9, ptc_control(controls.fragment.ptc_max))?; + } + 0x16068 => { + tile_writes += 1; + if tile_writes == 1 { + register.value = geometry; + } + } + 0x16020 => { + dispatch_writes += 1; + if dispatch_writes == 3 { + if register.value != 0 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + continue; + } + } + 0x160e0 => out.push(0x1a151, 0)?, + 0x10211 => { + out.push(0x0a741, 0)?; + register.value = controls.fragment.work_key; + } + 0x10420 => register.value = controls.fragment.work_key, + 0x14048 => register.value = p.usc_flist_hardware_buffer_id as i32 as i64 as u64, + _ => {} + } + out.push(register.number, register.value)?; + if register.number == 0x15381 { + out.push(0x153b1, controls.store_pipeline_2_bind)?; + out.push(0x153b9, controls.store_pipeline_2)?; + } + if register.number == 0x01731 { + out.push(0x0a741, 0)?; + } + } + if dispatch_writes != 3 || tile_writes != 2 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + out.finish() +} + +fn partial_registers( + common: &[RegisterWrite], + insert_after: u32, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; N], M5RenderBuildError> { + let mut out = RegisterList::new(); + for register in common { + let bits = if register.number == 0x10039 { + controls.partial_tile_flags + } else { + 0 + }; + out.push(register.number, register.value | bits)?; + if register.number == insert_after { + out.push(0x153b1, controls.partial_store_pipeline_2_bind)?; + out.push(0x153b9, controls.partial_store_pipeline_2)?; + } + } + out.finish() +} + +pub(crate) fn build_partial_store_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 18], M5RenderBuildError> { + partial_registers(&g17_render::partial_store_registers(p), 0x15381, controls) +} + +pub(crate) fn build_partial_resume_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 25], M5RenderBuildError> { + partial_registers(&g17_render::partial_resume_registers(p), 0x15381, controls) +} + +pub(crate) fn build_partial_load_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 12], M5RenderBuildError> { + partial_registers(&g17_render::partial_load_registers(p), 0x15309, controls) +} + +fn work_key( + p: &G17pRenderParameters, + metadata: RenderDescriptorMetadata, +) -> Result { + if metadata.context_id == 0 || metadata.context_id > 255 { + return Err(M5RenderBuildError::InvalidContext); + } + if metadata.ta_hardware_buffer_id != p.ta_hardware_buffer_id { + return Err(M5RenderBuildError::OwnershipMismatch); + } + metadata + .submission_ordinal + .checked_add(metadata.submission_ordinal / 2) + .ok_or(M5RenderBuildError::Common( + RenderBuildError::ArithmeticOverflow, + ))?; + Ok(((metadata.context_id as u64) << 8) | (metadata.command_buffer_state_word & 0x3f) as u64) +} + +fn alias_offset(alias: u64, offset: u64) -> Result { + alias.checked_add(offset).ok_or(M5RenderBuildError::Common( + RenderBuildError::ArithmeticOverflow, + )) +} + +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Construct the M5 TA record's common layout and all register programs. +/// Runtime ownership/completion fields must be populated before publication. +#[inline(never)] +pub(crate) fn build_ta_descriptor_base( + p: &G17pRenderParameters, + controls: &M5RenderControls, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + low_alias: u64, + raw: &mut [u8], +) -> Result<(), M5RenderBuildError> { + let mut controls = *controls; + controls.ta.work_key = work_key(p, metadata)?; + let program = alias_offset(low_alias, 0x60)?; + alias_offset(low_alias, g17_render::TA_DESCRIPTOR_SIZE as u64)?; + let registers = build_ta_registers(p, &controls)?; + g17_render::build_ta_descriptor_base(p, objects, metadata, ®isters, raw)?; + // Native boundaries: main work ends before the final a739/deflake/dispatch. + for (offset, count) in [ + (0x38, TA_REGISTER_COUNT - 3), + (0x3a, TA_REGISTER_COUNT), + (0x3c, TA_REGISTER_COUNT), + ] { + put_u16(raw, offset, count as u16); + } + put_u64(raw, 0x760, program); + Ok(()) +} + +/// Construct the M5 fragment record, including all three partial programs. +/// The shared encoder is pure and does not apply Neo diagnostic overrides. +#[inline(never)] +pub(crate) fn build_fragment_descriptor_base( + p: &G17pRenderParameters, + controls: &M5RenderControls, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + low_alias: u64, + raw: &mut [u8], +) -> Result<(), M5RenderBuildError> { + let mut controls = *controls; + controls.fragment.work_key = work_key(p, metadata)?; + alias_offset(low_alias, g17_render::FRAGMENT_DESCRIPTOR_SIZE as u64)?; + let main = build_fragment_registers(p, &controls)?; + let store = build_partial_store_registers(p, &controls)?; + let resume = build_partial_resume_registers(p, &controls)?; + let load = build_partial_load_registers(p, &controls)?; + g17_render::build_fragment_descriptor_base( + p, + objects, + metadata, + g17_render::G17pFragmentRegisters { + main: &main, + store: &store, + resume: &resume, + load: &load, + }, + raw, + )?; + put_u32(raw, 0x50, 1); + put_u32(raw, 0x90, 1); + for (offset, count) in [ + (0x80, FRAGMENT_REGISTER_COUNT - 3), + (0x82, FRAGMENT_REGISTER_COUNT - 1), + (0x84, FRAGMENT_REGISTER_COUNT - 1), + (0x88, FRAGMENT_REGISTER_COUNT), + ] { + put_u16(raw, offset, count as u16); + } + for (offset, target) in [ + (0x7a0, 0xa0), + (0xec0, 0x7c0), + (0x15e0, 0xee0), + (0x1d00, 0x1600), + ] { + put_u64(raw, offset, alias_offset(low_alias, target)?); + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/m5_render_target.rs b/drivers/gpu/drm/asahi/g18/fw/m5_render_target.rs new file mode 100644 index 0000000000000..c91d8bca493cf --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/m5_render_target.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Render-target allocation geometry and M5 viewport state. + +pub(crate) const PAGE_BYTES: usize = 0x4000; +pub(crate) const AUX_BYTES: usize = 0x8000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct M5RenderTargetLayout { + pub(crate) tilemap_bytes: usize, + pub(crate) heapmeta_offset: usize, + pub(crate) heapmeta_bytes: usize, + pub(crate) target_bytes: usize, + pub(crate) tpc_bytes: usize, +} + +fn align(value: usize, alignment: usize) -> Option { + Some(value.checked_add(alignment - 1)? & !(alignment - 1)) +} + +impl M5RenderTargetLayout { + pub(crate) fn new( + width: u32, height: u32, layers: u16, utile_width: u8, utile_height: u8, + clusters: u32, + ) -> Option { + if !(1..=16384).contains(&width) || !(1..=16384).contains(&height) + || !(1..=2048).contains(&layers) || clusters == 0 + || !matches!((utile_width, utile_height), (32, 32) | (32, 16) | (16, 16)) { + return None; + } + // Same 4x4 macrotile geometry as the register builders. M5 packs its + // boundaries in ten-bit fields; that does not change allocation size. + let tiles_x = (width as usize + 31) / 32; + let tiles_y = (height as usize + 31) / 32; + let per_mtile = align((tiles_x + 3) / 4, 4)? + .checked_mul(align((tiles_y + 3) / 4, 4)?)?; + let utiles = (32 / utile_width as usize) * (32 / utile_height as usize); + let tilemap_bytes = align(5usize.checked_mul(per_mtile)?.checked_mul(utiles)?, 4)? + .checked_mul(16)?.checked_mul(layers as usize)?; + let heapmeta_offset = align(tilemap_bytes, 0x1000)?; + let heapmeta_bytes = if layers > 1 { 0x300 } else { 0x200 }; + let target_bytes = align(heapmeta_offset.checked_add(heapmeta_bytes)?, PAGE_BYTES)?; + let tpc_bytes = align(8usize.checked_mul(per_mtile)?.checked_mul(utiles)? + .checked_mul(16)?.checked_mul(layers as usize)? + .checked_mul(clusters as usize)?, PAGE_BYTES)?; + Some(Self { tilemap_bytes, heapmeta_offset, heapmeta_bytes, target_bytes, tpc_bytes }) + } +} + +fn half_float_bits(value: u32) -> u32 { + // The admitted integer dimensions are exactly representable in binary32. + let highest = 31 - value.leading_zeros(); + ((highest + 126) << 23) | ((value << (23 - highest)) & 0x7fffff) +} + +/// Native M5 deflake page: viewport at +0x9c0, command header 0x3000. +/// G17P uses +0x900/0xc00 and must not supply this page. The eight nonzero +/// dwords match the M5 25G76 resource capture; dimensions are caller-owned. +pub(crate) fn initialize_deflake(page: &mut [u8], width: u32, height: u32) -> Option<()> { + if page.len() != PAGE_BYTES || !(1..=16384).contains(&width) + || !(1..=16384).contains(&height) { return None; } + page.fill(0); + let x = half_float_bits(width); + let y = half_float_bits(height); + for (offset, value) in [(0x9c0, 0x3000), + (0x9c4, 0x80000000 | ((width + 31) / 32 - 1)), + (0x9c8, (height + 31) / 32 - 1), (0x9d0, x), (0x9d4, x), + (0x9d8, y), (0x9dc, y | 0x80000000), (0x9e4, 0x3f800000)] { + page[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + Some(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reference_target_geometry() { + let layout = M5RenderTargetLayout::new(256, 256, 1, 32, 32, 1).unwrap(); + assert_eq!(layout.tilemap_bytes, 0x500); + assert_eq!(layout.heapmeta_offset, 0x1000); + assert_eq!(layout.heapmeta_bytes, 0x200); + assert_eq!(layout.target_bytes, PAGE_BYTES); + assert_eq!(layout.tpc_bytes, PAGE_BYTES); + } + + #[test] + fn larger_layered_targets_increase_capacity() { + let a = M5RenderTargetLayout::new(4096, 2048, 1, 32, 32, 1).unwrap(); + let b = M5RenderTargetLayout::new(4096, 2048, 2, 16, 16, 2).unwrap(); + assert_eq!(b.tilemap_bytes, a.tilemap_bytes * 8); + assert_eq!(b.tpc_bytes, a.tpc_bytes * 16); + assert!(b.heapmeta_offset >= b.tilemap_bytes); + assert!(b.target_bytes >= b.heapmeta_offset + b.heapmeta_bytes); + } + + #[test] + fn invalid_dimensions_leave_output_untouched() { + let mut page = [0xa5; PAGE_BYTES]; + assert_eq!(initialize_deflake(&mut page, 0, 256), None); + assert!(page.iter().all(|b| *b == 0xa5)); + assert!(M5RenderTargetLayout::new(16385, 1, 1, 32, 32, 1).is_none()); + assert!(M5RenderTargetLayout::new(1, 1, 0, 32, 32, 1).is_none()); + assert!(M5RenderTargetLayout::new(1, 1, 1, 8, 32, 1).is_none()); + assert!(M5RenderTargetLayout::new(1, 1, 1, 32, 32, 0).is_none()); + } + + #[test] + fn m5_viewport_is_distinct_and_dimension_derived() { + for (width, height) in [(256, 256), (1920, 1080), (1, 16384)] { + let mut page = [0xa5; PAGE_BYTES]; + initialize_deflake(&mut page, width, height).unwrap(); + let word = |off| u32::from_le_bytes(page[off..off + 4].try_into().unwrap()); + assert_eq!(word(0x900), 0); + assert_eq!(word(0x9c0), 0x3000); + assert_eq!(word(0x9d0), (width as f32 / 2.0).to_bits()); + assert_eq!(word(0x9d8), (height as f32 / 2.0).to_bits()); + assert_eq!(word(0x9dc), (-(height as f32 / 2.0)).to_bits()); + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/mod.rs b/drivers/gpu/drm/asahi/g18/fw/mod.rs new file mode 100644 index 0000000000000..6a951db464f42 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/mod.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 firmware record and register encoders. Nothing here allocates or publishes. + +#[allow(dead_code)] +pub(crate) mod compute; +#[allow(dead_code)] +pub(crate) mod compute_memory; +#[allow(dead_code)] +pub(crate) mod compute_params; +#[allow(dead_code)] +pub(crate) mod compute_state; +#[allow(dead_code)] +pub(crate) mod config; +#[allow(dead_code)] +pub(crate) mod graphics_payload; +#[allow(dead_code)] +pub(crate) mod initdata; +#[allow(dead_code)] +pub(crate) mod m5_render; +#[allow(dead_code)] +pub(crate) mod m5_render_target; +#[allow(dead_code)] +pub(crate) mod parameter_buffer; +#[allow(dead_code)] +pub(crate) mod render; +#[allow(dead_code)] +pub(crate) mod render_assembly; +#[allow(dead_code)] +pub(crate) mod render_command; +#[allow(dead_code)] +pub(crate) mod render_params; +#[allow(dead_code)] +pub(crate) mod render_passthrough; +#[allow(dead_code)] +pub(crate) mod scheduler; +#[allow(dead_code)] +pub(crate) mod sksm; +#[allow(dead_code)] +pub(crate) mod store; +#[allow(dead_code)] +pub(crate) mod submission; +#[allow(dead_code)] +pub(crate) mod uma; +#[allow(dead_code)] +pub(crate) mod vm_layout; +#[allow(dead_code)] +pub(crate) mod events; +#[allow(dead_code)] +pub(crate) mod rtkit; diff --git a/drivers/gpu/drm/asahi/g18/fw/parameter_buffer.rs b/drivers/gpu/drm/asahi/g18/fw/parameter_buffer.rs new file mode 100644 index 0000000000000..5f256a1e20a0f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/parameter_buffer.rs @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 (26A428) parameter buffer: PB state object, page list and PB-table row. +//! +//! G18 uses AGXParameterManagementVirtual (halNewParameterManager +//! 0xfffffe0008c50c78). Each PB block is 0x80000 bytes (PM +0x70, constant +//! 0xfffffe000714f800) of 32 KiB pages; the page list holds one u32 page +//! number (user GPU VA >> 15) per page. The encoders below reproduce, byte for +//! byte, the original writers executed by the native parameter-buffer audit: +//! AGXHWParamBufferManager::resetHWParamBufferState (0xfffffe0008c5e06c), +//! AGXParameterManagement::buildPageList (0xfffffe0008c5dc98) and +//! AGXParameterManagement::loadPBDesc (0xfffffe0008c60918) with the HAL400 +//! halLoadPBDesc (0xfffffe0008c40b40). +//! +//! Pure encoders only: nothing here allocates, maps or publishes. The kernel +//! owner (G18ParameterBuffer) lives in g18_graphics_queues.rs, which is a +//! g17_resources submodule and can use its MappedObject. + +#![allow(dead_code)] + +/// Native InternalResource size for the PB state (createNewHWPBManager +/// 0xfffffe0008c5fd90). Firmware caches the first 0x90 bytes (b000 0x13764) +/// and writes the GPU context ID to +0x08. +pub(crate) const G18_PB_STATE_BYTES: usize = 0xc0; +pub(crate) const G18_PB_STATE_FIRMWARE_BYTES: usize = 0x90; +pub(crate) const G18_PB_ROW_BYTES: usize = 16; +/// Native 0x4000-byte PB table (g18_resources.rs PBDESC_TABLE_SIZE): 0x400 rows. +pub(crate) const G18_PB_TABLE_ROWS: u32 = 0x400; +pub(crate) const G18_PB_BLOCK_BYTES: u64 = 0x80000; +pub(crate) const G18_PB_PAGE_BYTES: u64 = 0x8000; +pub(crate) const G18_PB_PAGES_PER_BLOCK: u32 = (G18_PB_BLOCK_BYTES / G18_PB_PAGE_BYTES) as u32; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum G18PbError { + Granule, + BlockIndex, + PageListTooSmall, + Slot, +} + +/// Parameter-management fields consumed by the state/page-list writers. +/// Names are native AGXParameterManagement offsets. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18PmConfig { + /// Block ring capacity (+0x148, u32): accelerator PB maximum >> 19. + pub pm_148: u32, + /// Current PB bytes (+0x138); state +0x34 = (pm_138 >> 19) * pm_1d8. + pub pm_138: u64, + /// Pages per block (+0x1d8; device PM config word 0, 0x10 on G18). + pub pm_1d8: u32, + /// Block granule (+0x70), 0x80000 on G18; state +0x58. + pub pm_70: u64, + pub pm_c0: u64, + pub pm_c8: u64, + /// State +0x80 = pm_154 << 4; +0x7c = pm_15c << 4. + pub pm_154: u32, + pub pm_15c: u32, + /// State +0x10 and TA command +0x1c. + pub pm_1d4: u32, +} + +/// Addresses and ring positions of one PB state. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18PbStateInputs { + pub config: G18PmConfig, + /// State +0x20: firmware VA of the page list (converted, identity on G18). + pub page_list_firmware: u64, + /// State +0x28: GPU VA of the page list; also encoded in the PB row. + pub page_list_gpu: u64, + /// State +0x30: page-list resource length word (resource +0x78). + pub page_list_bytes: u32, + /// State +0x64: GPU VA of the 4-byte submitted-operations counter. + pub counter_gpu: u64, + /// State +0x84: GPU VA of the 0x8000-byte discard page. + pub discard_gpu: u64, + /// PB table slot (TA HardwareBufferID), state +0x0c. + pub slot: u32, + /// Block ring positions: first block (PM +0x98 -> +8) and staged end. + pub start_block: u32, + pub end_block: u32, +} + +/// AGXSPBDescRingState, the HWPB manager's +0x34..+0x44 words. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18PbRing { + pub words: [u32; 4], +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn get32(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} + +/// PB state after the native first-TA sequence: resetHWParamBufferState, +/// buildPageList and (when the manager was not loaded before) loadPBDesc. +/// Bytes +0x08..+0x0c (firmware-owned context), +0x1c..+0x20 and +/// +0x90..+0xc0 are not written, as native. +pub(crate) fn encode_g18_pb_state( + out: &mut [u8; G18_PB_STATE_BYTES], + p: &G18PbStateInputs, + load: bool, +) { + let c = &p.config; + // resetHWParamBufferState 0x8c5e094..0x8c5e1e4. + put32(out, 0x38, c.pm_148); + put64(out, 0x3c, 0); + put64(out, 0x44, c.pm_c0); + put64(out, 0x4c, c.pm_c8); + put64(out, 0x20, p.page_list_firmware); + put64(out, 0x28, p.page_list_gpu); + put32(out, 0x30, p.page_list_bytes); + put32(out, 0x34, 0); + put64(out, 0x5c, 0); + put32(out, 0x54, u32::MAX); + put32(out, 0x58, c.pm_70 as u32); + put64(out, 0x64, p.counter_gpu); + put64(out, 0, 0); + out[0x6c..0x7c].fill(0); + put32(out, 0x8c, 0); + put64(out, 0x14, 0); + put32(out, 0x80, c.pm_154.wrapping_shl(4)); + put32(out, 0x7c, c.pm_15c.wrapping_shl(4)); + put64(out, 0x84, p.discard_gpu); + put32(out, 0x10, c.pm_1d4); + // buildPageList 0x8c5dd54..0x8c5dd74 and 0x8c5df3c. + let pages = g18_pb_page_count(c); + put32(out, 0x34, pages); + put32(out, 0x54, pages.wrapping_sub(1)); + put32(out, 0x3c, p.end_block); + put32(out, 0x40, p.start_block); + // loadPBDesc 0x8c609b8 on a manager that was never loaded. + if load { + put32(out, 0x0c, p.slot); + } +} + +/// State +0x34, the PB page count encoded in the row. +pub(crate) fn g18_pb_page_count(c: &G18PmConfig) -> u32 { + ((c.pm_138 >> 19) as u32).wrapping_mul(c.pm_1d8) +} + +/// buildPageList, G18 fast path (granule 0x80000): 16 consecutive page +/// numbers per ring slot from `start` up to, not including, `end` (mod the +/// ring capacity), written from the start of the list. `page_bases[slot]` is +/// that slot's block page base (block user GPU VA >> 15, 25 bits). +pub(crate) fn g18_pb_page_list( + out: &mut [u8], + c: &G18PmConfig, + page_bases: &[u32], + start: u32, + end: u32, +) -> Result { + if c.pm_70 != G18_PB_BLOCK_BYTES { + return Err(G18PbError::Granule); + } + if start >= c.pm_148 || end >= c.pm_148 || page_bases.len() < c.pm_148 as usize { + return Err(G18PbError::BlockIndex); + } + let mut slot = start; + let mut offset = 0usize; + while slot != end { + let base = page_bases[slot as usize]; + for page in 0..G18_PB_PAGES_PER_BLOCK { + let entry = out.get_mut(offset..offset + 4).ok_or(G18PbError::PageListTooSmall)?; + entry.copy_from_slice(&base.wrapping_add(page).to_le_bytes()); + offset += 4; + } + slot = (slot + 1) % c.pm_148; + } + Ok(offset / 4) +} + +/// halLoadPBDesc row. `page_list_gpu`/`page_count` are PB state +0x28/+0x34 +/// at load time; `ring` is the manager's +0x34..+0x44 words (all zero after +/// buildPageList, so zero for every first TA). `old` is the row's previous +/// contents: native keeps bits 1..2 of word 0, 22..27 of word 1, 22..31 of +/// word 2 and 22..30 of word 3. +pub(crate) fn g18_pb_row( + page_list_gpu: u64, + page_count: u32, + ring: G18PbRing, + old: [u8; G18_PB_ROW_BYTES], +) -> [u8; G18_PB_ROW_BYTES] { + let list = page_list_gpu; + let rebase = if list & (1 << 42) == 0 { 0x70_0000_0000 } else { 0 }; + let segment = (rebase + list) & 0x70_0000_0000 | (list >> 3) & 0x80_0000_0000; + let old = [get32(&old, 0), get32(&old, 4), get32(&old, 8), get32(&old, 12)]; + let word0 = ((list >> 4) as u32 & !7) | (old[0] & 7); + let word0 = (word0 & !1) | u32::from(ring.words[3] != 0); + let word1 = ((segment >> 8) as u32 & !0x0fff_ffff) | (old[1] & 0x0fff_ffff); + let word1 = (word1 & !0x3f_ffff) | (page_count & 0x3f_ffff); + let word2 = (old[2] & !0x3f_ffff) | (ring.words[0] & 0x3f_ffff); + let word3 = (old[3] & !0x3f_ffff) | (ring.words[1] & 0x3f_ffff); + let word3 = (word3 & !(1 << 31)) | ((ring.words[2] & 1) << 31); + let mut row = [0; G18_PB_ROW_BYTES]; + for (index, word) in [word0, word1, word2, word3].into_iter().enumerate() { + put32(&mut row, index * 4, word); + } + row +} + +/// A page-list VA without bit 42 is stored minus 64 GiB in a 3-bit segment, +/// so only [64 GiB, 512 GiB) round-trips; stage33c's 0x200_0000 decoded to +/// 0x80_0200_0000 and faulted in the PM (see the H1 parameter-buffer fault notes). +pub(crate) const fn g18_pb_row_address_valid(page_list_gpu: u64) -> bool { + page_list_gpu & 0x7f == 0 + && page_list_gpu & (1 << 42) == 0 + && page_list_gpu >= 0x10_0000_0000 + && page_list_gpu < 0x80_0000_0000 +} + +/// The page-list VA the Parameter Manager reads back from a row with word 1 +/// bit 31 clear. `None` for the bit-42 form, which Linux never programs. +pub(crate) fn g18_pb_row_decoded_address(row: &[u8; G18_PB_ROW_BYTES]) -> Option { + let (word0, word1) = (get32(row, 0), get32(row, 4)); + if word1 & (1 << 31) != 0 { + return None; + } + Some(((u64::from(word1 >> 28) & 7) << 36) + (u64::from(word0 & !7) << 4) + 0x10_0000_0000) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/render.rs b/drivers/gpu/drm/asahi/g18/fw/render.rs new file mode 100644 index 0000000000000..0daeb4af7da47 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/render.rs @@ -0,0 +1,811 @@ +// SPDX-License-Identifier: MIT +//! Pure 26A428 G18 render register lowering over the unchanged M5 builder. +//! +//! See the G18 render-register notes for the oracle, supported envelope and unresolved +//! runtime policy. All compact pointers here are already in the native G18 +//! user-process view. The job, not this codec, chooses/maps that address view. + +use crate::g18::fw::render_params::{G17pRenderParameters, RegisterWrite}; +use crate::g18::fw::m5_render::{self as m5_render, M5RenderControls, M5RenderStageControls}; +use crate::g18::fw::m5_render_target as m5_render_target; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G18RenderError { + M5(m5_render::M5RenderBuildError), + RegisterLayout, + UnsupportedGeometry, + CompactAddress, + UnsupportedFeature, +} + +impl From for G18RenderError { + fn from(e: m5_render::M5RenderBuildError) -> Self { + Self::M5(e) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderStageControls { + /// TA descriptor +1188 | (+1190 << 16); fragment +820 | (+828 << 16). + /// This is NOT the M5 d499/d419/a5a1 tuple. Runtime must supply pool policy. + pub(crate) resource_word: u64, + /// Native literal window triple (TA a109/a129/a149; fragment a101/a121/a141). + /// 8c4f4d4..8c4f56c / 8c3b358..8c3b3f0; no descriptor read. + pub(crate) address_windows: [u64; 3], + /// (TA +12c4 / fragment +154) << 8 | (+12d0 / +988) & 63. + pub(crate) work_key: u64, + /// Optional USC statistics mapping from +1468 / +bb0 pointee chain. + /// Native helper 8c4bdb8: absent -> 0, present -> VA & 7ffffffffc0, + /// OR context id << 56 and bit 0. This is NOT ta/fragment_status | 1. + pub(crate) usc_statistics: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderControls { + pub(crate) ta: G18RenderStageControls, + pub(crate) fragment: G18RenderStageControls, + /// Native TA +12d4/+12d5, device +4d4/+6b9, base 10c08860. + pub(crate) ta_tile_control: u64, + /// Native device +10ac4 and descriptor +1659; bit 0 belongs to dispatch. + pub(crate) ta_dispatch_control: u64, + /// Native fragment +819 and policy/device state; baseline 200000ff03ef. + pub(crate) fragment_raster_control: u64, + /// Native 16068 nongeometry bits; +8cb/+ca0, device +6b0. + pub(crate) fragment_tile_flags: u64, + /// Native fragment +982/+983 and device +4d4/+6b9; baseline 8860. + pub(crate) fragment_tile_control: u64, + /// Native descriptor +ac8 & 0x7f, emitted at 8c3c208..8c3c230. + /// Independent of work-key low bits (+988) and utile dimensions. + pub(crate) fragment_control: u64, + /// Native +770 / register 10051 sample control. This is independent of + /// the application's tile-storage block count used by metadata +4d0. + pub(crate) fragment_sample_control: u64, + /// Native +8cb and device +10aa4; first 160a8, final has bit 5 set. + pub(crate) fragment_dispatch_control: u64, + /// Native +7a0/+7a8. The second EOT binding is independent of +790/+798. + pub(crate) store_pipeline_2_bind: u64, + pub(crate) store_pipeline_2: u64, + /// Native partial helper uses +7b0/+7b8 for both EOT binding pairs. + pub(crate) partial_tile_flags: u64, + /// Native TA loader +11f8/+1200 (second tilemap/layer metadata). + pub(crate) ta_cluster_tilemap: u64, + pub(crate) ta_cluster_layermeta: u64, + /// Native TA loader +1218/+1220/+1228/+1238 (optional cluster resources). + pub(crate) ta_cluster_meta: [u64; 4], + /// Native TA loader +1240/+1248/+1250, copied from target +d8/+e0/+e8. + pub(crate) ta_cluster_control: [u64; 3], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18Program { + pub(crate) registers: [RegisterWrite; N], + pub(crate) rce_count: u16, + pub(crate) command_offset: u32, +} + +impl G18Program { + pub(crate) const fn empty() -> Self { + Self { + registers: [RegisterWrite { + number: 0, + value: 0, + }; N], + rce_count: 0, + command_offset: 0, + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18FragmentPrograms { + pub(crate) main: G18Program<92>, + pub(crate) store: G18Program<17>, + pub(crate) resume: G18Program<24>, + pub(crate) load: G18Program<12>, +} + +struct List<'a, const N: usize> { + program: &'a mut G18Program, + len: usize, +} +impl<'a, const N: usize> List<'a, N> { + fn new(program: &'a mut G18Program) -> Self { + Self { program, len: 0 } + } + fn push(&mut self, number: u32, value: u64) -> Result<(), G18RenderError> { + if self.len == N { + return Err(G18RenderError::RegisterLayout); + } + self.program.registers[self.len] = RegisterWrite { number, value }; + self.len += 1; + Ok(()) + } + fn finish(self, rce_count: u16, command_offset: u32) -> Result<(), G18RenderError> { + if self.len != N { + return Err(G18RenderError::RegisterLayout); + } + self.program.rce_count = rce_count; + self.program.command_offset = command_offset; + Ok(()) + } +} + +/// msaa163: the one multisample render the G18 path admits, Apple's 4x +/// render of run112 workload 8 (the MSAA scope +/// notes, section 2): 32x16 utiles and four samples (register 0x10009 = +/// 0x6002), the rotated-grid sample table 0xeaa26e26 (0x10019 / 3D +0x48), +/// eight-byte samples (3D +0x1fb0 = 8). Only `g18_application` admits it, +/// and only with `g18_admit_msaa4=1`. `p.samples` stays 1: that field is +/// the unused `set_samples` convention (32x32 utiles, Y-doubled geometry), +/// whereas the UAPI already names the halved utile. +pub(crate) const G18_MSAA4_UTILE: (u8, u8) = (32, 16); +pub(crate) const G18_MSAA4_UTILE_CONFIG: u64 = 0x6002; +pub(crate) const G18_MSAA4_MULTISAMPLE_CONTROL: u64 = 0xeaa2_6e26; +pub(crate) const G18_MSAA4_SAMPLES: u8 = 4; +pub(crate) const G18_MSAA4_SAMPLE_SIZE: u8 = 8; +/// Native331 RGBA32 two/four-sample renders use 16-byte samples, with the +/// same geometry and sample control. +pub(crate) const G18_MSAA_WIDE_SAMPLE_SIZE: u8 = 16; +/// Native464 retains three RGBA32 targets: 48-byte samples in a 96-KiB tile. +pub(crate) const G18_MSAA_MRT_SAMPLE_SIZE: u8 = 48; +/// Register 0x10051 / 3D +0x1f38: samples << 16 (native 0x40000 at 4x). +pub(crate) const G18_MSAA4_SAMPLE_CONTROL: u64 = (G18_MSAA4_SAMPLES as u64) << 16; +/// Partial-load list 0x10039 bit 16: native 0x200280 at 1x, 0x210280 at 4x. +pub(crate) const G18_MSAA4_LOAD_TILE_FLAG: u64 = 0x1_0000; + +/// Native326: two samples retain a 32x32 utile and use diagonal positions. +pub(crate) const G18_MSAA2_UTILE: (u8, u8) = (32, 32); +pub(crate) const G18_MSAA2_UTILE_CONFIG: u64 = 0xa001; +pub(crate) const G18_MSAA2_MULTISAMPLE_CONTROL: u64 = 0x44cc; + +pub(crate) fn g18_msaa2(p: &G17pRenderParameters) -> bool { + (p.utile_width, p.utile_height) == G18_MSAA2_UTILE + && p.utile_config == G18_MSAA2_UTILE_CONFIG + && p.multisample_control == G18_MSAA2_MULTISAMPLE_CONTROL + && p.samples == 1 + // Native673 combines the two-sample positions and 32x32 utiles with + // layered target metadata and all five complete register lists. + && (1..=2048).contains(&p.layers) +} + +pub(crate) fn g18_multisample(p: &G17pRenderParameters) -> bool { + g18_msaa2(p) || g18_msaa4(p) +} + +/// Whether `p` is the admitted 4x render (see [`G18_MSAA4_UTILE`]). +pub(crate) fn g18_msaa4(p: &G17pRenderParameters) -> bool { + (p.utile_width, p.utile_height) == G18_MSAA4_UTILE + && p.utile_config == G18_MSAA4_UTILE_CONFIG + && p.multisample_control == G18_MSAA4_MULTISAMPLE_CONTROL + && p.samples == 1 + // Native618 compares layered 32x16 targets and all five register + // lists with the same four-sample positions and partial-load mode. + && (1..=2048).contains(&p.layers) +} + +/// Rasterization samples (3D +0x50): 4 for the admitted 4x render, else the +/// existing `p.samples`. +pub(crate) fn g18_sample_count(p: &G17pRenderParameters) -> u32 { + if g18_msaa4(p) { + u32::from(G18_MSAA4_SAMPLES) + } else if g18_msaa2(p) { + 2 + } else { + u32::from(p.samples) + } +} + +/// 3D +0x54 macro size in utiles, (x << 16) | y: the register builders' +/// macro_size (g17_render.rs geometry). 4x splits the utile in Y, so y +/// doubles (native 0x40004 -> 0x40008); 32x32 renders keep the one-utile +/// formula unchanged. +pub(crate) fn g18_macro_size(p: &G17pRenderParameters) -> u32 { + let tiles_x = (p.width + 31) / 32; + let tiles_y = (p.height + 31) / 32; + let macro_x = ((tiles_x + 3) / 4 + 3) & !3; + let macro_y = ((tiles_y + 3) / 4 + 3) & !3; + if g18_msaa4(p) { + let utiles_x = 32 / u32::from(G18_MSAA4_UTILE.0); + let utiles_y = 32 / u32::from(G18_MSAA4_UTILE.1); + ((macro_x * utiles_x) << 16) | (macro_y * utiles_y) + } else { + (macro_x << 16) | macro_y + } +} + +fn validate(p: &G17pRenderParameters) -> Result<(), G18RenderError> { + if !(1..=16384).contains(&p.width) + || !(1..=16384).contains(&p.height) + || !(1..=2048).contains(&p.layers) + || p.samples != 1 + || (!g18_multisample(p) + && ((p.utile_width, p.utile_height) != (32, 32) + || p.utile_config != 0xa000 + || p.multisample_control != 0x88)) + { + return Err(G18RenderError::UnsupportedGeometry); + } + if p.sampler_count != 0 + || p.sampler_array != 0 + || p.occlusion_query_base & 0xf != 0 + || p.ta_timestamp_start != 0 + || p.ta_timestamp_end != 0 + || p.fragment_timestamp_start != 0 + || p.fragment_timestamp_end != 0 + || p.absolute_pointers + || p.native_ta_registers != 0 + || p.bg_resource_prefix != 0 + || (p.layers == 1 && (p.stencil_aux_buffer != 0 + || p.stencil_stride != 0 || p.stencil_aux_stride != 0)) + || p.stencil_clear_value > 0xff + { + return Err(G18RenderError::UnsupportedFeature); + } + Ok(()) +} + +fn m5_controls(c: &G18RenderControls) -> M5RenderControls { + let stage = |s: G18RenderStageControls| M5RenderStageControls { + resource_flags: 0, + resource_control: 0, + ptc_max: 0, + work_key: s.work_key, + }; + M5RenderControls { + ta: stage(c.ta), + fragment: stage(c.fragment), + ta_tile_control: c.ta_tile_control, + ta_dispatch_control: c.ta_dispatch_control, + fragment_raster_control: c.fragment_raster_control, + fragment_tile_flags: c.fragment_tile_flags, + store_pipeline_2_bind: c.store_pipeline_2_bind, + store_pipeline_2: c.store_pipeline_2, + partial_store_pipeline_2_bind: 0, + partial_store_pipeline_2: 0, + partial_tile_flags: c.partial_tile_flags, + } +} + +/// Convert ONLY compact-address inputs for the shared M5 arithmetic. The +/// emitted values remain the caller's G18 view; no physical alias is invented. +fn m5_parameters(p: &G17pRenderParameters) -> Result { + validate(p)?; + if p.encoder >= 1 << 39 + || p.encoder & 3 != 0 + || [p.deflake_1, p.deflake_2, p.deflake_3] + .iter() + .any(|&v| v > u32::MAX as u64 || v & 3 != 0) + { + return Err(G18RenderError::CompactAddress); + } + let mut m = *p; + m.encoder += 0x10_0000_0000; + m.deflake_1 += 0x10_0000_0000; + m.deflake_2 += 0x10_0000_0000; + m.deflake_3 += 0x10_0000_0000; + Ok(m) +} + +#[inline(never)] +fn common_ta( + p: &G17pRenderParameters, + c: &M5RenderControls, +) -> Result<[RegisterWrite; 82], m5_render::M5RenderBuildError> { + m5_render::build_ta_registers(p, c) +} + +#[inline(never)] +fn common_fragment( + p: &G17pRenderParameters, + c: &G18RenderControls, +) -> Result<[RegisterWrite; 90], m5_render::M5RenderBuildError> { + m5_render::build_fragment_registers(p, &m5_controls(c)) +} + +#[inline(never)] +pub(crate) fn build_g18_ta_program( + p: &G17pRenderParameters, + c: &G18RenderControls, +) -> Result, G18RenderError> { + let mut program = G18Program::empty(); + build_g18_ta_program_into(&mut program, p, c)?; + Ok(program) +} + +/// H1 writes into heap-owned storage rather than nesting return-value arrays. +#[inline(never)] +pub(crate) fn build_g18_ta_program_into(program: &mut G18Program<91>, + p: &G17pRenderParameters, c: &G18RenderControls) -> Result<(), G18RenderError> { + let common = common_ta(&m5_parameters(p)?, &m5_controls(c))?; + let mut out = List::new(program); + for r in &common { + let (mut n, mut v) = (r.number, r.value); + match n { + 0x1748 | 0xd499 | 0xd419 | 0xa5a1 | 0xd471 | 0x14320 | 0x1740 => continue, + 0x1c8f8 => out.push(0xa5e9, c.ta.resource_word)?, + // Fixed J873 device capability, independent of user PPP flags. + // Native device +6b0 bit 16 is emitted as bit 9 (8c4f004). + 0x10121 => v |= 0x200, + 0x1c079 | 0x1c9d8 => { + v = p + .layermeta + .checked_sub(p.context_base) + .ok_or(G18RenderError::CompactAddress)? + } + 0x1a099 | 0x1a0a1 | 0x1a069 | 0x1a071 | 0x1a0c9 | 0x1a0d1 => n -= 0x10, + 0x1a0f1 => n = 0x1a0e1, + 0x1a149 => { + n = 0x1a111; + v = 1; + } + // Native owner +28 masks the TA cycle/scene slot to 64 bytes. + 0x1ca30 | 0x16c39 => v &= !63, + 0x14a1 => n = 0x11201, + 0x14308 => n = 0x10249, + 0x14318 => { + n = 0xa619; + v = c.ta.usc_statistics; + } + _ => {} + } + out.push(n, v)?; + match n { + 0x1c9c8 => { + out.push(0x1c041, c.ta_cluster_tilemap)?; + out.push(0x1c9d0, c.ta_cluster_tilemap)?; + } + 0x1c880 if out.len < 30 => { + out.push(0x1c948, c.ta_cluster_meta[0])?; + out.push(0x1c888, c.ta_cluster_meta[1])?; + out.push(0x1c890, c.ta_cluster_control[0])?; + out.push(0x1c918, c.ta_cluster_control[1])?; + } + 0x1c9d8 => { + out.push(0x1c089, c.ta_cluster_layermeta)?; + out.push(0x1c9e0, c.ta_cluster_layermeta)?; + out.push(0x16c41, c.ta_cluster_meta[2])?; + out.push(0x1ca40, c.ta_cluster_meta[2])?; + out.push(0x1c9a8, c.ta_cluster_control[2])?; + out.push(0x1c920, c.ta_cluster_meta[3])?; + } + 0x10169 => { + for (n, v) in [0xa109, 0xa129, 0xa149].iter().zip(c.ta.address_windows) { + out.push(*n, v)?; + } + } + _ => {} + } + } + // Native descriptor +1608; the final three writes are outside the RCE tag. + out.finish(88, 0x60)?; + Ok(()) +} + +#[inline(never)] +fn fragment_main( + program: &mut G18Program<92>, + p: &G17pRenderParameters, + c: &G18RenderControls, +) -> Result<(), G18RenderError> { + validate(p)?; + let common = common_fragment(p, c)?; + let mut out = List::new(program); + let mut tiles = 0; + for r in &common { + let (mut n, mut v) = (r.number, r.value); + match n { + 0xd491 | 0xd429 | 0xa5a9 | 0xd469 => continue, + 0x1739 => n = 0x8139, + 0x1731 => n = 0x8131, + 0x10791 => { + for (n, v) in [0xa101, 0xa121, 0xa141] + .iter() + .zip(c.fragment.address_windows) + { + out.push(*n, v)?; + } + } + 0x15321 => out.push(0xa5f1, c.fragment.resource_word)?, + 0x16060 => v = p.layermeta, + 0x16138 => v = c.fragment_tile_control, + 0x101e9 => v = c.fragment_control & 0x7f, + 0x10051 => v = c.fragment_sample_control, + 0x1ca28 => v &= !15, + 0x16068 => { + tiles += 1; + out.push( + 0x160a8, + c.fragment_dispatch_control | if tiles == 2 { 0x20 } else { 0 }, + )?; + } + 0x1a0a9 | 0x1a0b1 | 0x1a079 | 0x1a081 | 0x1a0d9 | 0x1a0e1 => n -= 0x10, + 0x1a0f9 => n = 0x1a0e9, + 0x1a151 => { + n = 0x1a119; + v = 1; + } + 0x1499 => n = 0x111f9, + 0x14048 => n = 0x10251, + 0x14080 => { + n = 0xa621; + v = c.fragment.usc_statistics; + } + _ => {} + } + out.push(n, v)?; + } + out.finish(88, 0xa0) +} + +#[inline(never)] +pub(crate) fn build_g18_fragment_programs( + p: &G17pRenderParameters, + c: &G18RenderControls, +) -> Result { + let mut programs = G18FragmentPrograms { + main: G18Program::empty(), + store: G18Program::empty(), + resume: G18Program::empty(), + load: G18Program::empty(), + }; + build_g18_fragment_programs_into(&mut programs, p, c)?; + Ok(programs) +} + +#[inline(never)] +pub(crate) fn build_g18_fragment_programs_into(programs: &mut G18FragmentPrograms, + p: &G17pRenderParameters, c: &G18RenderControls) -> Result<(), G18RenderError> { + fragment_main(&mut programs.main, p, c)?; + fragment_partials(programs, p, c) +} + +#[inline(never)] +fn fragment_partials( + programs: &mut G18FragmentPrograms, + p: &G17pRenderParameters, + c: &G18RenderControls, +) -> Result<(), G18RenderError> { + let mut mc = m5_controls(c); + mc.partial_store_pipeline_2_bind = p.partial_store_pipeline_bind; + mc.partial_store_pipeline_2 = p.partial_store_pipeline; + fn partial( + r: &[RegisterWrite], + offset: u32, + ) -> Result, G18RenderError> { + let mut program = G18Program::empty(); + let mut out = List::new(&mut program); + for r in r { + if r.number != 0x15359 { + out.push(r.number, r.value)?; + } + } + out.finish(N as u16, offset)?; + Ok(program) + } + let mut partial_p = *p; + // G18 helper 8c4c1d0 drops main-pass +8d2/+8d3 flags for a split. + partial_p.tile_config &= 0x1ffff; + partial_p.partial_depth_flags = p.depth_flags; + let mut load_mc = mc; + // Original G18 partial-load helper reads +790/+798 (main EOT), whereas + // store/resume read +7b0/+7b8. M5 used the partial pair for all three. + load_mc.partial_store_pipeline_2_bind = p.store_pipeline_bind; + load_mc.partial_store_pipeline_2 = p.store_pipeline; + programs.store = partial( + &m5_render::build_partial_store_registers(&partial_p, &mc)?, + 0x7c0, + )?; + programs.resume = partial(&m5_render::build_partial_resume_registers(p, &mc)?, 0xee0)?; + programs.load = partial( + &m5_render::build_partial_load_registers(p, &load_mc)?, + 0x1600, + )?; + // msaa163: native 4x keeps tile_config bit 16 in the partial-load list + // (0x10039 = 0x210280 against 0x200280 at 1x, MSAA notes s2.1; the M5 + // 4x load word +0x161c moves the same way). Store/resume do not move. + if g18_multisample(p) { + // Native326 retains the same partial-load flag at two samples. + for register in &mut programs.load.registers { + if register.number == 0x10039 { + register.value |= G18_MSAA4_LOAD_TILE_FLAG; + } + } + } + // Native G18 partial helpers retain main stencil flags and add the + // transfer required by each program when a plane is present. The + // ordinary main list already forwards these fields unchanged. + fn stencil_flags(program: &mut G18Program, + p: &G17pRenderParameters, transfer: u64, store: bool) { + let mask = (1 << 4) | (1 << 8) | (1 << 14) | (1 << 18); + let flags = (p.depth_flags & mask) + | if p.stencil_buffer != 0 { transfer } else { 0 } + | if store && p.stencil_aux_buffer != 0 { 1 << 8 } else { 0 }; + for register in &mut program.registers { + if register.number == 0x15319 { + register.value = (register.value & !mask) | flags; + } + } + } + stencil_flags(&mut programs.store, p, 1 << 18, true); + stencil_flags(&mut programs.resume, p, (1 << 14) | (1 << 18), true); + stencil_flags(&mut programs.load, p, 1 << 14, false); + Ok(()) +} + +pub(crate) fn g18_default_controls(p: &G17pRenderParameters) -> G18RenderControls { + let stage = G18RenderStageControls { + resource_word: 0, + address_windows: [0xffff_0000_0000, 0x70_0000_0000, 0x71_0000_0000], + work_key: 0, + usc_statistics: 0, + }; + G18RenderControls { + ta: stage, + fragment: stage, + ta_tile_control: 0x10c08860, + ta_dispatch_control: 0x700, + fragment_raster_control: 0x200000ff03ef, + fragment_tile_flags: 0x20_0003_6011, + fragment_tile_control: 0x8860, + // Original a/b/g1 user descriptors and working native run 112. + fragment_control: 0x1c, + // Capture parameters already carry native sample control here. + fragment_sample_control: p.tib_blocks, + fragment_dispatch_control: 0, + store_pipeline_2_bind: p.store_pipeline_bind, + store_pipeline_2: p.store_pipeline, + partial_tile_flags: 0x200000, + ta_cluster_tilemap: 0, + ta_cluster_layermeta: 0, + ta_cluster_meta: [0; 4], + ta_cluster_control: [0; 3], + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G18Capture { + ColorA, + ColorB, + GearsG1, + FragmentWitness, +} + +/// Captured user fields plus explicitly unallocated kernel-owned fields. +/// The job MUST replace resource addresses, IDs, cycle/metrics and policy. +/// Geometry is constructed for the capture's 256x256 single-layer target; +/// native oracle parity injects that same geometry into the owned RT loader. +pub(crate) fn g18_capture_parameters( + capture: G18Capture, +) -> (G17pRenderParameters, G18RenderControls) { + let mut p = G17pRenderParameters::source_rsrc8(); + p.width = 256; + p.height = 256; + p.context_base = 0; + p.tilemap = 0; + p.layermeta = 0; + p.heapmeta = 0; + p.tpc = 0; + p.deflake_1 = 0x68980; + p.deflake_2 = 0x680c0; + p.deflake_3 = 0x68000; + p.encoder = 0x18000; + p.ta_status = 0; + p.fragment_status = 0; + p.store_pipeline_bind = 0; + p.partial_store_pipeline_bind = 0; + p.load_pipeline_bind = 0xffff800000000040; + p.partial_load_pipeline_bind = p.load_pipeline_bind; + let (shader, aux, id) = match capture { + G18Capture::ColorA => (0x10000168000, 0x10000280000, 0x0e0260a2u64), + G18Capture::ColorB => (0x10000168000, 0x10000280000, 0x1c0268ba), + G18Capture::GearsG1 => (0x100001b0000, 0x100002c8000, 0x4a13d3dc), + G18Capture::FragmentWitness => (0x10000170000, 0x10000288000, 0x0c0060a2), + }; + p.load_pipeline = shader + 0x240; + p.partial_load_pipeline = shader; + p.store_pipeline = shader + 0x480; + p.partial_store_pipeline = p.store_pipeline; + p.scissor_array = 0; + p.depth_bias_array = 0; + p.aux_fb = aux; + p.utile_config = 0xa000; + p.multisample_control = 0x88; + // Native J873 device +6b0 bit 16 becomes PPP bit 9 (8c4f004). + // Keep the same 0x202 used by live native draws. + p.ppp_control = 0x202; + p.tib_blocks = 0; + p.tile_config = 0xc10280; + p.aux_fb_flags = if capture == G18Capture::GearsG1 { + 0x30c000 + } else { + 0x30c001 + }; + p.cycle = 0; + p.record_index = 0x7000000001; + p.lifecycle = id << 32; + p.fragment_lifecycle = id << 32; + p.merge_upper_x_bits = 0; + p.merge_upper_y_bits = 0; + p.ta_timestamp_start = 0; + p.ta_timestamp_end = 0; + p.fragment_timestamp_start = 0; + p.fragment_timestamp_end = 0; + if capture == G18Capture::GearsG1 { + p.depth_buffer = 0x100000c8000; + p.depth_aux_buffer = 0x10000108000; + p.depth_flags = 0x80044; + p.partial_depth_flags = 0x88044; + p.depth_dimensions = 0xff00ff; + } + let c = g18_default_controls(&p); + (p, c) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderTargetLayout { + pub(crate) tilemap_bytes: usize, + pub(crate) heapmeta_offset: usize, + pub(crate) heapmeta_bytes: usize, + pub(crate) target_bytes: usize, + pub(crate) tpc_bytes: usize, + pub(crate) deflake_bytes: usize, + pub(crate) aux_bytes: usize, +} + +/// Original G18 geometry and updateGtpOnlyBuffers packing for one layer, +/// 32x32 utiles and one sample. `kind` selects clustering, not physical +/// cluster count: native TPC sizing still uses both J873 clusters for kind 1. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18NativeTargetLayout { + pub layout: G18RenderTargetLayout, + pub slice_sizes: [usize; 8], + pub slice_offsets: [usize; 8], + pub cluster_control: [u64; 3], +} + +/// Native target layout for one layer. msaa163: `utile` is the render's +/// utile. The tilemap and the TPC slice scale with utiles per 32x32 tile, the +/// M5 rule (`m5_render_target.rs:36-43`) and the same term that doubles +/// size1/size3 (TA 0x1c0b1/0x1c850/0x1c0a9). Native 4x (32x16) measures TA +/// +0x788 = 0x2000 against 0x1000 at 1x (MSAA notes s2.2). Only the witnessed +/// 32x16 single-target layout (kind 1) is admitted besides 32x32, which is +/// unchanged. +pub(crate) fn g18_native_target_layout(w: u32, h: u32, kind: u32, utile: (u8, u8)) + -> Option { + g18_native_target_layout_layers(w, h, 1, kind, utile) +} + +/// Original G18 target preparation also allocates a tilemap and TPC for each +/// layer. Layer metadata is one separate 0x100-byte region, not one per layer. +/// Native598 executes the original parser and preparation through 2048 layers, +/// including the 32x16 four-sample kind-1 geometry. Submission admission remains +/// separate: allocation arithmetic alone does not qualify layered rendering. +pub(crate) fn g18_native_target_layout_layers( + w: u32, h: u32, layers: u16, kind: u32, utile: (u8, u8), +) -> Option { + if !(1..=16384).contains(&w) || !(1..=16384).contains(&h) + || !(1..=2048).contains(&layers) || !(1..=2).contains(&kind) { + return None; + } + let utiles = match utile { + (32, 32) => 1, + G18_MSAA4_UTILE if kind == 1 => (32 / G18_MSAA4_UTILE.0 as usize) + * (32 / G18_MSAA4_UTILE.1 as usize), + _ => return None, + }; + let mx = ((((w as usize + 31) / 32 + 3) / 4) + 3) & !3; + let my = ((((h as usize + 31) / 32 + 3) / 4) + 3) & !3; + let layers = usize::from(layers); + let tilemap = (5usize * 16 * mx * my * utiles).checked_mul(layers)?; + let layermeta_bytes = if layers > 1 { 0x100usize } else { 0 }; + // Explicit experiment policy, not inferred live macOS configuration: + // segment mode 4 (native table gives 64), range differences zero, 8 PB + // blocks of 512 KiB. The original generators receive these same inputs. + let mut slice_sizes = [0; 8]; + slice_sizes[0] = (2usize * 128 * mx * my * utiles).checked_mul(layers)?; + if kind == 2 { + slice_sizes[1] = tilemap.checked_mul(64)?; + slice_sizes[2] = layermeta_bytes * 64; + slice_sizes[4] = 0x680 + 2 * 0x300; + slice_sizes[5] = 2 * 0x5c0; + slice_sizes[6] = (((0x400000usize / 0x8000 + 0x17ff) / 0x1800) + 1) * 8 * 64; + slice_sizes[7] = 0x80; + } + let alignments = if kind == 2 { [0x40, 0x40, 0x100, 0, 0x40, 0x40, 0x80, 0x40] } + else { [0x40, 0, 0, 0, 0, 0, 0, 0] }; + let mut slice_offsets = [0; 8]; + let mut bytes = 0usize; + for n in 0..8 { + let align = alignments[n]; + if align != 0 { bytes = bytes.checked_add(align - 1)? & !(align - 1); } + slice_offsets[n] = bytes; + bytes = bytes.checked_add(slice_sizes[n])?; + } + let tilemap_bytes = tilemap.checked_add(0xfff)? & !0xfff; + let heapmeta_offset = tilemap_bytes.checked_add(layermeta_bytes)?; + let target_bytes = heapmeta_offset.checked_add(0x200 + 0x3fff)? & !0x3fff; + Some(G18NativeTargetLayout { + layout: G18RenderTargetLayout { tilemap_bytes, heapmeta_offset, + heapmeta_bytes: 0x200, target_bytes, + tpc_bytes: bytes.checked_add(0x3fff)? & !0x3fff, deflake_bytes: 0xc000, aux_bytes: 0x10000 }, + slice_sizes, slice_offsets, + cluster_control: if kind == 1 { [1, 0, 0x80] } else { [0x40, 4, 0x80] }, + }) +} + +/// Correctly rounded positive IEEE-754 division of 0x3fddb3d9 by a target +/// dimension. Integer arithmetic keeps the kernel free of FP/SIMD use. +pub(crate) fn g18_merge_coefficient(dimension: u32) -> Option { + if !(1..=16384).contains(&dimension) { return None; } + let b = 31 - dimension.leading_zeros(); + let shift = b + u32::from(0xddb3d9u64 < (u64::from(dimension) << (23 - b))); + let numerator = 0xddb3d9u64 << shift; + let denominator = u64::from(dimension); + let mut mantissa = numerator / denominator; + let remainder = numerator % denominator; + if remainder * 2 > denominator || (remainder * 2 == denominator && mantissa & 1 != 0) { + mantissa += 1; + } + Some(((127 - shift) << 23) + mantissa as u32 - (1 << 23)) +} + +/// Conservative 2x M5 storage pending native G18 geometry qualification. +/// Only H1/H2 geometry is admitted; clusters must be one or two. +pub(crate) fn g18_render_target_layout( + w: u32, + h: u32, + layers: u16, + utile: (u8, u8), + samples: u8, + clusters: u32, +) -> Option { + if layers != 1 || utile != (32, 32) || samples != 1 || !(1..=2).contains(&clusters) { + return None; + } + let m = m5_render_target::M5RenderTargetLayout::new(w, h, layers, utile.0, utile.1, clusters)?; + let tilemap_bytes = m.tilemap_bytes.checked_mul(2)?; + let heapmeta_offset = tilemap_bytes.checked_add(0xfff)? & !0xfff; + let heapmeta_bytes = m.heapmeta_bytes.checked_mul(2)?; + let target_bytes = core::cmp::max( + m.target_bytes.checked_mul(2)?, + heapmeta_offset + .checked_add(heapmeta_bytes)? + .checked_add(0x3fff)? + & !0x3fff, + ); + Some(G18RenderTargetLayout { + tilemap_bytes, + heapmeta_offset, + heapmeta_bytes, + target_bytes, + tpc_bytes: m.tpc_bytes.checked_mul(2)?, + deflake_bytes: 0xc000, + aux_bytes: 0x10000, + }) +} + +/// Full 0xc000-byte captured G18 deflake allocation. Viewport is at +2c00, +/// not M5's +9c0. Both colour and gears captures witness these eight dwords. +pub(crate) fn initialize_g18_deflake(page: &mut [u8], w: u32, h: u32) -> Option<()> { + if page.len() != 0xc000 || !(1..=16384).contains(&w) || !(1..=16384).contains(&h) { + return None; + } + fn half(v: u32) -> u32 { + let b = 31 - v.leading_zeros(); + ((b + 126) << 23) | ((v << (23 - b)) & 0x7fffff) + } + page.fill(0); + for (off, v) in [ + (0x2c00, 0x3000), + (0x2c04, 0x80000000 | ((w + 31) / 32 - 1)), + (0x2c08, (h + 31) / 32 - 1), + (0x2c10, half(w)), + (0x2c14, half(w)), + (0x2c18, half(h)), + (0x2c1c, half(h) | 0x80000000), + (0x2c24, 0x3f800000), + ] { + page[off..off + 4].copy_from_slice(&v.to_le_bytes()); + } + Some(()) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/render_assembly.rs b/drivers/gpu/drm/asahi/g18/fw/render_assembly.rs new file mode 100644 index 0000000000000..86ad9028ad369 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/render_assembly.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Pure runtime assembly shared by the kernel owner and the host dry-run. +//! Register arithmetic belongs to g18_render; command fields to render-cmd. +#![allow(dead_code)] +use crate::{g18::fw::render_params::G17pRenderParameters, g18::address_map as va, + g18::fw::graphics_payload::fnv64, g18::fw::render as regs, g18::fw::render_command as cmd, + g18::fw::sksm::M5SksmRce}; + +pub(crate) const TA_OFFSET: usize = 0; +pub(crate) const FRAGMENT_OFFSET: usize = 0x4000; +pub(crate) const TARGET_OFFSETS: [u64; 6] = [0, 0x20000, 0x40000, 0x50000, 0x60000, 0x80000]; +// Fixed virtual reservations, not eager allocations. Native tilemap/TPC sizes +// scale with layers: even a 32x32x64 target needs a 0x40000-byte TPC. Keep +// addresses stable when an application grows an already-owned target. +const NATIVE_TARGET_OFFSETS: [u64; 6] = + [0, 0x400000, 0xc00000, 0xc10000, 0xc20000, 0xc30000]; +pub(crate) const TARGET_NAMES: [&str; 6] = ["tilemap-heapmeta", "tpc", "deflake", "ta-status", "3d-status", "auxiliary"]; +pub(crate) const DIGEST_NAMES: [&str; 7] = ["ta-command", "3d-command", "ta-registers", + "3d-main", "3d-store", "3d-resume", "3d-load"]; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct TargetPlan { + pub layout: regs::G18RenderTargetLayout, + pub sizes: [usize; 6], + pub offsets: [u64; 6], + pub native: Option, +} + +pub(crate) fn target_plan(p: &G17pRenderParameters) -> Option { + target_plan_for_kind(p, crate::g18::uapi::render_policy::target_kind()) +} + +pub(crate) fn target_plan_for_kind(p: &G17pRenderParameters, kind: u32) -> Option { + // msaa163: the admitted 4x render (32x16 utiles) plans only with the + // native single-target layout, whose TPC/tilemap scale with its utiles; + // the legacy (kind 0) and clustered (kind 2) layouts have no 4x witness. + let msaa4 = regs::g18_msaa4(p); + if !(1..=2048).contains(&p.layers) || p.samples != 1 + || (!msaa4 && (p.utile_width, p.utile_height) != (32, 32)) + || (msaa4 && kind != 1) { + return None; + } + let native = if kind == 0 { None } else { + Some(regs::g18_native_target_layout_layers(p.width, p.height, p.layers, kind, + (p.utile_width, p.utile_height))?) + }; + let layout = if let Some(n) = native { n.layout } else { + regs::g18_render_target_layout(p.width, p.height, p.layers, + (p.utile_width, p.utile_height), p.samples, 2)? + }; + // Stable VAs across application resizes. The extra slices share the TPC + // allocation; reserve 4 MiB for it, covering 1280x720 without overlap. + let offsets = match kind { + 1 => NATIVE_TARGET_OFFSETS, + 2 => [0, 0x100000, 0x500000, 0x510000, 0x520000, 0x600000], + _ => TARGET_OFFSETS, + }; + let sizes = [layout.target_bytes, layout.tpc_bytes, layout.deflake_bytes, + 0x4000, 0x4000, layout.aux_bytes]; + for (i, size) in sizes.iter().enumerate() { + let end = offsets[i].checked_add(*size as u64)?; + if i + 1 < sizes.len() && end > offsets[i + 1] { return None; } + if end > va::GRAPHICS_TARGET_BYTES as u64 { return None; } + } + Some(TargetPlan { layout, sizes, offsets, native }) +} + +/// Native 3D uses full target VAs; TA subtracts the mapping's context base. +/// The owner maps identical backing at the compact and biased views. +pub(crate) fn apply_target_plan(p: &mut G17pRenderParameters, plan: &TargetPlan) { + let a = plan.offsets.map(|offset| va::GRAPHICS_TARGET_OFFSET + offset); + p.context_base = va::COMPACT_BIAS; + p.tilemap = va::COMPACT_BIAS + a[0]; + p.heapmeta = p.tilemap + plan.layout.heapmeta_offset as u64; + // Native target preparation places layer metadata directly after the + // rounded tilemap; the heap follows it. Legacy kind 0 keeps its layout. + p.layermeta = if plan.native.is_some() { + p.tilemap + plan.layout.tilemap_bytes as u64 + } else { p.heapmeta }; + p.tpc = va::COMPACT_BIAS + a[1]; + // Deflake has a separate task-address encoding; keep these compact. + p.deflake_3 = a[2]; p.deflake_2 = a[2] + 0xc0; p.deflake_1 = a[2] + 0x980; + p.ta_status = a[3]; p.fragment_status = a[4]; p.aux_fb = a[5]; + // Generator does not admit timestamp registers yet; command timestamp + // pointers are separately supplied by the owned command inputs. + p.ta_timestamp_start = 0; p.ta_timestamp_end = 0; + p.fragment_timestamp_start = 0; p.fragment_timestamp_end = 0; +} + +pub(crate) fn apply_target_controls(p: &mut G17pRenderParameters, + c: &mut regs::G18RenderControls, plan: &TargetPlan) { + if let Some(n) = plan.native { + let pointer = |i: usize| if n.slice_sizes[i] == 0 { 0 } + else { p.tpc - p.context_base + n.slice_offsets[i] as u64 }; + c.ta_cluster_tilemap = pointer(1); + c.ta_cluster_layermeta = pointer(2); + c.ta_cluster_meta = [pointer(4), pointer(5), pointer(6), pointer(7)]; + c.ta_cluster_control = n.cluster_control; + p.merge_upper_x_bits = regs::g18_merge_coefficient(p.width).expect("validated width"); + p.merge_upper_y_bits = regs::g18_merge_coefficient(p.height).expect("validated height"); + } +} + +#[derive(Debug)] +pub(crate) enum AssemblyError { Registers(regs::G18RenderError), Command(cmd::G18RenderCommandError) } +impl From for AssemblyError { fn from(e: regs::G18RenderError) -> Self { Self::Registers(e) } } +impl From for AssemblyError { fn from(e: cmd::G18RenderCommandError) -> Self { Self::Command(e) } } + +pub(crate) struct EncodedJob { pub rce: [[M5SksmRce; 4]; 2], pub digests: [u64; 7] } + +// Only integer fields: zero is a valid initial representation. Kept on the +// heap by the kernel caller, including across the register-builder calls. +pub(crate) struct Workspace { pub ta: regs::G18Program<91>, pub fragment: regs::G18FragmentPrograms } + +fn view(p: ®s::G18Program) -> cmd::G18ProgramRef<'_> { + cmd::G18ProgramRef { registers: &p.registers, rce_count: p.rce_count, command_offset: p.command_offset } +} + +#[inline(never)] +#[cfg(test)] +pub(crate) fn encode( + ta: &mut [u8; cmd::G18_TA_COMMAND_BYTES], fragment: &mut [u8; cmd::G18_3D_COMMAND_BYTES], + p: &G17pRenderParameters, controls: ®s::G18RenderControls, + inputs: &cmd::G18RenderCommandInputs, +) -> Result { + let mut workspace = Workspace { ta: regs::G18Program::empty(), fragment: regs::G18FragmentPrograms { + main: regs::G18Program::empty(), store: regs::G18Program::empty(), + resume: regs::G18Program::empty(), load: regs::G18Program::empty() } }; + encode_into(ta, fragment, p, controls, inputs, &mut workspace) +} + +#[inline(never)] +pub(crate) fn encode_into( + ta: &mut [u8; cmd::G18_TA_COMMAND_BYTES], fragment: &mut [u8; cmd::G18_3D_COMMAND_BYTES], + p: &G17pRenderParameters, controls: ®s::G18RenderControls, + inputs: &cmd::G18RenderCommandInputs, workspace: &mut Workspace, +) -> Result { + ta.fill(0); fragment.fill(0); + regs::build_g18_ta_program_into(&mut workspace.ta, p, controls)?; + regs::build_g18_fragment_programs_into(&mut workspace.fragment, p, controls)?; + let t = &workspace.ta; + let f = &workspace.fragment; + // Native main boundaries N-4 / N-2 / N, per render-cmd §2. + let programs = cmd::G18FragmentProgramRefs { main: view(&f.main), main_mid_count: 90, + store: view(&f.store), resume: view(&f.resume), load: view(&f.load) }; + let fr_rce = cmd::encode_g18_3d_command(fragment, + va::GRAPHICS_JOB_LOW + FRAGMENT_OFFSET as u64, inputs, &programs)?; + // The linked 3D tuple in inputs is finalized before encoding TA. + let ta_rce = cmd::encode_g18_ta_command(ta, va::GRAPHICS_JOB_LOW, inputs, &view(&t))?; + let digests = [fnv64(ta), fnv64(fragment), fnv64(&ta[0x60..0x60 + 91 * 12]), + fnv64(&fragment[0xa0..0xa0 + 92 * 12]), fnv64(&fragment[0x7c0..0x7c0 + 17 * 12]), + fnv64(&fragment[0xee0..0xee0 + 24 * 12]), fnv64(&fragment[0x1600..0x1600 + 12 * 12])]; + Ok(EncodedJob { rce: [ta_rce, fr_rce], digests }) +} + +/// Runtime variants are deliberately a closed set. H2 repeats three frames; +/// no caller may infer retirement from an output hash before the next frame. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RenderVariant { Color, TaOnly, Clear, GearsThreeFrames } +impl RenderVariant { + pub(crate) fn parse(value: u32) -> Option { match value { + 0 => Some(Self::Color), 1 => Some(Self::TaOnly), 2 => Some(Self::Clear), + 3 => Some(Self::GearsThreeFrames), _ => None, + } } + pub(crate) fn stage_count(self) -> u32 { if self == Self::TaOnly { 1 } else { 2 } } + pub(crate) fn frames(self) -> u32 { if self == Self::GearsThreeFrames { 3 } else { 1 } } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/render_command.rs b/drivers/gpu/drm/asahi/g18/fw/render_command.rs new file mode 100644 index 0000000000000..8acdb17cca17c --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/render_command.rs @@ -0,0 +1,925 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 (J873/T8152, 26A428 HAL400) TA and 3D firmware commands, the M6 +//! channel configuration, SKSM kick-entry operands and tag14 kicks. +//! +//! Every byte follows the original AGXTAChannelSKSM::submitBuffer +//! (0xfffffe0008c837ac) and AGX3DChannelSKSM::submitBuffer (0xfffffe0008bbb500) +//! field writers, the register-list placement of the HAL400 generators +//! (TAChannelSKSM::generateRegisterList 0xfffffe0008c4e1b4, +//! 3DChannelSKSM::generateRegisterListFor3D 0xfffffe0008c3abe8) and the +//! KSMKickQueue TA/3D wrappers (0xfffffe0008c4a9e4 / 0xfffffe0008c4a794). +//! Parity is established by a native render-command audit, +//! which executes those functions unmodified. +//! +//! These are pure serializers. They never compute register values (the +//! programs come from g18_render), never allocate, map or publish, and never +//! choose an address: every kernel-owned address is an input that the job +//! must own and have mapped in the context that dereferences it. Native field +//! names are kept as descriptor offsets (`dNNNN` = native host descriptor +//! +0xNNNN) wherever a guessed meaning would be unproven. +//! +//! Command selectors (first u32 of every inner-ring command): TA 0, 3D 1, +//! 3D fast blit 2 (AGFIChannelCommandFastBlit, 0x9e0 bytes, emitted only when +//! 3D descriptor +0x840 bit 0 is set; not encoded here), compute 3, +//! tag15 channel configuration 15, tag14 add-kicks 14. + +#![allow(dead_code)] + +use crate::g18::fw::render_params::RegisterWrite; +use crate::g18::fw::sksm::{self as m5_sksm, M5SksmDependency, M5SksmOperands, M5SksmQueue, M5SksmRce}; + +pub(crate) const G18_TA_COMMAND_BYTES: usize = 0x980; +pub(crate) const G18_3D_COMMAND_BYTES: usize = 0x2240; +/// Selector-2 3D fast-blit form. Recorded for completeness; never emitted. +pub(crate) const G18_FAST_BLIT_COMMAND_BYTES: usize = 0x9e0; +pub(crate) const G18_CHANNEL_CONFIG_BYTES: usize = 0x98; +pub(crate) const G18_M5_CHANNEL_CONFIG_BYTES: usize = 0x86; +pub(crate) const G18_KICK_BYTES: usize = 21; + +pub(crate) const G18_TA_SELECTOR: u32 = 0; +pub(crate) const G18_3D_SELECTOR: u32 = 1; +pub(crate) const G18_FAST_BLIT_SELECTOR: u32 = 2; +pub(crate) const G18_CONFIG_SELECTOR: u32 = 15; + +/// TA register program: one list at command +0x60. +pub(crate) const G18_TA_PROGRAM_OFFSET: u32 = 0x60; +/// 3D main, partial store, partial resume and partial load programs. +pub(crate) const G18_3D_PROGRAM_OFFSETS: [u32; 4] = [0xa0, 0x7c0, 0xee0, 0x1600]; +/// Each list slot: 0x700 bytes of 12-byte entries, then the GPU address of the +/// list (+0x700, u64) and its entry count (+0x708, u16). +const PROGRAM_AREA: usize = 0x700; +const REGISTER_BYTES: usize = 12; +pub(crate) const G18_PROGRAM_CAPACITY: usize = PROGRAM_AREA / REGISTER_BYTES; +/// The TA generator mirrors the first 0x1c898 value into command +0x85e and +/// later emits 0x1c899 with bit 0 set (0xfffffe0008c4f690 / 0x8c4fff0). +const TA_DISPATCH_REGISTER: u32 = 0x1c898; +/// generateRegisterListFor3D mirrors its 0x10038 value (appended as the +/// 64-bit entry 0x10039) into command +0x1f58 (0x8c3bfe8), then clears +0x1f60 +/// (0x8c3c0d0). Register numbers are matched without the width bit 0. +const FRAGMENT_TILE_REGISTER: u32 = 0x10038; + +/// Native KSMKickQueue QoS class table 0xfffffe00071524d8, index 3/4/5 for +/// TA/3D/CL when the scheduler has no override (+0xc68 == 0). +pub(crate) const G18_TA_QOS_CLASS: u8 = 12; +pub(crate) const G18_3D_QOS_CLASS: u8 = 24; +/// encodeTAKickEntry / encode3DKickEntry dependency-type argument. +pub(crate) const G18_TA_DEPENDENCY_TYPE: u8 = 2; +pub(crate) const G18_3D_DEPENDENCY_TYPE: u8 = 3; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum G18RenderCommandError { + ProgramTooLong, + ProgramOffset, + RceCount, + MissingDispatchMirror, + AddressOverflow, + InvalidCompletionState, + Kick, + Sksm, +} + +/// A borrowed register program, the slice form of g18_render's +/// `G18Program` (registers, native RCE metadata count, list offset). +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18ProgramRef<'a> { + pub registers: &'a [RegisterWrite], + /// Native generator metadata slot count, used as the SKSM RCE tag and the + /// command's first boundary (TA: N-3; 3D main: N-4). + pub rce_count: u16, + pub command_offset: u32, +} + +/// The four 3D programs. Native always encodes all four. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18FragmentProgramRefs<'a> { + /// `rce_count` is the +0x80 boundary (native N-4). + pub main: G18ProgramRef<'a>, + /// Second main boundary, +0x82 and +0x84 (native N-2, after 0x16020). + pub main_mid_count: u16, + pub store: G18ProgramRef<'a>, + pub resume: G18ProgramRef<'a>, + pub load: G18ProgramRef<'a>, +} + +/// Native per-stage completion identity copied into the command tail. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18StageCompletion { + /// Firmware-written stamp words: stamp-memory A/B base plus + /// IOGPUEventMachine::getStampOffset(queue) = queue * 4. + pub stamp_addresses: [u64; 2], + /// Event-machine stamp of this submission (vendevtCommandRec +0). + pub stamp: u32, + /// Kick-queue index (vendevtCommandRec +4; the qid on Linux). + pub queue_index: u32, + /// Channel +0x38. + pub ordinal: u32, + /// Host descriptor +0x48 nonzero. + pub has_wait: bool, + /// Hardware completion word, [`g18_render_completion`]. M5's + /// `(1 << 40) | generation` is this value for pass 1, stamp 0x100. + pub completion: u64, + /// Submission key: TA descriptor +0xe68 / 3D descriptor +0x100. Also + /// the key argument of the command-pointer publication. + pub key: u64, +} + +/// Hardware-stamp conversion shared by completion words and barrier stamps +/// (HAL400 convertEventMachineTSToHWTS arithmetic; identical to compute). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18HardwareStamp { + pub stamp: u32, + /// Kick-queue entry count (+0x10). + pub entries: u32, + /// Event-machine generation of the stamp's queue, half selected by the + /// stamp's sign (+8 for stamp < 0x80000000, else +16). + pub generation: u64, + /// Kick-queue ring base (+0x90). + pub base: u64, +} + +pub(crate) fn g18_hardware_stamp(p: &G18HardwareStamp) -> Result { + if p.entries == 0 { + return Err(G18RenderCommandError::InvalidCompletionState); + } + let value = (u64::from(p.stamp >> 8) | p.generation << 24).wrapping_sub(1); + let quotient = (value / u64::from(p.entries)) as u32; + let remainder = (value as u32).wrapping_sub(quotient.wrapping_mul(p.entries)); + let slot = (p.base as u32 & 0xff) + (remainder & 0xff); + let carry = slot / p.entries; + let index = (slot - carry * p.entries) & 0xff; + let phase = ((p.base >> 8) as u32) + .wrapping_add(quotient) + .wrapping_add(carry); + Ok(u64::from(index) | u64::from(phase) << 8) +} + +/// Completion word: hardware stamp | completion scratch payload << 40. The +/// payload is the 3D channel's per-pass counter (+0x1d4, incremented by the +/// 3D submit); the TA of the same pass carries the same value. +pub(crate) fn g18_render_completion( + p: &G18HardwareStamp, + pass: u8, +) -> Result { + Ok(g18_hardware_stamp(p)? | u64::from(pass) << 40) +} + +/// TA fields for the TA's own linked 3D submission (AGX3DWorkQueue:: +/// submitCommand 0x8c93938..0x8c93a50 writes them after the 3D submit). +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18TaLinked3d { + /// 3D channel +0x18 (its queue index) -> TA +0x79c. + pub queue_index: u32, + /// 3D channel +0x1d0 (the 3D's event stamp) -> TA +0x7a0. + pub stamp: u32, + /// TA descriptor +0x1580 bit 0 -> TA +0x7a4. + pub flag: u8, + /// 3D completion word -> TA +0x7a8 (40 bits) and +0x7b0/+0x7b1 (pass). + pub completion: u64, +} + +/// Native TA +0x89e policy bit: accelerator +0x6c0 against descriptor +/// +0x1658/+0x1198/+0x11a7 (0x8c83f6c..0x8c83fa4, 0x8c84448..0x8c8446c). +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18TaPolicy { + pub a6c0: u32, + pub d1658: u8, + pub d1198: u32, +} + +/// Optional four-address tail (+0x95f..+0x97f), as compute's +0xfe5. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ExtraAddresses { + /// Descriptor count word (TA +0x13d8, 3D +0xa90). + pub count: u32, + pub addresses: [u64; 4], + /// Accelerator +0x116b8 bits 0..1. + pub policy: u8, +} + +/// Every TA command field outside the register program. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18TaCommandFields { + /// AGXCommandQueue +0x628 before its increment. + pub sequence: u64, + pub completion: G18StageCompletion, + /// TA HardwareBufferID (PB descriptor-table slot), d1280. + pub pb_slot: u32, + /// Parameter management +0x1d4, d12a8. + pub pm_1d4: u32, + /// PB state object (firmware VA), d1288. Same value as tag15 +0x7a. + pub pb_state: u64, + /// Parameter management +0xa8[+0x2c], d1290. + pub pm_scene: u64, + /// Parameter management +0xb8, d12a0. + pub pm_b8: u64, + pub d12d0: u32, + pub d12c0: u32, + pub d12c8: u64, + pub d11b8: u64, + pub d11d8: u64, + pub linked_3d: G18TaLinked3d, + pub d1068: u64, + pub d1070: u64, + pub d10f8: u64, + pub d08a0: u8, + pub d08a4: u32, + pub d08a8: u32, + pub d08ac: u32, + pub d08b0: u64, + pub d1418: [u64; 2], + pub d1428: u64, + pub d1430: u64, + /// UMA pool state firmware VA (UMA data record +0x20), d1458. + pub uma_state: u64, + pub d1474: u8, + pub d11a4: u8, + pub d11a7: u8, + pub policy: G18TaPolicy, + pub extra: G18ExtraAddresses, +} + +/// Every 3D command field outside the four register programs. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G183dCommandFields { + pub sequence: u64, + pub completion: G18StageCompletion, + /// Completion scratch payload (3D channel +0x1d4 after this submit) -> +0x90. + pub pass: u8, + pub d158: u64, + pub d150: u32, + /// Descriptor +0x980 == 2 -> +0x2174. + pub d980_is_2: bool, + pub d08cf: u8, + pub dbbc: u8, + /// 3D side of the shared PM state (firmware VA of d940) -> +0x28. + pub pm_state: u64, + /// Parameter management +0xa8[+0x2c], d948 -> +0x30. + pub pm_scene: u64, + /// Parameter management +0xb8, d958 -> +0x38. + pub pm_b8: u64, + pub d08d8: u64, + pub d08f0: u64, + pub d08f8: u32, + pub d0900: u32, + pub d0908: u32, + pub d0910: [u64; 2], + pub d0920: u64, + pub d0930: u32, + pub d04e0: u64, + pub d04e8: u64, + pub d04f0: u64, + pub d0510: u64, + /// d518..d688 as 23 16-byte pairs, copied to +0x1d58..+0x1ec8. + pub d0518: [[u64; 2]; 23], + pub d0690: u64, + pub d0698: u64, + pub d06a0: [u64; 2], + pub d06b0: u64, + pub d06b8: [u64; 2], + pub d06d8: u64, + pub d06f0: u64, + pub d06f8: u64, + pub d0708: u64, + pub d0728: u64, + pub d0730: u64, + pub d0750: u64, + pub d0758: u64, + pub d0770: [u64; 2], + pub d0780: [u64; 2], + pub d0790: [u64; 2], + pub d07b0: [u64; 2], + pub d07c0: u64, + pub d07c8: u32, + pub d07cc: u64, + pub d07d4: u32, + pub d07d8: u64, + pub d07e8: u64, + pub d07f8: u32, + pub d04d0: u32, + pub d08c4: u8, + pub dc98: u32, + pub dc9c: u8, + pub d08a0: u8, + pub d08a4: u32, + pub d08a8: u32, + pub d08ac: u32, + pub dad0: [u64; 2], + pub dae0: u64, + pub dae8: u64, + /// UMA pool state firmware VA, dba0. + pub uma_state: u64, + /// Timestamp pair, d8b0/d8b8 -> +0x2198/+0x21a0. + pub d08b0: [u64; 2], + /// d8c5, d8c6, d8c7, d8cb bit 0 -> +0x2124..+0x2130. + pub d08c5_flags: [u8; 4], + pub d08cd: u8, + pub extra: G18ExtraAddresses, +} + +/// Inputs shared by both commands of a render pass (contract §6.2). +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18RenderCommandInputs { + /// TA descriptor +0x12c4 / 3D descriptor +0x154 (the GPU context ID). + pub context_id: u32, + /// Descriptor +0x198: the 0xc8-byte scheduler record (TA +0x10, 3D +0x20). + pub scheduler_record: u64, + pub ta: G18TaCommandFields, + pub fragment: G183dCommandFields, +} + +fn put8(out: &mut [u8], offset: usize, value: u8) { + out[offset] = value; +} + +fn put16(out: &mut [u8], offset: usize, value: u16) { + out[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn put128(out: &mut [u8], offset: usize, value: [u64; 2]) { + put64(out, offset, value[0]); + put64(out, offset + 8, value[1]); +} + +fn read32(out: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(out[offset..offset + 4].try_into().unwrap()) +} + +fn completion_tail(out: &mut [u8], base: usize, c: &G18StageCompletion) { + put64(out, base, c.stamp_addresses[0]); + put64(out, base + 8, c.stamp_addresses[1]); + put32(out, base + 0x10, c.stamp); + put32(out, base + 0x14, c.queue_index); + put32(out, base + 0x18, c.ordinal); + put32(out, base + 0x1c, u32::from(c.has_wait)); +} + +fn extra_tail(out: &mut [u8], flags: usize, addresses: usize, e: &G18ExtraAddresses) { + out[flags] &= 0xf2; + if e.count != 0 || e.addresses.iter().any(|&value| value != 0) { + out[flags] |= 1 | ((e.policy & 3) << 2); + for (index, value) in e.addresses.iter().enumerate() { + put64(out, addresses + index * 8, *value); + } + } +} + +/// Validate one program slot before any byte is written. +fn check_program(p: &G18ProgramRef<'_>, offset: u32) -> Result<(), G18RenderCommandError> { + if p.command_offset != offset { + return Err(G18RenderCommandError::ProgramOffset); + } + if p.registers.len() > G18_PROGRAM_CAPACITY { + return Err(G18RenderCommandError::ProgramTooLong); + } + if usize::from(p.rce_count) > p.registers.len() || p.rce_count > 0x7f { + return Err(G18RenderCommandError::RceCount); + } + Ok(()) +} + +/// Native RCEBufferEncoder entry: the header keeps the destination word's +/// bits 0xfffc0006, then the 64-bit value follows unaligned at +4. +fn place_program(out: &mut [u8], low_alias: u64, p: &G18ProgramRef<'_>) -> Result { + let offset = p.command_offset as usize; + let address = low_alias + .checked_add(u64::from(p.command_offset)) + .ok_or(G18RenderCommandError::AddressOverflow)?; + put16(out, offset + PROGRAM_AREA + 8, 0); + put64(out, offset + PROGRAM_AREA, address); + for (index, register) in p.registers.iter().enumerate() { + let entry = offset + index * REGISTER_BYTES; + let header = (read32(out, entry) & 0xfffc0006) | (register.number & 0x3fff9); + put32(out, entry, header); + put64(out, entry + 4, register.value); + } + put16(out, offset + PROGRAM_AREA + 8, p.registers.len() as u16); + Ok(address) +} + +/// Encode the TA firmware command (selector 0) and return the SKSM RCE list. +/// +/// `out` must hold the command allocation's current contents: like native, +/// untouched bytes and header bits 0xfffc0006 of each list entry are kept. +/// `low_alias` is the command's GPU address in the job context (native +/// descriptor +0x1008); the RCE list and +0x760 name it. +pub(crate) fn encode_g18_ta_command( + out: &mut [u8; G18_TA_COMMAND_BYTES], + low_alias: u64, + i: &G18RenderCommandInputs, + ta: &G18ProgramRef<'_>, +) -> Result<[M5SksmRce; 4], G18RenderCommandError> { + check_program(ta, G18_TA_PROGRAM_OFFSET)?; + let dispatch = ta + .registers + .iter() + .find(|r| r.number & !1 == TA_DISPATCH_REGISTER) + .ok_or(G18RenderCommandError::MissingDispatchMirror)? + .value; + let t = &i.ta; + put32(out, 0, G18_TA_SELECTOR); + put64(out, 4, t.sequence); + put32(out, 0xc, i.context_id); + put64(out, 0x937, t.d12c8); + put32(out, 0x8d6, t.d12c0); + put32(out, 0x48, t.d12d0); + put32(out, 0x18, t.pb_slot); + put32(out, 0x1c, t.pm_1d4); + put64(out, 0x20, t.pb_state); + put64(out, 0x28, t.pm_scene); + put64(out, 0x30, t.pm_b8); + put64(out, 0x10, i.scheduler_record); + put64(out, 0x780, t.d11b8); + put64(out, 0x788, t.d11d8); + put32(out, 0x79c, t.linked_3d.queue_index); + put32(out, 0x7a0, t.linked_3d.stamp); + put32(out, 0x7a4, u32::from(t.linked_3d.flag & 1)); + put64(out, 0x7a8, t.linked_3d.completion & 0xff_ffff_ffff); + let pass = (t.linked_3d.completion >> 40) as u8; + put8(out, 0x7b1, pass); + put8(out, 0x7b0, pass); + put64(out, 0x7b6, t.d1068); + put64(out, 0x7be, t.d1070); + put64(out, 0x7d6, t.d10f8); + put128(out, 0x7de, [0, 0]); + put64(out, 0x7ee, 0); + put32(out, 0x7b2, 0); + put32(out, 0x86a, u32::from(t.d08a0 & 1)); + put32(out, 0x86e, t.d08a4); + put32(out, 0x872, t.d08a8); + put32(out, 0x876, t.d08ac); + put128(out, 0x87a, [0, 0]); + put128(out, 0x90e, t.d1418); + put64(out, 0x91e, t.d1428); + put64(out, 0x926, t.d1430); + put64(out, 0x92e, t.uma_state); + put8(out, 0x936, t.d1474 & 1); + put64(out, 0x8fe, t.d08b0); + put32(out, 0x892, u32::from(t.d11a4 & 1)); + put32(out, 0x89a, u32::from(t.d11a7 & 1)); + put32(out, 0x89e, u32::from(g18_ta_policy_bit(&t.policy, t.d11a7))); + put32(out, 0x8a2, 0); + put64(out, 0x8f6, 0); + completion_tail(out, 0x8a6, &t.completion); + put64(out, 0x8c6, t.completion.completion); + put64(out, 0x8ce, t.completion.key); + // Register generator placement (zeroing first, as 0x8c4fd28..0x8c4fd30). + put32(out, 0x38, 0); + put16(out, 0x3c, 0); + put32(out, 0x40, 0); + let address = place_program(out, low_alias, ta)?; + put16(out, 0x38, ta.rce_count); + put16(out, 0x3a, ta.registers.len() as u16); + put16(out, 0x3c, ta.registers.len() as u16); + put64(out, 0x85e, dispatch); + extra_tail(out, 0x97f, 0x95f, &t.extra); + let empty = M5SksmRce { address: 0, tag: 0 }; + Ok([M5SksmRce { address, tag: ta.rce_count as u8 }, empty, empty, empty]) +} + +/// Encode the 3D firmware command (selector 1) with all four programs and +/// return the SKSM RCE list (main, store, resume, load). +/// +/// `low_alias` is the command's GPU address in the job context (native +/// descriptor +0x470). Untouched bytes of `out` are kept, as native does. +pub(crate) fn encode_g18_3d_command( + out: &mut [u8; G18_3D_COMMAND_BYTES], + low_alias: u64, + i: &G18RenderCommandInputs, + f: &G18FragmentProgramRefs<'_>, +) -> Result<[M5SksmRce; 4], G18RenderCommandError> { + let programs = [f.main, f.store, f.resume, f.load]; + for (program, offset) in programs.iter().zip(G18_3D_PROGRAM_OFFSETS) { + check_program(program, offset)?; + } + for partial in &programs[1..] { + if usize::from(partial.rce_count) != partial.registers.len() { + return Err(G18RenderCommandError::RceCount); + } + } + if f.main_mid_count < f.main.rce_count || usize::from(f.main_mid_count) > f.main.registers.len() { + return Err(G18RenderCommandError::RceCount); + } + let tile = f + .main + .registers + .iter() + .find(|r| r.number & !1 == FRAGMENT_TILE_REGISTER) + .ok_or(G18RenderCommandError::MissingDispatchMirror)? + .value; + let t = &i.fragment; + put32(out, 0, G18_3D_SELECTOR); + put64(out, 4, t.sequence); + put32(out, 0xc, i.context_id); + put64(out, 0x21d1, t.d158); + put32(out, 0x2170, t.d150); + put32(out, 0x2174, u32::from(t.d980_is_2)); + put32(out, 0x2100, u32::from(t.d08cf & 1)); + put32(out, 0x20f8, 0); + // generateFrgBarrierRegister is a G18 stub (0x8c39224): 24 zero bytes. + put128(out, 0x2178, [0, 0]); + put64(out, 0x2188, 0); + put8(out, 0x21d0, t.dbbc & 1); + put64(out, 0x20, i.scheduler_record); + put64(out, 0x28, t.pm_state); + put64(out, 0x30, t.pm_scene); + put64(out, 0x38, t.pm_b8); + put32(out, 0x58, t.d08f8); + put64(out, 0x40, t.d08d8); + put64(out, 0x48, t.d08f0); + put32(out, 0x50, t.d0900); + put32(out, 0x54, t.d0908); + put128(out, 0x60, t.d0910); + put64(out, 0x70, t.d0920); + put32(out, 0x78, t.d0930); + put64(out, 0x1d30, t.d04e8); + put64(out, 0x1d20, t.d04e0); + put64(out, 0x1d40, t.d04f0); + for (index, pair) in t.d0518.iter().enumerate() { + put128(out, 0x1d58 + index * 16, *pair); + } + put64(out, 0x1ec8, t.d0698); + put64(out, 0x1ed0, t.d06f8); + put64(out, 0x1ed8, t.d0730); + put64(out, 0x1f28, t.d06d8); + put128(out, 0x1ee0, t.d06a0); + put64(out, 0x1ef0, t.d0728); + put64(out, 0x1ef8, t.d06b0); + put64(out, 0x1f00, t.d0708); + put64(out, 0x1f08, t.d0758); + put64(out, 0x1f30, t.d06f0); + put128(out, 0x1f10, t.d06b8); + put64(out, 0x1f20, t.d0750); + put64(out, 0x1f68, t.d07c0); + put32(out, 0x1f70, t.d07c8); + put128(out, 0x1f48, t.d0780); + put128(out, 0x1f74, t.d0790); + put64(out, 0x1f88, t.d07cc); + put32(out, 0x1f90, t.d07d4); + put128(out, 0x1f94, t.d07b0); + put64(out, 0x1fa8, t.d07d8); + put32(out, 0x1fb0, t.d04d0); + put128(out, 0x1f38, t.d0770); + put64(out, 0x1d50, t.d0510); + put64(out, 0x1fc0, t.d07e8); + put32(out, 0x1fc8, t.d07f8); + put64(out, 0x1fd0, t.d0690); + out[0x1fdc..0x20dc].fill(0); + put32(out, 0x20dc, 0); + put128(out, 0x20e0, [0, 0]); + put32(out, 0x1fd8, 0); + put32(out, 0x20f0, 0); + put8(out, 0x90, t.pass); + put32(out, 0x20fc, u32::from(t.d08c4 & 1)); + put32(out, 0x21f9, t.dc98); + put8(out, 0x21fd, t.dc9c); + put32(out, 0x2104, u32::from(t.d08a0 & 1)); + put32(out, 0x2108, t.d08a4); + put32(out, 0x210c, t.d08a8); + put32(out, 0x2110, t.d08ac); + put128(out, 0x2114, [0, 0]); + put128(out, 0x21a8, t.dad0); + put64(out, 0x21b8, t.dae0); + put64(out, 0x21c0, t.dae8); + put64(out, 0x21c8, t.uma_state); + put128(out, 0x2198, t.d08b0); + completion_tail(out, 0x2140, &t.completion); + put64(out, 0x2160, t.completion.completion); + put64(out, 0x2190, 0); + for (index, flag) in t.d08c5_flags.iter().enumerate() { + put32(out, 0x2124 + index * 4, u32::from(flag & 1)); + } + put32(out, 0x2138, u32::from(t.d08cd & 1)); + put64(out, 0x2168, t.completion.key); + extra_tail(out, 0x221e, 0x21fe, &t.extra); + put32(out, 0x221f, 0); + // generateRegisterListFor3D placement (zeroing at 0x8c3c7e4..0x8c3c7ec). + put32(out, 0x80, 0); + put16(out, 0x84, 0); + put32(out, 0x88, 0); + let mut rce = [M5SksmRce { address: 0, tag: 0 }; 4]; + for (slot, program) in rce.iter_mut().zip(programs.iter()) { + let address = place_program(out, low_alias, program)?; + *slot = M5SksmRce { address, tag: program.rce_count as u8 }; + } + put64(out, 0x1f58, tile); + put64(out, 0x1f60, 0); + put16(out, 0x80, f.main.rce_count); + put16(out, 0x82, f.main_mid_count); + put16(out, 0x84, f.main_mid_count); + put16(out, 0x88, f.main.registers.len() as u16); + Ok(rce) +} + +fn g18_ta_policy_bit(p: &G18TaPolicy, d11a7: u8) -> bool { + if p.a6c0 == 0 || !(1..=2).contains(&p.d1658) || p.d1198 == 3 { + return false; + } + match p.a6c0 { + 1 | 4 => d11a7 & 1 == 0, + 3 => true, + _ => false, + } +} + +// ---- M6 tag15 channel configuration ---------------------------------------- + +/// Channel-configuration layout. M6 inserts a u32 flag at +0x6e and a u64 at +/// +0x72, moving the parameter-buffer tuple from +0x6e..+0x86 to +0x7a..+0x92. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ChannelForm { + /// T8142/25G76, 0x86 bytes (m5_channel.rs). + M5, + /// T8152/26A428, 0x98 bytes (GFX b000 tag15 handler 0x12824 reads 0x98). + M6, +} + +impl ChannelForm { + pub(crate) const fn bytes(self) -> usize { + match self { + ChannelForm::M5 => G18_M5_CHANNEL_CONFIG_BYTES, + ChannelForm::M6 => G18_CHANNEL_CONFIG_BYTES, + } + } +} + +/// UMA pool tuple (+0x36 state, +0x3e generation, +0x46 slot). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18ConfigUma { + pub state: u64, + /// FList +0x1c0 when the descriptor asks for it, else None (u64::MAX). + pub generation: Option, + pub slot: u32, +} + +/// Parameter-buffer tuple. Firmware b000 0x13708..0x137c0 caches 0x90 +/// bytes of `state` into its slot table (0x71420 + slot * 0x50) when the +/// generation changes, stores the GPU context at state +0x08 and sets the +/// queue/manager bits. Native TA writer 0x8c83a34..0x8c83a9c. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18ConfigParameterBuffer { + /// PB state object firmware VA (TA descriptor +0x1288). + pub state: u64, + /// Descriptor +0xdc8 object's +0x68 (PB generation). + pub generation: u64, + /// TA HardwareBufferID / PB table slot (+0x1280). + pub slot: u32, + /// TA descriptor +0x1578: the linked 3D queue index. + pub manager: u32, +} + +/// All native tag15 fields. Offsets are for the M6 form. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18ChannelConfig { + /// Kick queue +0x80/+0x88: SKSM entry ring low and firmware addresses. + pub ring_low: u64, + pub ring_high: u64, + pub queue_id: u16, + /// +0x1a: install the queue (TA d+0xe9d, 3D d+0x191, compute d+0x181). + pub install: bool, + /// Kick queue +0x1c. + pub priority: u32, + /// +0x22: stage word, TA 0 / 3D 1 / compute 2. Copied to the firmware + /// queue slot +0x20 (b000 0x12240/0x1241c). + pub data_master: u32, + /// +0x26 flag and +0x2a value (TA d+0xeaa/+0xeb0, 3D d+0x308/+0x310). + /// Native stores the value even when the flag is clear. + pub state_update_flag: bool, + pub state_update: u64, + pub context_id: u32, + pub uma: G18ConfigUma, + /// AGXCommandQueue +0x618 -> +0xe0: the scheduler control record. + pub channel_control: u64, + pub update_scheduler: bool, + /// AGXCommandQueue +0x868 -> +0x10. + pub scheduler_index: u32, + /// Channel +0x58 -> +0x44 (AGXChannel::getPid). + pub pid: u32, + /// Channel +0x58 -> +0x3c (AGXChannel::getSubPriority). Compute uses 2. + pub sub_priority: u32, + pub update_context: bool, + /// Descriptor +0x58 gart's +0x2b8 bit 0 (b000 sets/clears context bit). + pub gart_flag: bool, + /// M6 +0x6e flag and +0x72 value (TA d+0xe9f/+0xea0, 3D d+0x2fa/ + /// +0x300); the value is stored regardless of the flag. Absent on M5. + pub extension_flag: bool, + pub extension: u64, + pub parameter_buffer: Option, +} + +/// Encode a tag15 record. Native writes bytes 0..form.bytes() except the +/// trailing M6 padding +0x92..+0x98, which is left to the caller's buffer. +pub(crate) fn encode_channel_config( + form: ChannelForm, + p: &G18ChannelConfig, + out: &mut [u8], +) -> Result { + let bytes = form.bytes(); + if out.len() < bytes { + return Err(G18RenderCommandError::AddressOverflow); + } + put64(out, 0, u64::from(G18_CONFIG_SELECTOR)); + put64(out, 8, p.ring_low); + put64(out, 0x10, p.ring_high); + put16(out, 0x18, p.queue_id); + put32(out, 0x1a, u32::from(p.install)); + put32(out, 0x1e, p.priority); + put32(out, 0x22, p.data_master); + put32(out, 0x26, u32::from(p.state_update_flag)); + put64(out, 0x2a, p.state_update); + put32(out, 0x32, p.context_id); + put64(out, 0x36, p.uma.state); + put64(out, 0x3e, p.uma.generation.unwrap_or(u64::MAX)); + put32(out, 0x46, p.uma.slot); + put64(out, 0x4a, p.channel_control); + put32(out, 0x52, u32::from(p.update_scheduler)); + put32(out, 0x56, p.scheduler_index); + put32(out, 0x5a, p.pid); + put32(out, 0x5e, p.sub_priority); + put32(out, 0x62, u32::from(p.update_context)); + put32(out, 0x66, u32::from(p.gart_flag)); + put32(out, 0x6a, 0); + let pb = match form { + ChannelForm::M6 => { + put32(out, 0x6e, u32::from(p.extension_flag)); + put64(out, 0x72, p.extension); + 0x7a + } + ChannelForm::M5 => 0x6e, + }; + match p.parameter_buffer { + Some(b) => { + put64(out, pb, b.state); + put64(out, pb + 8, b.generation); + put32(out, pb + 0x10, b.slot); + put32(out, pb + 0x14, b.manager); + } + None => { + put64(out, pb, 0); + put64(out, pb + 8, u64::MAX); + put64(out, pb + 0x10, u64::MAX); + } + } + Ok(bytes) +} + +// ---- SKSM kick entry and tag14 kick ----------------------------------------- + +/// Graphics stage of a kick queue. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum G18RenderStage { + Ta, + ThreeD, +} + +impl G18RenderStage { + const fn dependency_type(self) -> u8 { + match self { + G18RenderStage::Ta => G18_TA_DEPENDENCY_TYPE, + G18RenderStage::ThreeD => G18_3D_DEPENDENCY_TYPE, + } + } + + pub(crate) const fn qos_class(self) -> u8 { + match self { + G18RenderStage::Ta => G18_TA_QOS_CLASS, + G18RenderStage::ThreeD => G18_3D_QOS_CLASS, + } + } + + const fn data_master(self) -> m5_sksm::M5DataMaster { + match self { + G18RenderStage::Ta => m5_sksm::M5DataMaster::Tiling, + G18RenderStage::ThreeD => m5_sksm::M5DataMaster::Fragment, + } + } +} + +/// One vendevtBarrierRec entry, before hardware-stamp conversion. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18Barrier { + /// Event-machine stamp of the producer (e.g. the TA's 0x100). + pub stamp: u32, + /// Producer's queue index; also its qid on Linux. + pub queue_index: u8, + /// Barrier record flag word: 1 selects the stage dependency type. + pub flag: u32, + /// Event-machine generation of the producer queue for this stamp. + pub generation: u64, +} + +/// AGX3DChannelSKSM::submitBuffer 0x8bbba14..0x8bbba80: move the first +/// barrier on the paired TA queue (3D descriptor +0xe60; 0x80 for fast blit) +/// to the front of the list, swapping it with entry 0. +pub(crate) fn g18_order_3d_barriers(barriers: &mut [G18Barrier], ta_queue_index: u32) { + if let Some(index) = barriers.iter().position(|b| u32::from(b.queue_index) == ta_queue_index) { + barriers.swap(0, index); + } +} + +/// Inputs of encodeTAKickEntry / encode3DKickEntry beyond the kick queue. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18KickEntryInputs<'a> { + pub stage: G18RenderStage, + /// Kick queue id/priority/current stamp/previous stamp (+0xc/+0x1c/ + /// +0x48/+0x98/+0xa0); `submitted` counts entries in flight. + pub queue: M5SksmQueue, + /// Kick queue +0x10 entries and +0x90 base (barrier conversion). + pub entries: u32, + pub base: u64, + /// This submission's event stamp and its queue's event generation, for + /// the implicit predecessor when the queue has no previous stamp. + pub stamp: u32, + pub generation: u64, + pub barriers: &'a [G18Barrier], + /// TA descriptor +0xfdc / 3D +0x444 (AGXSPipelineDependencyType). + pub pipeline_dependency: u32, + /// AGXCommandQueue +0x868 -> +0x10 (QoS byte 32). + pub scheduler_index: u8, + /// Per-stage QoS word (0xffff without a scheduler override). + pub qos_word: u16, + /// Command firmware VA and the channel's +0x80 queue address. + pub payload: [u64; 2], + pub rce: [M5SksmRce; 4], + /// Descriptor +0x4c bit 0 (TA) or (3D +0xc9c == 3 ? 1 : +0x4c) -> bit 1. + pub flag_bit1: bool, + /// TA +0x15f0 / 3D +0xb78 -> bit 61; TA +0x15f1 / 3D +0x838 -> bit 63. + pub flag_bit61: bool, + pub flag_bit63: bool, + /// Event flag word 3: bit 63 config +0xe20, bit 62 extra command + /// present, bit 61 accelerator +0x6b6 & 0xc0. + pub event_flags: u64, + /// Descriptor +0x178, or accelerator +0x11f28[index & 0x7f]. + pub auxiliary: u64, +} + +/// Build the HAL400 kick-entry operands for m5_sksm::prepare_m5_sksm_entry. +/// The dependency list follows encodeDependencyList 0x8c4a5a4. +pub(crate) fn g18_kick_entry_operands<'a>( + p: &G18KickEntryInputs<'_>, + barriers: &'a mut [M5SksmDependency], +) -> Result<(M5SksmOperands<'a>, u64), G18RenderCommandError> { + if barriers.len() < p.barriers.len() || p.barriers.len() >= 32 { + return Err(G18RenderCommandError::Sksm); + } + let kind = |flag: bool| if flag { p.stage.dependency_type() } else { 1 }; + for (out, b) in barriers.iter_mut().zip(p.barriers.iter()) { + let stamp = g18_hardware_stamp(&G18HardwareStamp { + stamp: b.stamp, entries: p.entries, generation: b.generation, base: p.base, + })?; + *out = M5SksmDependency { queue_id: b.queue_index & 0x7f, stamp, kind: kind(b.flag == 1) }; + } + let first = g18_hardware_stamp(&G18HardwareStamp { + stamp: p.stamp, entries: p.entries, generation: p.generation, base: p.base, + })? + .wrapping_sub(1); + let count = p.barriers.len(); + Ok((M5SksmOperands { + barriers: &barriers[..count], + first_predecessor: first, + implicit_dependency_type: kind(p.pipeline_dependency == 1), + mcache: None, + flag_bit1: p.flag_bit1, + flag_bit61: p.flag_bit61, + flag_bit63: p.flag_bit63, + payload: p.payload, + qos_byte_32: p.scheduler_index, + qos_class_40: p.stage.qos_class(), + qos_word_48: p.qos_word, + event_mask: [0, 0, 0, p.event_flags], + rce_kind: 0, + rce: p.rce, + auxiliary: p.auxiliary, + }, first)) +} + +/// Encode the kick entry into its 0x200-byte slot image. +pub(crate) fn encode_g18_kick_entry( + p: &G18KickEntryInputs<'_>, +) -> Result { + let mut storage = [M5SksmDependency { queue_id: 0, stamp: 0, kind: 0 }; 31]; + let (operands, _) = g18_kick_entry_operands(p, &mut storage)?; + m5_sksm::prepare_m5_sksm_entry(p.queue, operands).map_err(|_| G18RenderCommandError::Sksm) +} + +/// Native tag14 add-kicks record (TA 0x8c8430c..0x8c84334, 3D 0x8bbc8f0.. +/// 0x8bbc91c): count is the descriptor's submitted-kick total (NOP entries +/// plus this one), DM 0 for TA and 1 for 3D. +pub(crate) fn encode_g18_kick( + stage: G18RenderStage, + queue_id: u8, + priority: u8, + stamp: u64, + count: u16, +) -> Result<[u8; G18_KICK_BYTES], G18RenderCommandError> { + m5_sksm::encode_m5_sksm_add_kicks_for(stage.data_master(), queue_id, priority, stamp, count) + .map_err(|_| G18RenderCommandError::Kick) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/render_params.rs b/drivers/gpu/drm/asahi/g18/fw/render_params.rs new file mode 100644 index 0000000000000..893d174936fbc --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/render_params.rs @@ -0,0 +1,2877 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! Pure T8140/G17P render-command construction. +//! +//! The byte layout comes from m1n1 +//! `916548424c00c72176326e3811badedce5714a99`. The source profile carries the +//! hardware-tested TA and fragment register recipes, the compiler-derived EOT +//! resource binding `0x8`, full-width load binding, and native-aligned BG/EOT +//! program pointers. Descriptor construction also writes the duplicated structural +//! pipeline fields. Queue construction returns ordered memory writes only; +//! allocation, cache maintenance, barriers, doorbells, MMIO, and completion +//! stay with the retained runtime. + +const TA_REGISTER_COUNT: usize = 73; +const FRAGMENT_REGISTER_COUNT: usize = 89; + +/// Both embedded register lists match the kext EXACTLY -- same writes, same +/// unique registers, same order -- verified against 2048-byte captures of the +/// native TA and 3D command records (macos-trace recs2048-gfx.txt): +/// +/// TA 73 writes, 71 unique, list ends at +0x3cc +/// 3D 89 writes, 86 unique, list ends at +0x4cc +/// +/// A 1024-byte capture cuts the 3D list at exactly +0x400 and hides its last +/// 17 writes, which briefly looked like 14 registers "Metal never writes" -- +/// they were entries 72..85. Assert the counts so a future truncated capture +/// cannot produce that conclusion again. +const _: () = assert!(TA_REGISTER_COUNT == 73); +const _: () = assert!(FRAGMENT_REGISTER_COUNT == 89); +const PARTIAL_STORE_REGISTER_COUNT: usize = 16; +const PARTIAL_RESUME_REGISTER_COUNT: usize = 23; +const PARTIAL_LOAD_REGISTER_COUNT: usize = 10; +const REGISTER_BYTES: usize = 12; + +pub(crate) const TA_DESCRIPTOR_SIZE: usize = 0x9c0; +pub(crate) const FRAGMENT_DESCRIPTOR_SIZE: usize = 0x2240; +/// Native render-setup field copied through command descriptor +0xc00 to the +/// final 3D command tail at +0x2110. This is independent of SGX's GPC +/// performance-state map register. +const G17P_FRAGMENT_TAIL_SETUP_WORD: u32 = u32::MAX; +/// J700 shared-runtime flag snapshot copied by native 3D submit into type-1 +/// command lanes +0x2124/+0x2128. +/// +/// The kext resolves the accelerator's shared event-stamp/HWDSID internal +/// resource at +0x1a130, obtains its CPU mapping, reads byte bit 0 at mapping +/// +0x2ad/+0x2ae, and widens them before per-draw register generation. The +/// working Neo Metal snapshot contains 1/1. +const G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT: [u32; 2] = [1, 1]; +pub(crate) const G17P_EOT_RESOURCE_SPEC: u64 = 0x8; +pub(crate) const G17P_LOAD_RESOURCE_SPEC: u64 = 0x0007_8000_0000_0040; +pub(crate) const G17P_SOURCE_BG_PIPELINE: u64 = 0x0000_0100_0199_0240; +pub(crate) const G17P_SOURCE_EOT_PIPELINE: u64 = 0x0000_0100_0199_0640; +/// Native `mcacheApertureBufferSetup` writes its packed write ranges here in +/// the 3D firmware command record. The KSM entry points at this table. +pub(crate) const G17P_FRAGMENT_MCACHE_OFFSET: usize = 0x6a0; +/// The tiler's MCache aperture table. `mcacheApertureBufferSetup` is called for +/// the TA with `ksmBufferInfo = recTA+0x60` and writes its entries at +/// `ksmBufferInfo + 0x600 + i*0x10`, i.e. `recTA + 0x660` -- the exact analogue +/// of the 3D's `rec3D + 0x6a0`. +pub(crate) const G17P_TILING_MCACHE_OFFSET: usize = 0x660; +pub(crate) const G17P_MCACHE_MAPPING_SIZE: usize = 0x10; +/// Temporary diagnostic mirror of the four Register Command Encoder programs. +/// Native keeps the complete programs in the 3D command itself: the first is +/// at command +0xa0 and the remaining phases follow at a 0x720-byte stride. +/// Each phase's +0x700 trailer points back at that same embedded phase and its +/// +0x708 word carries the entry count. KSM must bind those complete embedded +/// structures, not this entries-only mirror. +pub(crate) const G17P_FRAGMENT_RCE_STORAGE_SIZE: usize = 0x4000; +pub(crate) const G17P_FRAGMENT_RCE_PROGRAM_OFFSET: u64 = 0xa0; +pub(crate) const G17P_FRAGMENT_RCE_PROGRAM_STRIDE: u64 = 0x720; + +const TILE_WIDTH: u64 = 32; +const TILE_HEIGHT: u64 = 32; +const MACRO_TILES_X: u64 = 4; +const MACRO_TILES_Y: u64 = 4; +const REGION_ENTRY_SIZE: u64 = 5; +const EVENT_SELECTOR: u32 = 0x0e; +const EVENT_SUBTYPE_BASE: u32 = 0x0001_0000; +const EVENT_RECORD_SIZE: usize = 0x40; +const RING_SLOT_SIZE: usize = 0x18; +const QUEUE_WRITE_OFFSET: u64 = 0x40; +/// Base added by G17P's compact VDM control-stream register (0x1c880). +/// The register retains the legacy 40-bit {TTBR selector, 39-bit offset} +/// shape, so lower-root user addresses are encoded relative to this base. +const G17P_USER_BASE: u64 = 0x10_0000_0000; +const G17P_VDM_USER_WINDOW_SIZE: u64 = 1 << 39; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RenderBuildError { + ZeroDimensions, + LayerCountOutOfRange, + UtileDimensions, + AddressBeforeContext, + UserAddressOutOfRange, + VdmAddressOutOfRange, + BufferTooSmall, + ArithmeticOverflow, + QueueGridOutOfRange, + RingSlotOutOfRange, + McacheAddressOutOfRange, + McacheRangeEmpty, + McacheSelectorOutOfRange, +} + +impl RenderBuildError { + /// Stable short name for one descriptor-build refusal. Callers flatten this + /// enum to `EINVAL`, which erased the distinction between a geometry + /// rejection and an address that sits below the render context base. + pub(crate) fn name(&self) -> &'static str { + match self { + Self::ZeroDimensions => "zero-dimensions", + Self::LayerCountOutOfRange => "layer-count-out-of-range", + Self::UtileDimensions => "utile-dimensions", + Self::AddressBeforeContext => "address-before-context", + Self::UserAddressOutOfRange => "user-address-out-of-range", + Self::VdmAddressOutOfRange => "vdm-address-out-of-range", + Self::BufferTooSmall => "buffer-too-small", + Self::ArithmeticOverflow => "arithmetic-overflow", + Self::QueueGridOutOfRange => "queue-grid-out-of-range", + Self::RingSlotOutOfRange => "ring-slot-out-of-range", + Self::McacheAddressOutOfRange => "mcache-address-out-of-range", + Self::McacheRangeEmpty => "mcache-range-empty", + Self::McacheSelectorOutOfRange => "mcache-selector-out-of-range", + } + } +} + +/// One resource-derived G17P MCache write-range mapping. +/// +/// This is the exact input shape consumed by +/// `AGXAcceleratorG16::halMCacheApertureSetup`: address and size are in +/// 128-byte cache lines; `hwsid` is the allocated hardware DSID; `selector` +/// names the command's GART/UAT context; and `range_shift` is the native +/// write-range class (clamped to four by the kext). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PMcacheMapping { + pub(crate) address: u64, + pub(crate) size_units: u32, + pub(crate) hwsid: u8, + pub(crate) selector: u8, + pub(crate) range_shift: u16, +} + +/// Encode the two qwords written by the G17P kext at 3D descriptor +/// `+0x6a0 + index * 0x10`. +pub(crate) fn encode_g17p_mcache_mapping( + mapping: G17PMcacheMapping, +) -> Result<[u8; G17P_MCACHE_MAPPING_SIZE], RenderBuildError> { + if mapping.address & 0x7f != 0 || mapping.address >> 48 != 0 { + return Err(RenderBuildError::McacheAddressOutOfRange); + } + if mapping.size_units == 0 { + return Err(RenderBuildError::McacheRangeEmpty); + } + if mapping.selector > 0x3f { + return Err(RenderBuildError::McacheSelectorOutOfRange); + } + + let start = mapping.address >> 7; + let end = start + .checked_add(mapping.size_units as u64) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + // qword 1 reserves bits 41..46 for the range policy. + if start >> 41 != 0 || end >> 41 != 0 { + return Err(RenderBuildError::McacheAddressOutOfRange); + } + + let q0 = start + | ((mapping.selector as u64) << 41) + | ((mapping.hwsid as u64) << 47) + | (1u64 << 55); + let range_shift = core::cmp::min(mapping.range_shift, 4) as u32; + let q1 = end | (1u64 << 41) | (1u64 << (42 + range_shift)); + let mut out = [0u8; G17P_MCACHE_MAPPING_SIZE]; + out[..8].copy_from_slice(&q0.to_le_bytes()); + out[8..].copy_from_slice(&q1.to_le_bytes()); + Ok(out) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWrite { + pub(crate) number: u32, + pub(crate) value: u64, +} + +impl RegisterWrite { + const fn new(number: u32, value: u64) -> Self { + Self { number, value } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17pRenderParameters { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) context_base: u64, + /// Allocation epoch of the trusted execution-context lease, shared by TA/3D. + pub(crate) context_generation: u8, + pub(crate) tilemap: u64, + /// Per-layer tiler metadata, which native places between the tilemap and + /// the heap metadata. It is empty, and so equal to `heapmeta`, for a + /// single-layer target. + pub(crate) layermeta: u64, + pub(crate) heapmeta: u64, + pub(crate) tpc: u64, + /// Geometry size of the owned TPC subregion, before capacity page rounding. + pub(crate) tpc_size: u64, + pub(crate) deflake_1: u64, + pub(crate) deflake_2: u64, + pub(crate) deflake_3: u64, + pub(crate) encoder: u64, + pub(crate) ta_status: u64, + pub(crate) fragment_status: u64, + pub(crate) store_pipeline_bind: u64, + pub(crate) store_pipeline: u64, + pub(crate) load_pipeline_bind: u64, + pub(crate) load_pipeline: u64, + pub(crate) partial_store_pipeline_bind: u64, + pub(crate) partial_store_pipeline: u64, + pub(crate) partial_load_pipeline_bind: u64, + pub(crate) partial_load_pipeline: u64, + pub(crate) scissor_array: u64, + pub(crate) depth_bias_array: u64, + pub(crate) aux_fb: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth_buffer: u64, + pub(crate) stencil_buffer: u64, + pub(crate) depth_aux_buffer: u64, + pub(crate) stencil_aux_buffer: u64, + pub(crate) depth_stride: u64, + pub(crate) stencil_stride: u64, + pub(crate) depth_aux_stride: u64, + pub(crate) stencil_aux_stride: u64, + pub(crate) depth_clear_value_bits: u32, + pub(crate) stencil_clear_value: u32, + pub(crate) depth_flags: u64, + /// ZLS control used by every firmware partial-store/load/resume phase. + /// Unlike the pass-boundary control, an attached depth/stencil surface + /// must be both loaded and stored across an internal split. + pub(crate) partial_depth_flags: u64, + pub(crate) depth_dimensions: u64, + pub(crate) merge_upper_x_bits: u32, + pub(crate) merge_upper_y_bits: u32, + pub(crate) layers: u16, + /// Sample count: 1, 2 or 4. Defaults to 1, so every existing parameter + /// vector keeps its exact register program. + pub(crate) samples: u8, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) utile_config: u64, + pub(crate) multisample_control: u64, + pub(crate) ppp_control: u64, + pub(crate) tib_blocks: u64, + pub(crate) tile_config: u64, + pub(crate) aux_fb_flags: u64, + pub(crate) aux_fb_page_count: u64, + pub(crate) cycle: u64, + pub(crate) record_index: u64, + /// TA event correlation written to 0x1ca10/0x014a1/0x0a349. + /// Native sources this independently from the fragment correlation. + pub(crate) lifecycle: u64, + /// Fragment event correlation written to 0x160e0/0x01499/0x0a341. + pub(crate) fragment_lifecycle: u64, + pub(crate) sampler_array: u64, + pub(crate) sampler_count: u32, + pub(crate) process_empty_tiles: bool, + pub(crate) ta_timestamp_start: u64, + pub(crate) ta_timestamp_end: u64, + pub(crate) fragment_timestamp_start: u64, + pub(crate) fragment_timestamp_end: u64, + /// Write the VDM control-stream register (0x1c880) as a full GPU virtual + /// address instead of the native user-process compact form. + /// + /// SCOPE CORRECTED against the macOS oracle. This flag once covered every + /// register `ta_offset` feeds, justified from the legacy G14X path where + /// all of them are whole pointers. A real Metal render pass on THIS chip + /// says otherwise: the captured TA command record writes tilemap, TPC and + /// heapmeta as small context-relative offsets (see `ta_offset`). The legacy + /// encoding does not transfer to G17P, so those six registers are always + /// relative now and this flag applies to the VDM stream alone. Native first + /// lowers the task VA to a task-relative offset, maps the same backing at + /// 0x1000000000 + offset in the GART window, and writes that offset here. + /// Linux userspace supplies the full GART-alias VA; this kernel subtracts + /// the architectural 0x1000000000 base. Hardware adds it back for compact + /// pointers whose bit 39 is clear. + pub(crate) absolute_pointers: bool, + /// + /// Legacy diagnostic OR-mask for the BG resource-specifier high 17 bits. + /// Native calls this field PassControlMaskType and derives it from shader + /// outputs, including extension of the highest active pass bit; it is not + /// an address class or a universal constant. Main and partial shaders can + /// legitimately differ. Default zero preserves the userspace mask; + /// a nonzero value is an explicit diagnostic override only. + pub(crate) bg_resource_prefix: u64, + /// Historical opt-in diagnostic changing byte 2 of the USC private-memory + /// budget qword at TA +0x93d. It must not override +0x94d: that byte is the + /// allocation generation of the execution context, not a PM mode flag. + /// The captured budget is not yet derived from our pool allocation. + pub(crate) native_pm_bytes: bool, + /// Hardware-buffer ID allocated for this TA command. Native allocates it + /// from the accelerator's 0x7f-entry `AGXHardwareBufferIDManager`, stores + /// it in the host-side `AGXTACommandDescriptor` at +0x1248/+0x928, and the + /// register generator emits that value as register 0x1c830. Those two host + /// offsets are not fields in this final 0x9c0-byte firmware command. + pub(crate) ta_hardware_buffer_id: u32, + /// HardwareBufferID of the retained USC private-memory FList shared by TA + /// and 3D. Native allocates this from the USC manager (accelerator + /// +0x133c8), independently of the TA parameter-buffer manager above, and + /// emits the same ID in TA register 0x14308 and fragment register 0x14048. + pub(crate) usc_flist_hardware_buffer_id: u32, + /// Diagnostic capture-value bitmask retained for fields whose proper + /// inputs are not modeled yet. In particular, the 0xa5a* values are + /// assembled from workload-derived USC pool limits. Their captured values + /// are not architectural literals and must not become defaults. + pub(crate) native_ta_registers: u32, + /// Retained GPU-ID diagnostic snapshot, not the fragment parameter-buffer + /// owner. The old 0x1c838 use confused host descriptor +0x928 with a + /// different object's transient raw-command +0x928 field. + pub(crate) gpc_perf_state_map: u32, + /// Low nibble of SGX 0xe01480, copied by native 3D submit to tail +0x210c. + pub(crate) gpc_perf_state_map_low: u32, + /// Bit 0 of SGX 0xe0141c, copied by native 3D submit to tail +0x2104. + pub(crate) gpc_perf_state_control: u32, + /// GPU address of the selected native 0x80 Scene-state entry, or 0 for + /// none. The command-record binding is carried by `RenderDescriptorObjects` + /// in both TA and 3D; this copy is retained for diagnostics. + pub(crate) parameter_buffer: u64, +} + +impl G17pRenderParameters { + /// Apply the witnessed multisample register triple for a sample count. + /// + /// Captured constants, not derived: from Apple's own renders on this part + /// (logs/m5-msaa-native-hv-20260915-101837) at one, two and four samples, + /// with a same-mode control in a second oracle process establishing the + /// noise floor. `multisample_control` is the sample POSITION table -- + /// 0x88 is the centre (8,8), 0x44cc is (4,4),(12,12), and 0xeaa26e26 + /// unpacks to (14,10),(10,2),(6,14),(2,6), the rotated-grid 4x pattern in + /// 1/16 pixel units. `utile_config`'s low nibble is log2(samples); + /// `tib_blocks` is samples << 16, zero at one sample. + pub(crate) fn set_samples(&mut self, samples: u8) -> bool { + let (multisample_control, utile_config, tib_blocks) = match samples { + 1 => (0x88, 0xa000, 0), + 2 => (0x44cc, 0xa001, 0x2_0000), + 4 => (0xeaa2_6e26, 0x6002, 0x4_0000), + _ => return false, + }; + self.samples = samples; + self.multisample_control = multisample_control; + self.utile_config = utile_config; + self.tib_blocks = tib_blocks; + true + } + + pub(crate) const fn source_rsrc8() -> Self { + Self { + width: 64, + height: 64, + context_base: 0x0000_0010_0000_0000, + context_generation: 1, + tilemap: 0x0000_0010_001b_0000, + layermeta: 0x0000_0010_001b_5000, + heapmeta: 0x0000_0010_001b_5000, + tpc: 0x0000_0010_0024_0000, + tpc_size: 0x800, + deflake_1: 0x0000_0010_0006_82a0, + deflake_2: 0x0000_0010_0006_8020, + deflake_3: 0x0000_0010_0006_8000, + encoder: 0x0000_0010_0001_8000, + ta_status: 0x0000_0010_0007_8000, + fragment_status: 0x0000_0010_001a_8000, + store_pipeline_bind: G17P_EOT_RESOURCE_SPEC, + store_pipeline: G17P_SOURCE_EOT_PIPELINE, + load_pipeline_bind: G17P_LOAD_RESOURCE_SPEC, + load_pipeline: G17P_SOURCE_BG_PIPELINE, + partial_store_pipeline_bind: 0, + partial_store_pipeline: 0, + partial_load_pipeline_bind: 0, + partial_load_pipeline: 0, + scissor_array: 0x0000_0100_019a_0000, + depth_bias_array: 0x0000_0100_01af_8000, + aux_fb: 0x0000_0100_01aa_8000, + occlusion_query_base: 0, + depth_buffer: 0, + stencil_buffer: 0, + depth_aux_buffer: 0, + stencil_aux_buffer: 0, + depth_stride: 0, + stencil_stride: 0, + depth_aux_stride: 0, + stencil_aux_stride: 0, + depth_clear_value_bits: 0x3f80_0000, + stencil_clear_value: 0, + depth_flags: 0, + partial_depth_flags: 0, + depth_dimensions: 0, + merge_upper_x_bits: 0x3cdd_b3d9, + merge_upper_y_bits: 0x3cdd_b3d9, + layers: 1, + samples: 1, + utile_width: 32, + utile_height: 32, + utile_config: 0xa000, + multisample_control: 0x88, + ppp_control: 0x202, + tib_blocks: 8, + tile_config: 0x10280, + aux_fb_flags: 0xc001, + aux_fb_page_count: 0x10_0000, + cycle: 0x17_8020, + record_index: 0x8_0005, + lifecycle: 0, + fragment_lifecycle: 0, + sampler_array: 0, + sampler_count: 0, + process_empty_tiles: true, + ta_timestamp_start: 0xffff_fc20_0002_4c68, + ta_timestamp_end: 0, + fragment_timestamp_start: 0xffff_fc20_0002_4c68, + fragment_timestamp_end: 0xffff_fc20_0002_4c70, + absolute_pointers: false, + bg_resource_prefix: 0, + native_pm_bytes: false, + ta_hardware_buffer_id: 0, + usc_flist_hardware_buffer_id: 0, + native_ta_registers: 0, + gpc_perf_state_map: 0, + gpc_perf_state_map_low: 0, + gpc_perf_state_control: 0, + parameter_buffer: 0, + } + } +} + +#[derive(Debug, Copy, Clone)] +struct Geometry { + tiles_x: u64, + tiles_y: u64, + size1: u64, + size2: u64, + size3: u64, + x_blocks: u64, + y_blocks: u64, + screen: u64, + pixels: u64, + macro_size: u64, +} + +const fn align(value: u64, alignment: u64) -> u64 { + (value + alignment - 1) & !(alignment - 1) +} + +fn geometry(parameters: &G17pRenderParameters) -> Result { + if parameters.width == 0 || parameters.height == 0 { + return Err(RenderBuildError::ZeroDimensions); + } + if parameters.layers == 0 || parameters.layers > 2048 { + return Err(RenderBuildError::LayerCountOutOfRange); + } + + let width = parameters.width as u64; + let height = parameters.height as u64; + let tiles_x = (width + TILE_WIDTH - 1) / TILE_WIDTH; + let tiles_y = (height + TILE_HEIGHT - 1) / TILE_HEIGHT; + let (utile_width, utile_height) = (parameters.utile_width, parameters.utile_height); + if !matches!((utile_width, utile_height), (32, 32) | (32, 16) | (16, 16)) { + return Err(RenderBuildError::UtileDimensions); + } + let utiles_per_tile_x = TILE_WIDTH / utile_width as u64; + let utiles_per_tile_y = TILE_HEIGHT / utile_height as u64; + // Four samples split the utile in Y. Measured against Apple's own + // four-sample render: size1 0x14 -> 0x28, size3 0x20 -> 0x40 and + // macro_size 0x40004 -> 0x40008, where only the y term moves. All three + // follow from doubling this one term; two samples do not split. + let utiles_per_tile_y = if parameters.samples == 4 { + utiles_per_tile_y * 2 + } else { + utiles_per_tile_y + }; + let utiles_per_tile = utiles_per_tile_x * utiles_per_tile_y; + let mtile_x1 = align((tiles_x + MACRO_TILES_X - 1) / MACRO_TILES_X, 4); + let mtile_y1 = align((tiles_y + MACRO_TILES_Y - 1) / MACRO_TILES_Y, 4); + let mtile_x2 = 2 * mtile_x1; + let mtile_x3 = 3 * mtile_x1; + let mtile_y2 = 2 * mtile_y1; + let mtile_y3 = 3 * mtile_y1; + let mtile_stride = mtile_x1 * mtile_y1; + let size1 = (REGION_ENTRY_SIZE * mtile_stride * utiles_per_tile + 3) / 4; + + Ok(Geometry { + tiles_x, + tiles_y, + size1, + size2: mtile_stride, + size3: 2 * mtile_stride * utiles_per_tile, + x_blocks: mtile_x3 | (mtile_x2 << 9) | (mtile_x1 << 18), + y_blocks: mtile_y3 | (mtile_y2 << 9) | (mtile_y1 << 18), + screen: ((tiles_y - 1) << 12) | (tiles_x - 1), + pixels: (width - 1) | ((height - 1) << 16), + macro_size: mtile_y1 * utiles_per_tile_y | ((mtile_x1 * utiles_per_tile_x) << 16), + }) +} + +/// Context-relative offset. ALWAYS relative -- a real Metal render pass on this +/// chip writes small offsets into these registers, not addresses. +/// +/// Oracle: the TA command record captured from `mtlg.m` (macos-trace +/// `recs-gfx.txt`, dumped by `recs.d` at `requestChannelCommandTA`) carries an +/// embedded register list that decodes field-for-field against this function's +/// output -- same registers, same order, 12-byte entries of u32 number plus u64 +/// value -- with these values: +/// +/// 0x01748 = 1 (matches) +/// 0x10141 = 0x200 (matches) +/// 0x1c039 = 0x0119ba00 tilemap <- OFFSET, not a VA +/// 0x1c9c8 = 0x0119ba00 tilemap +/// 0x1c0a1 = 0x01290000 tpc <- OFFSET +/// 0x1c031 = 0x800000000119ca00 heapmeta | bit63 (matches the form) +/// 0x1c9c0 = 0x800000000119ca00 heapmeta +/// 0x1c051 = 0x003a0012006b0003 (matches the constant exactly) +/// +/// Writing full virtual addresses here -- which is what the legacy G14X path +/// does, and what an earlier version of this driver was changed to do -- points +/// the tiler's tilemap, TPC and heapmeta at values the firmware cannot use. +fn ta_resource_offset( + parameters: &G17pRenderParameters, + address: u64, +) -> Result { + address + .checked_sub(parameters.context_base) + .ok_or(RenderBuildError::AddressBeforeContext) +} + +/// Compact user/GART address consumed by the TA register and descriptor +/// fields. Native maps these objects in the architectural 0x1000000000 window +/// and publishes the offset, not the full address. Descriptor +0x7d6 is only +/// 32 bits wide, so reject aliases that cannot be represented consistently. +fn ta_user_offset(address: u64) -> Result { + let offset = address + .checked_sub(G17P_USER_BASE) + .ok_or(RenderBuildError::UserAddressOutOfRange)?; + + if offset > u32::MAX as u64 { + return Err(RenderBuildError::UserAddressOutOfRange); + } + + Ok(offset) +} + +fn vdm_user_offset(address: u64) -> Result { + let offset = address + .checked_sub(G17P_USER_BASE) + .ok_or(RenderBuildError::VdmAddressOutOfRange)?; + + if offset >= G17P_VDM_USER_WINDOW_SIZE { + return Err(RenderBuildError::VdmAddressOutOfRange); + } + + Ok(offset) +} + +pub(crate) fn build_ta_registers( + p: &G17pRenderParameters, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], RenderBuildError> { + build_ta_registers_for_work_key(p, render_work_key(1, 0)) +} + +fn build_ta_registers_for_work_key( + p: &G17pRenderParameters, + work_key: u64, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], RenderBuildError> { + let g = geometry(p)?; + let tilemap = ta_resource_offset(p, p.tilemap)?; + let layermeta = ta_resource_offset(p, p.layermeta)?; + let heapmeta = ta_resource_offset(p, p.heapmeta)?; + let tpc = ta_resource_offset(p, p.tpc)?; + let deflake_1 = ta_user_offset(p.deflake_1)?; + let deflake_2 = ta_user_offset(p.deflake_2)?; + let deflake_3 = ta_user_offset(p.deflake_3)?; + // The VDM control stream is the one pointer the 192-byte capture window + // does not reach (register 0x1c880 sits past the end of the dump), and it + // is the one that is not relative to the driver-private `context_base`. + // macOS first lowers the task VA to a task-relative offset, then maps the + // backing at 0x1000000000 + offset in the GART window. Linux userspace gives + // us that full alias VA, so this hardware encoder subtracts the GART base. + // Bit 39 selects the upper UAT root; refuse an offset that would silently + // change roots. Keep the full-address escape hatch diagnostic-only. + let encoder = if p.absolute_pointers { + p.encoder + } else { + vdm_user_offset(p.encoder)? + }; + // generateTARegisterListG16Common: the layer count less one with bit 15, + // plus bit 14 and a process-empty-tiles select for a layered target. The + // pipe mirror 0x1c9e8 keeps only the layered bit and the count. + let mut layer_mode = 0x8000 | (p.layers as u64 - 1); + if p.layers > 1 { + layer_mode |= 0x4000 | if p.process_empty_tiles { 0x2000 } else { 0x1000 }; + } + + Ok([ + RegisterWrite::new(0x01748, 1), + // Native N=1 uses 0x200 and N=2 uses 0x0. The one-layer source profile + // selects the N=1 value, matching descriptor byte +0x789 above. + RegisterWrite::new(0x10141, 0x200), + RegisterWrite::new(0x1c039, tilemap), + RegisterWrite::new(0x1c9c8, tilemap), + RegisterWrite::new(0x1c0a1, tpc), + RegisterWrite::new(0x1c031, heapmeta | 0x8000_0000_0000_0000), + RegisterWrite::new(0x1c9c0, heapmeta | 0x8000_0000_0000_0000), + RegisterWrite::new(0x1c051, 0x003a_0012_006b_0003), + RegisterWrite::new(0x1c061, 1), + RegisterWrite::new(0x10149, p.utile_config), + RegisterWrite::new(0x10139, p.multisample_control), + RegisterWrite::new(0x10111, deflake_1), + RegisterWrite::new(0x1c9b0, deflake_1), + RegisterWrite::new(0x10119, deflake_2), + RegisterWrite::new(0x1c9b8, deflake_2), + RegisterWrite::new(0x1c958, 1), + RegisterWrite::new(0x1c950, deflake_3 | 0x0004_0000_0000_0000), + RegisterWrite::new(0x1c930, 0), + RegisterWrite::new(0x1c880, encoder), + RegisterWrite::new(0x1c079, layermeta), + RegisterWrite::new(0x1c9d8, layermeta), + RegisterWrite::new(0x10151, 0), + RegisterWrite::new(0x1c199, 0), + RegisterWrite::new(0x1c1a1, 0), + RegisterWrite::new(0x1c1a9, 0), + RegisterWrite::new(0x1c1b1, 0), + RegisterWrite::new(0x1c1b9, 0), + RegisterWrite::new(0x1c8f8, 0x8860), + RegisterWrite::new(0x1c0b1, g.size1), + RegisterWrite::new(0x1c850, g.size1), + // Silicon holds 0x10131 at 0x88 at ONE, TWO and FOUR samples while + // 0x10139 carries the real sample position table (all five captures in + // logs/m5-msaa-native-hv-20260915-101837). Driving this from + // multisample_control would write the position table here, which + // Apple never does. + RegisterWrite::new(0x10131, 0x88), + RegisterWrite::new(0x10121, p.ppp_control), + RegisterWrite::new(0x10129, g.pixels), + RegisterWrite::new(0x101b9, g.screen), + RegisterWrite::new(0x1c069, g.x_blocks), + RegisterWrite::new(0x1c071, g.y_blocks), + RegisterWrite::new(0x1c081, g.size2), + RegisterWrite::new(0x1c0a9, g.size3), + RegisterWrite::new(0x10171, 0x100), + RegisterWrite::new(0x10169, layer_mode), + RegisterWrite::new(0x0a309, 0), + RegisterWrite::new(0x1c8e0, u64::MAX), + RegisterWrite::new(0x1c8e8, u64::MAX), + RegisterWrite::new(0x1c898, 0), + RegisterWrite::new(0x101e1, 0x1c), + RegisterWrite::new(0x1c9e8, layer_mode & 0x47ff), + RegisterWrite::new(0x1a099, 0), + RegisterWrite::new(0x1a0a1, 0), + RegisterWrite::new(0x1a069, 0), + RegisterWrite::new(0x1a071, 0), + RegisterWrite::new(0x1a0c9, 0), + RegisterWrite::new(0x1a0d1, 0), + RegisterWrite::new(0x101c9, 0), + RegisterWrite::new(0x0d471, 0), + RegisterWrite::new(0x1a0f1, 8), + RegisterWrite::new(0x10799, 0xff_0000), + // Native sources this from AGXHardwareBufferIDManager. It is neither a + // PBDesc ring index nor a capture-stable literal. + RegisterWrite::new(0x1c830, p.ta_hardware_buffer_id as u64), + RegisterWrite::new(0x1ca30, p.cycle), + RegisterWrite::new(0x16c39, p.cycle), + RegisterWrite::new(0x1c910, p.record_index), + // Native derives the high word from the TA AGXUMADataRec PTC/DUPM + // maximum at host descriptor +0x1488, copied through command +0x1158. + // halUpdatePTCMaxCount computes max(0x60, min(pool_input / active_kicks, + // 0x200)); Linux does not yet carry that pool input. Follow the kext's + // exact zero-source fallback 0x60_0040_0020. Bit 1 retains one working + // Metal trace's workload-derived 0xfe value as a diagnostic only. + RegisterWrite::new( + 0x0a5a1, + if p.native_ta_registers & 2 != 0 { + 0x0000_00fe_0040_0020 + } else { + 0x0000_0060_0040_0020 + }, + ), + RegisterWrite::new(0x0d419, 0x0000_0002_0000_0001), + RegisterWrite::new(0x1ca10, p.lifecycle), + RegisterWrite::new(0x014a1, p.lifecycle), + RegisterWrite::new(0x0a349, p.lifecycle), + RegisterWrite::new(0x10209, work_key), + RegisterWrite::new(0x1c9f0, work_key), + RegisterWrite::new(0x14320, work_key), + // AGXTACommandDescriptor::prepare receives the shared USC FList ID + // through descriptor +0x1410. Optional GART/feature payloads are absent + // in Linux, so the retained owner's ID is the complete value. + RegisterWrite::new(0x14308, p.usc_flist_hardware_buffer_id as u64), + RegisterWrite::new(0x14318, p.ta_status | 1), + RegisterWrite::new(0x01740, 1), + RegisterWrite::new(0x1c880, deflake_1), + RegisterWrite::new(0x1c898, 1), + ]) +} + +/// DIAGNOSTIC. Write native's value into TA-descriptor fields where ours differs +/// structurally at MATCHED 256x256 geometry (recs3.txt TA record, stride 0x9c0). +/// The tiler bins ~60 bytes of a 1.28 MB TVB with every input verified correct, +/// so these are what is left that we demonstrably send differently. +fn apply_g17p_ta_oracle(raw: &mut [u8]) { + let mask = crate::g18::fw::config::TA_ORACLE; + if mask == 0 { + return; + } + if mask & 0x01 != 0 { + put_u64(raw, 0x370, 0x500); + put_u64(raw, 0x388, 0x500); + put_u64(raw, 0x8d0, 0x500); + } + if mask & 0x02 != 0 { + put_u64(raw, 0x018, 3); + } + if mask & 0x04 != 0 { + put_u32(raw, 0x33c, 0xa9); + } + if mask & 0x08 != 0 { + put_u64(raw, 0x7d8, 0x6); + put_u32(raw, 0x798, 0xa); + } +} + +pub(crate) fn build_fragment_registers( + p: &G17pRenderParameters, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], RenderBuildError> { + build_fragment_registers_for_work_key(p, render_work_key(1, 1)) +} + +/// As `build_fragment_registers`, plus the live entry count. The count is +/// always `FRAGMENT_REGISTER_COUNT`; the pair form is kept because the +/// descriptor's count word at +0x7a8 must agree with the slice actually +/// written, and having one source for both makes them impossible to desync. +pub(crate) fn build_fragment_registers_counted( + p: &G17pRenderParameters, +) -> Result<([RegisterWrite; FRAGMENT_REGISTER_COUNT], usize), RenderBuildError> { + Ok(( + build_fragment_registers_for_work_key(p, render_work_key(1, 1))?, + FRAGMENT_REGISTER_COUNT, + )) +} + +fn build_fragment_registers_for_work_key( + p: &G17pRenderParameters, + work_key: u64, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], RenderBuildError> { + let g = geometry(p)?; + let tile_state = 0x3717f + | ((g.tiles_x - 1) << 44) + | ((g.tiles_y - 1) << 53) + | 0x20_0000_0000 + | (if p.layers > 1 { 0x1_0000_0000 } else { 0 }) + | ((p.utile_config & 0xf000) << 28); + + Ok([ + RegisterWrite::new(0x01739, 1), + RegisterWrite::new(0x10009, p.utile_config), + RegisterWrite::new(0x15379, p.store_pipeline_bind), + RegisterWrite::new(0x15381, p.store_pipeline), + RegisterWrite::new(0x15369, p.load_pipeline_bind), + RegisterWrite::new(0x15371, p.load_pipeline), + RegisterWrite::new(0x15131, p.merge_upper_x_bits as u64), + RegisterWrite::new(0x15139, p.merge_upper_y_bits as u64), + RegisterWrite::new(0x100a1, 0), + RegisterWrite::new(0x15069, 0), + RegisterWrite::new(0x15071, 0), + RegisterWrite::new(0x16058, 0), + RegisterWrite::new(0x10019, p.multisample_control), + RegisterWrite::new(0x100b1, g.macro_size), + RegisterWrite::new(0x16030, g.macro_size), + RegisterWrite::new(0x100d9, g.screen), + RegisterWrite::new(0x0a301, 0), + RegisterWrite::new(0x10791, 0xff_0200), + RegisterWrite::new(0x16098, p.heapmeta), + RegisterWrite::new(0x15109, p.scissor_array), + RegisterWrite::new(0x15101, p.depth_bias_array), + RegisterWrite::new(0x15021, p.aux_fb_flags), + RegisterWrite::new(0x15211, ((p.height as u64) << 32) | p.width as u64), + RegisterWrite::new(0x15049, p.aux_fb_page_count), + RegisterWrite::new(0x10051, p.tib_blocks), + RegisterWrite::new(0x15321, p.depth_dimensions), + RegisterWrite::new(0x15301, p.depth_clear_value_bits as u64), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15311, p.occlusion_query_base), + RegisterWrite::new(0x15319, p.depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + RegisterWrite::new(0x15351, 0), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x15231, 0), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15239, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15401, p.depth_stride), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15409, p.stencil_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15411, p.depth_aux_stride), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15419, p.stencil_aux_stride), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x16429, p.tilemap), + RegisterWrite::new(0x16060, p.layermeta), + RegisterWrite::new(0x16431, (4 * g.size1) << 24), + RegisterWrite::new(0x10039, p.tile_config), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16451, 0), + RegisterWrite::new(0x15359, 0), + RegisterWrite::new(0x100b8, 0x8860), + RegisterWrite::new(0x16461, p.aux_fb), + RegisterWrite::new(0x16090, p.aux_fb), + RegisterWrite::new(0x101e9, 0x1c), + RegisterWrite::new(0x160a8, 0), + RegisterWrite::new(0x16068, tile_state), + RegisterWrite::new(0x1a0a9, 0), + RegisterWrite::new(0x1a0b1, 0), + RegisterWrite::new(0x1a079, 0), + RegisterWrite::new(0x1a081, 0), + RegisterWrite::new(0x1a0d9, 0), + RegisterWrite::new(0x1a0e1, 0), + RegisterWrite::new(0x101c1, 0), + RegisterWrite::new(0x0d469, 0), + RegisterWrite::new(0x1a0f9, 8), + // Fragment analog of TA 0x0a5a1. Native derives its high word from the + // 3D AGXUMADataRec PTC/DUPM maximum at host descriptor +0xc00, copied + // through command +0x818. Linux has no owned pool-size input yet, so + // use the exact 0x60 zero-source fallback. Bit 3 retains one captured, + // workload-derived 0xd5 value only as a diagnostic. + RegisterWrite::new( + 0x0a5a9, + if p.native_ta_registers & 8 != 0 { + 0x0000_00d5_0040_0020 + } else { + 0x0000_0060_0040_0020 + }, + ), + RegisterWrite::new(0x0d429, 0x0000_0002_0000_0001), + RegisterWrite::new(0x160e0, p.fragment_lifecycle), + RegisterWrite::new(0x01499, p.fragment_lifecycle), + RegisterWrite::new(0x0a341, p.fragment_lifecycle), + // The fragment must use the same parameter-buffer owner as TA 1c830. + // Native TA::prepare stores its allocated HWPB ID at host +928; the + // same derived host descriptor is submitted through both channels. + // generate3DRegisterList reads that HOST field at 8a5c7cc. Its wrapper + // at 8a515d8 tail-calls with x1=raw command, x2=host descriptor: the + // unused channel argument in the mangled common-function name is NOT + // present in the optimized calling convention. Raw command +928 has + // a different producer and must not be used to infer this register. + RegisterWrite::new(0x1c838, p.ta_hardware_buffer_id as u64), + RegisterWrite::new(0x1ca28, p.cycle), + RegisterWrite::new(0x10211, work_key), + RegisterWrite::new(0x10420, work_key), + // AGX3DCommandDescriptor::prepare receives this ID from the USC pool's + // HardwareBufferIDManager and generate3DRegisterList ORs it with an + // optional GART payload and feature bit. Linux currently has neither + // optional source, so publish the retained USC FList owner's exact ID. + RegisterWrite::new(0x14048, p.usc_flist_hardware_buffer_id as u64), + RegisterWrite::new(0x14080, p.fragment_status | 1), + RegisterWrite::new(0x01731, 1), + RegisterWrite::new(0x16020, 1), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16068, 0x4_0000), + ]) +} + +/// Size of one userspace global-ID group reserved by +/// `IOGPUDeviceUserClient::s_get_next_gid_group` on macOS. +pub(crate) const G17P_RENDER_GID_GROUP_STRIDE: u32 = 0x0100_0000; + +/// Compose the predecessor/current IDs used by one native render pair. +/// +/// macOS reserves a 0x01000000-ID range and gives its old global-counter value +/// to Metal. Metal returns that predecessor in render setup; the kernel then +/// allocates consecutive current IDs from the advanced global counter, 3D +/// first and TA second. The RCE programs and command tails both carry the +/// resulting `{predecessor,current}` pair. These are object IDs, not queue +/// submission ordinals. +pub(crate) const fn g17p_render_lifecycle_pair( + predecessor: u32, + fragment_current: u32, +) -> Option<(u64, u64)> { + if predecessor == 0 || fragment_current == 0 { + return None; + } + let tiling_current = match fragment_current.checked_add(1) { + Some(value) if value != 0 => value, + _ => return None, + }; + let predecessor = (predecessor as u64) << 32; + Some(( + predecessor | fragment_current as u64, + predecessor | tiling_current as u64, + )) +} + +pub(crate) fn partial_store_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_STORE_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15379, p.partial_store_pipeline_bind), + RegisterWrite::new(0x15381, p.partial_store_pipeline), + RegisterWrite::new(0x10039, p.tile_config), + RegisterWrite::new(0x15359, 0x20), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +pub(crate) fn partial_resume_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_RESUME_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15379, p.partial_store_pipeline_bind), + RegisterWrite::new(0x15381, p.partial_store_pipeline), + RegisterWrite::new(0x15369, p.partial_load_pipeline_bind), + RegisterWrite::new(0x15371, p.partial_load_pipeline), + RegisterWrite::new(0x10039, p.tile_config & 0xffff), + RegisterWrite::new(0x15359, 0x20), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +pub(crate) fn partial_load_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_LOAD_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15369, p.partial_load_pipeline_bind), + RegisterWrite::new(0x15371, p.partial_load_pipeline), + RegisterWrite::new(0x10039, p.tile_config & 0xffff), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +fn put_u8(raw: &mut [u8], offset: usize, value: u8) { + raw[offset] = value; +} + +#[cfg(test)] +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +// v91 WORD_STORES: aligned fields use one store (see g17_store). +#[cfg(not(test))] +#[inline(always)] +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + crate::g18::fw::store::put_u16(raw, offset, value) +} + +#[cfg(test)] +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +// v91 WORD_STORES: aligned fields use one store (see g17_store). +#[cfg(not(test))] +#[inline(always)] +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + crate::g18::fw::store::put_u32(raw, offset, value) +} + +#[cfg(test)] +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +// v91 WORD_STORES: aligned fields use one store (see g17_store). +#[cfg(not(test))] +#[inline(always)] +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + crate::g18::fw::store::put_u64(raw, offset, value) +} + +fn encode_registers(raw: &mut [u8], offset: usize, registers: &[RegisterWrite]) { + for (index, register) in registers.iter().enumerate() { + let entry = offset + index * REGISTER_BYTES; + put_u32(raw, entry, register.number); + put_u64(raw, entry + 4, register.value); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RenderDescriptorKind { + Tiling, + Fragment, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderDescriptorObjects { + pub(crate) record_a: u64, + pub(crate) shared: u64, + pub(crate) record_b: u64, + pub(crate) zero: u64, +} + +impl RenderDescriptorObjects { + /// Select one 0x100-byte Pool-A record while leaving every other object + /// pointer unchanged. + pub(crate) fn with_pool_a_record_index( + mut self, + record_index: usize, + ) -> Result { + let offset = record_index + .checked_mul(0x100) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + self.record_a = self + .record_a + .checked_add(offset as u64) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + Ok(self) + } + + /// Replace the stage-local shared object without changing the other three + /// descriptor pointers. Final native publishes keep TA +0x20 and Fragment + /// +0x28 identical; callers must therefore apply any HWPB substitution to + /// the shared object used by both descriptors, not to Fragment alone. + pub(crate) const fn with_shared(mut self, shared: u64) -> Self { + self.shared = shared; + self + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderDescriptorMetadata { + pub(crate) context_id: u32, + pub(crate) submission_ordinal: u64, + /// Monotonic state tag allocated by the command buffer's 36-entry state + /// ring. This is neither the selected ring index nor any QID/HWID. Native + /// retains the full word in the selected state object and places only its + /// low six bits in the stage work key. + pub(crate) command_buffer_state_word: u32, + /// TA HardwareBufferID copied to firmware descriptor +0x18. + /// + /// Every working native descriptor corpus couples this field to register + /// 0x1c830 exactly (for example 2/2, 4/4, and 5/5). The same allocator + /// result is also PBState +0x0c and the descriptor-ring index consumed by + /// `halLoadPBDesc`. + pub(crate) ta_hardware_buffer_id: u32, + pub(crate) submit_sequence: u64, +} + +impl RenderDescriptorMetadata { + pub(crate) const fn source(kind: RenderDescriptorKind) -> Self { + Self { + context_id: 1, + submission_ordinal: 0, + command_buffer_state_word: match kind { + RenderDescriptorKind::Tiling => 0, + RenderDescriptorKind::Fragment => 1, + }, + ta_hardware_buffer_id: 0, + submit_sequence: match kind { + RenderDescriptorKind::Tiling => 1, + RenderDescriptorKind::Fragment => 0, + }, + } + } +} + +/// Native's TA/3D register-list producer couples the command-buffer state tag +/// to the descriptor's allocated application-GART context. The physical QID, +/// QoS HardwareBufferID, descriptor-array slot and 36-entry state-ring index +/// are independent namespaces and must not be substituted here. +const fn render_work_key(context_id: u32, command_buffer_state_word: u32) -> u64 { + ((context_id as u64) << 8) | ((command_buffer_state_word & 0x3f) as u64) +} + +/// The completion record must name the same generation as the rendered writes. +/// Native nextCommandBufferState stores the allocated word at EventControl +8; +/// generate3DRegisterList uses its low six bits in 0x10211/0x10420. B1 uses those +/// same bits in the cache-maintenance request before publishing the stamp. +/// Keep the full word here: physical slot ordinals and ring indices are not +/// substitutes for the command-buffer allocator's generation. +pub(crate) fn set_render_event_control_generation( + event_control: &mut [u8], + command_buffer_state_word: u32, +) -> Result<(), RenderBuildError> { + let generation = event_control + .get_mut(8..12) + .ok_or(RenderBuildError::BufferTooSmall)?; + generation.copy_from_slice(&command_buffer_state_word.to_le_bytes()); + Ok(()) +} + +fn descriptor_work_ordinal(ordinal: u64) -> u32 { + (ordinal as u32).wrapping_add((ordinal / 2) as u32) +} + +fn write_common_descriptor( + kind: RenderDescriptorKind, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + registers: &[RegisterWrite], + raw: &mut [u8], +) { + let (selector, pointers, register_offset, pointer_gap) = match kind { + RenderDescriptorKind::Tiling => (0u32, 0x10usize, 0x60usize, 8usize), + RenderDescriptorKind::Fragment => (1u32, 0x20usize, 0xa0usize, 0usize), + }; + put_u32(raw, 0, selector); + put_u64(raw, 4, metadata.submit_sequence); + put_u32(raw, 0x0c, metadata.context_id); + + let addresses = [ + objects.record_a, + objects.shared, + objects.record_b, + objects.zero, + ]; + let mut offset = pointers; + for (index, address) in addresses.iter().enumerate() { + put_u64(raw, offset, *address); + offset += 8; + if index == 0 { + offset += pointer_gap; + } + } + encode_registers(raw, register_offset, registers); + + let work = descriptor_work_ordinal(metadata.submission_ordinal); + match kind { + RenderDescriptorKind::Tiling => { + put_u32(raw, 0x18, metadata.ta_hardware_buffer_id); + put_u32(raw, 0x48, work); + } + RenderDescriptorKind::Fragment => {} + } +} + +pub(crate) fn build_ta_descriptor( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < TA_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + let registers = build_ta_registers_for_work_key( + parameters, + render_work_key(metadata.context_id, metadata.command_buffer_state_word), + )?; + build_ta_descriptor_base(parameters, objects, metadata, ®isters, raw)?; + apply_g17p_ta_oracle(raw); + Ok(()) +} + +pub(crate) fn build_ta_descriptor_base( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + registers: &[RegisterWrite], + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < TA_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + if registers.is_empty() || registers.len() > (0x660 - 0x60) / REGISTER_BYTES { + return Err(RenderBuildError::BufferTooSmall); + } + raw[..TA_DESCRIPTOR_SIZE].fill(0); + write_common_descriptor( + RenderDescriptorKind::Tiling, + objects, + metadata, + registers, + raw, + ); + + // Like the fragment boundaries, these are generated for every native TA + // command (25G83 88bdfc4/88be090/88be1a4), not only a queue's first kick. + // The final three entries are 01740, 1c880, 1c898; firmware reads +38 and + // +3a/+3c to select the corresponding embedded RCE boundaries. + put_u16(raw, 0x38, (TA_REGISTER_COUNT - 2) as u16); + put_u16(raw, 0x3a, TA_REGISTER_COUNT as u16); + put_u16(raw, 0x3c, TA_REGISTER_COUNT as u16); + + let deflake = ta_user_offset(parameters.deflake_1)?; + let sampler_max = if parameters.sampler_count == 0 { + 0 + } else { + parameters.sampler_count + 1 + }; + put_u32( + raw, + 0x0768, + (((registers.len() * REGISTER_BYTES) as u32) << 16) | registers.len() as u32, + ); + put_u64(raw, 0x0780, parameters.tpc); + // Native copies RTMemory's geometry-derived TPC size beside its pointer + // on every submission (25G83 88fd970..974), independent of queue lifetime. + put_u64(raw, 0x0788, parameters.tpc_size); + put_u32(raw, 0x07d6, deflake as u32); + put_u32(raw, 0x0876, u32::MAX); + put_u64(raw, 0x087a, parameters.sampler_array); + put_u32(raw, 0x0882, parameters.sampler_count); + put_u32(raw, 0x0886, sampler_max); + // Present in 436/438 valid G17P TA records, including both exact 32x32 + // geometry matches. This belongs to the TA output/activation tail; leaving + // it zero is a structural mismatch before the linked 3D job is generated. + put_u32(raw, 0x0892, 1); + put_u8(raw, 0x0932, 0x44); + put_u8(raw, 0x093c, 1); + put_u8(raw, 0x094d, parameters.context_generation); + put_u64(raw, 0x08fe, parameters.ta_timestamp_start); + put_u64(raw, 0x0906, parameters.ta_timestamp_end); + // +0x090e/+0x0916 are caller-visible timestamp destinations. Firmware + // writes them only inside its timestamp aperture, so they stay zero; the + // completion publishes the pass envelope to userspace instead. + // AGXTAChannelSKSM::submitBuffer mirrors the same dependency/current pair + // consumed by the TA RCE program into the firmware command. These fields + // are deliberately unaligned in the native 0x9c0-byte command layout. + put_u32(raw, 0x086e, (parameters.lifecycle >> 32) as u32); + put_u64(raw, 0x08ce, parameters.lifecycle as u32 as u64); + Ok(()) +} + +/// Build the TA descriptor used by the proven native cold opening. +/// +/// The first-work path applies its pool/lifecycle fields after the common +/// builder has generated the RCE boundaries required by every command. +pub(crate) fn build_cold_opening_ta_descriptor( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + context_id: u32, + command_buffer_state_word: u32, + ta_hardware_buffer_id: u32, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + let mut metadata = RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling); + metadata.context_id = context_id; + metadata.command_buffer_state_word = command_buffer_state_word; + metadata.ta_hardware_buffer_id = ta_hardware_buffer_id; + build_ta_descriptor(parameters, objects, metadata, raw)?; + put_u8(raw, 0x93e, 0xd0); + put_u8( + raw, + 0x93f, + if parameters.native_pm_bytes { + 0x87 + } else { + 0x91 + }, + ); + Ok(()) +} + +fn put_embedded_registers( + raw: &mut [u8], + header_offset: usize, + program_offset: usize, + registers: &[RegisterWrite], +) { + let bytes = registers.len() * REGISTER_BYTES; + put_u32( + raw, + header_offset, + ((bytes as u32) << 16) | registers.len() as u32, + ); + encode_registers(raw, program_offset, registers); +} + +pub(crate) struct G17pFragmentRegisters<'a> { + pub(crate) main: &'a [RegisterWrite], + pub(crate) store: &'a [RegisterWrite], + pub(crate) resume: &'a [RegisterWrite], + pub(crate) load: &'a [RegisterWrite], +} + +pub(crate) fn build_fragment_descriptor( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < FRAGMENT_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + let registers = build_fragment_registers_for_work_key( + parameters, + render_work_key(metadata.context_id, metadata.command_buffer_state_word), + )?; + let store = partial_store_registers(parameters); + let resume = partial_resume_registers(parameters); + let load = partial_load_registers(parameters); + build_fragment_descriptor_base( + parameters, + objects, + metadata, + G17pFragmentRegisters { + main: ®isters, + store: &store, + resume: &resume, + load: &load, + }, + raw, + )?; + apply_g17p_fragment_oracle(raw); + Ok(()) +} + +pub(crate) fn build_fragment_descriptor_base( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + programs: G17pFragmentRegisters<'_>, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < FRAGMENT_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + if programs.main.is_empty() + || programs.main.len() > (0x6a0 - 0xa0) / REGISTER_BYTES + || programs.store.len() > (0xec0 - 0x7c0) / REGISTER_BYTES + || programs.resume.len() > (0x15e0 - 0xee0) / REGISTER_BYTES + || programs.load.len() > (0x1d00 - 0x1600) / REGISTER_BYTES + { + return Err(RenderBuildError::BufferTooSmall); + } + raw[..FRAGMENT_DESCRIPTOR_SIZE].fill(0); + let geometry = geometry(parameters)?; + let register_count = programs.main.len(); + write_common_descriptor( + RenderDescriptorKind::Fragment, + objects, + metadata, + programs.main, + raw, + ); + + put_u64(raw, 0x40, parameters.tilemap); + put_u64(raw, 0x48, parameters.multisample_control); + // Native copies the render target's sample count through host +0x8c8 + // into every fragment descriptor at +0x50 (25G83: 0x8839d80, + // 0x8837af8..0x8837b08). It is not first-work metadata. The UAPI has + // already encoded log2(samples) in UTILE_CONFIG bits [1:0]. + put_u32(raw, 0x50, 1u32 << (parameters.utile_config & 3)); + put_u32(raw, 0x54, geometry.macro_size as u32); + put_u32(raw, 0x68, parameters.merge_upper_x_bits); + put_u32(raw, 0x6c, parameters.merge_upper_y_bits); + put_u64(raw, 0x78, geometry.tiles_x * geometry.tiles_y); + + // RCE boundaries belong to every command, including retained queue kicks. + // Native emits these from the running register count (25G83 88c0740, + // 88c07ac, 88c08c8, 88c0970); firmware consumes +80/+82/+84/+88 when + // selecting an entry in this command's embedded program. The fixed recipe + // ends with 01731=1, 16020=1, 16020=0, 16068=40000. Boundaries are after + // 01731, after the first 16020, that same entry, and the complete program. + // Keeping these only in the cold wrapper left ordinary restages at zero. + put_u16(raw, 0x80, (register_count - 3) as u16); + put_u16(raw, 0x82, (register_count - 2) as u16); + put_u16(raw, 0x84, (register_count - 2) as u16); + put_u16(raw, 0x88, register_count as u16); + + put_u32( + raw, + 0x07a8, + (((register_count * REGISTER_BYTES) as u32) << 16) | register_count as u32, + ); + put_embedded_registers(raw, 0x0ec8, 0x07c0, programs.store); + put_embedded_registers(raw, 0x15e8, 0x0ee0, programs.resume); + put_embedded_registers(raw, 0x1d08, 0x1600, programs.load); + + put_u64(raw, 0x1d20, parameters.depth_bias_array); + put_u64(raw, 0x1d30, parameters.scissor_array); + put_u64(raw, 0x1d40, parameters.occlusion_query_base); + put_u64(raw, 0x1e78, parameters.load_pipeline_bind); + put_u64(raw, 0x1e80, parameters.load_pipeline); + put_u64(raw, 0x1ea8, parameters.partial_load_pipeline_bind); + put_u64(raw, 0x1eb0, parameters.partial_load_pipeline); + put_u32(raw, 0x1ec0, 0x0404_0404); + put_u64(raw, 0x1f38, parameters.tib_blocks); + put_u64(raw, 0x1f40, parameters.aux_fb_flags); + put_u32(raw, 0x1f48, parameters.width); + put_u32(raw, 0x1f4c, parameters.height); + put_u64(raw, 0x1f50, parameters.aux_fb_page_count); + put_u64(raw, 0x1f58, parameters.tile_config); + put_u32(raw, 0x1f78, parameters.store_pipeline_bind as u32); + put_u64(raw, 0x1f7c, parameters.store_pipeline); + put_u32(raw, 0x1f98, parameters.partial_store_pipeline_bind as u32); + put_u64(raw, 0x1f9c, parameters.partial_store_pipeline); + put_u32(raw, 0x1fa8, parameters.depth_clear_value_bits); + put_u64(raw, 0x1fac, (parameters.tib_blocks << 33) | 0x300); + // Native copies isMemorylessRender through the UMD render record +0x240, + // parsed record +0x1b8, and host descriptor +0x8bf into this byte. It is + // not a TIB-size predicate: native's memoryless policy also selects the + // shortened SPM phase programs. The admitted render UAPI represents + // memory-backed resources and has no memoryless-render policy or matching + // SPM-program construction. Keep that policy clear even for >8 TIB + // blocks; supporting memoryless renders requires modeling both together. + // Empty-tile processing remains independent in register 0x10039 bit 16. + put_u8(raw, 0x2100, 0); + // Exact G17P tail copied by AGX3DChannelSKSM::submitBuffer. +0x2104 is the + // accelerator control snapshot and +0x210c its low map nibble. +0x2110 is + // a separate render-setup field: prepare copies descriptor +0xc00 to setup + // +0x818, processRenderSetup carries it to +0x89c, and submitBuffer emits + // it here. Working Metal commands carry the invariant all-ones value. + put_u32(raw, 0x2104, parameters.gpc_perf_state_control); + // AGX3DChannelSKSM::submitBuffer mirrors the same dependency/current pair + // consumed by the fragment RCE program into the firmware command. + put_u32(raw, 0x2108, (parameters.fragment_lifecycle >> 32) as u32); + put_u32(raw, 0x210c, parameters.gpc_perf_state_map_low); + put_u32(raw, 0x2110, G17P_FRAGMENT_TAIL_SETUP_WORD); + // Native submitBuffer zeros +0x2114..+0x2123. Sampler state reaches the + // GPU through its real register/program path, not these command-tail + // lanes. The first two following flag lanes reproduce the working Neo + // shared-runtime snapshot; the final two come from per-command descriptor + // bytes and retain their constructor-zero values until those inputs are + // modeled. + put_u32( + raw, + 0x2124, + G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT[0], + ); + put_u32( + raw, + 0x2128, + G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT[1], + ); + put_u64(raw, 0x2168, parameters.fragment_lifecycle as u32 as u64); + put_u64(raw, 0x2198, parameters.fragment_timestamp_start); + put_u64(raw, 0x21a0, parameters.fragment_timestamp_end); + // +0x21a8/+0x21b0: caller-visible destinations, zero as in the TA record. + put_u8(raw, 0x21cc, 0x53); + put_u8(raw, 0x21d6, 1); + put_u8(raw, 0x21e7, parameters.context_generation); + put_u8(raw, 0x2208, 1); + put_u8(raw, 0x220c, 1); + put_u8(raw, 0x222d, g17p_frg_hwdsid_mode()); + Ok(()) +} + +fn apply_g17p_fragment_oracle(raw: &mut [u8]) { + // Neo diagnostic overrides stay outside the shared encoder so M5 + // register programs and geometry never inherit these substitutions. + let oracle = crate::g18::fw::config::FRAG_ORACLE; + if oracle & 0x01 != 0 { + put_u64(raw, 0x158, 0x7007); + } + if oracle & 0x02 != 0 { + put_u64(raw, 0x488, 1); + } + if oracle & 0x04 != 0 { + // native 0x8000005300000000 against our 0x0000005300000000 + let mut cur = [0u8; 8]; + cur.copy_from_slice(&raw[0x21c8..0x21d0]); + put_u64(raw, 0x21c8, u64::from_le_bytes(cur) | (1u64 << 63)); + } + if oracle & 0x08 != 0 { + put_u64(raw, 0x78, 0x40); + } + if oracle & 0x10 != 0 { + put_u64(raw, 0x458, 3); + put_u64(raw, 0x470, 0x500); + } + if oracle & 0x20 != 0 { + put_u64(raw, 0x1f48, 0x0000_0100_0000_0100); + } + if oracle & 0x40 != 0 { + // fields native leaves ZERO and we populate + put_u64(raw, 0x1d0, 0); + put_u64(raw, 0x1f0, 0); + put_u64(raw, 0x1f8, 0); + } + if oracle & 0x80 != 0 { + put_u32(raw, 0x1d8 + 4, 0x3f80_0000); + } +} + +/// FRG +0x222d mode byte: Linux has always written 1; all 160 native light +/// FRG captures hold 0. Diagnostic parameter `g17p_frg_hwdsid_clear`. +#[cfg(not(test))] +fn g17p_frg_hwdsid_mode() -> u8 { + u8::from(*crate::module_parameters::g17p_frg_hwdsid_clear.value() == 0 + && !crate::g18::compat::timing_buffer::diag_knob(crate::g18::compat::timing_buffer::DIAG_FRG_222D_CLEAR)) +} + +#[cfg(test)] +fn g17p_frg_hwdsid_mode() -> u8 { 1 } + +/// Build the fragment descriptor used by the proven native cold opening. +pub(crate) fn build_cold_opening_fragment_descriptor( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + context_id: u32, + command_buffer_state_word: u32, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + let mut metadata = RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment); + metadata.context_id = context_id; + metadata.command_buffer_state_word = command_buffer_state_word; + build_fragment_descriptor(parameters, objects, metadata, raw)?; + // +90 is staged with the channel's completion payload at publication. + // A newly published native 3D queue carries 1 here; retained submissions + // clear it. This lifecycle is independent of the outer-slot first-submit + // byte and appears identically on cold QIDs 1 and 8 in both valid traces. + put_u32(raw, 0x215c, 1); + Ok(()) +} + +/// Stage the TA's link to the physical fragment command, not its ring slot. +/// +/// Native TA submit copies host +0x1528 as a low-40-bit qword at +0x7a8 and +/// its bits 47:40 as two independent bytes at +0x7b0/+0x7b1 (8a99220..34). +/// Host +0xe60 supplies the full packed qword at +0x8c6 (8a99414..1c). +/// Its producer converts the TA channel timestamp, whereas +0x1528 converts +/// the fragment channel timestamp (8aa9e30..aaa05c). Both carry the 3D channel +/// payload. Preserve those distinct sources even when queue seeds coincide. +/// Refresh from the actual retained producer immediately before publication; +/// the payload counter wraps independently of the physical timestamp. +pub(crate) fn apply_g17p_ta_linked_completion_scratch( + tiling_timestamp: u64, + fragment_timestamp: u64, + payload: u8, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if tiling_timestamp > 0xff_ffff_ffff || fragment_timestamp > 0xff_ffff_ffff { + return Err(RenderBuildError::ArithmeticOverflow); + } + if raw.len() < TA_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + put_u64(raw, 0x07a8, fragment_timestamp); + put_u8(raw, 0x07b0, payload); + put_u8(raw, 0x07b1, payload); + put_u64(raw, 0x08c6, tiling_timestamp | ((payload as u64) << 40)); + Ok(()) +} + +/// Apply the queue-local fields that change as the canonical three-item group +/// is refreshed in place. `descriptor_gpu_va` is the low VM address firmware +/// uses for the descriptor's self pointers. +pub(crate) fn apply_g17p_retained_descriptor_fields( + kind: RenderDescriptorKind, + descriptor_gpu_va: u64, + runtime_control: u64, + queue_id: u8, + linked_fragment_queue_id: u8, + submission_ordinal: u64, + cold_opening: bool, + native_pm_bytes: bool, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + let item_number = submission_ordinal + .checked_add(1) + .ok_or(RenderBuildError::ArithmeticOverflow)? & 0xffffffffff; + // The native queue timestamp is 40 bits. Stamp-info carries its low + // 24 bits shifted by eight, with ordinary u32 wrap as on M1/M2. + let event_value = (item_number as u32).wrapping_shl(8); + match kind { + RenderDescriptorKind::Tiling => { + if raw.len() < TA_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + put_u64( + raw, + 0x0760, + descriptor_gpu_va + .checked_add(0x60) + .ok_or(RenderBuildError::ArithmeticOverflow)?, + ); + // Native TA links directly to the physical 3D kick queue which + // will consume this submission. This is queue allocator state, + // not the type-1 fragment descriptor's distinct QID-0 RCE field. + put_u32(raw, 0x079c, u32::from(linked_fragment_queue_id)); + put_u32(raw, 0x07a0, event_value); + apply_g17p_ta_linked_completion_scratch( + u64::from(item_number), u64::from(item_number), item_number as u8, raw, + )?; + // Native writes a u32 at +0x8b6 (8a993f4). +0x8b4/+0x8b5 + // belong to the preceding stamp pointer, not this event value. + put_u32(raw, 0x08b6, event_value); + // B1 indexes the registered scheduler-resource table with this + // value while admitting the completed TA, then passes the full + // word to FUN_15424(linked_3d_qid, ta_qid). It is the TA queue's + // own physical QID, not a captured literal: two independent Metal + // commands carry 7 here while +0x79c links QID 8, and 9 here while + // +0x79c links QID 10. Leaving it zero made Linux consult the + // unrelated QID-0 scheduler state at the TA-to-3D handoff. + put_u32( + raw, + 0x08ba, + if crate::g18::fw::config::RENDER_TA_QID_FIELD != 0 { + u32::from(queue_id) + } else { + 0 + }, + ); + // Retain the existing Linux TA drain policy while its visibility + // boundaries are audited. Native +0x8c2 is a pending IO-fence + // boolean (host descriptor +0x48), not a priority/QID encoding. + // Firmware ORs it into TSQ node +0x20 and clears it on a drain; + // this TA flag does not guarantee a later fragment drain. + // Keep this independent of the opt-in priority diagnostic. + put_u32(raw, 0x08c2, 1); + // Native's host +0x1288 source is a u32 at +0x8d6 (8a990b8), + // immediately after the qword lifecycle field beginning +0x8ce. + put_u32(raw, 0x08d6, submission_ordinal as u32); + put_u64(raw, 0x0934, runtime_control); + if cold_opening { + put_u8(raw, 0x093e, 0xd0); + put_u8(raw, 0x093f, if native_pm_bytes { 0x87 } else { 0x91 }); + } + } + RenderDescriptorKind::Fragment => { + if raw.len() < FRAGMENT_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + for (at, target) in [ + (0x07a0, 0x00a0), + (0x0ec0, 0x07c0), + (0x15e0, 0x0ee0), + (0x1d00, 0x1600), + ] { + put_u64( + raw, + at, + descriptor_gpu_va + .checked_add(target) + .ok_or(RenderBuildError::ArithmeticOverflow)?, + ); + } + put_u32(raw, 0x2150, event_value); + // Valid native queues prove +0x2154 is the physical 3D QID: it is + // 1/3/8 for qids 1/3/8, respectively. The old literal only happened + // to match the first captured qid. + put_u32(raw, 0x2154, u32::from(queue_id)); + // Preserve the existing first-submission policy pending a + // visibility-boundary audit. The captured cold/later values do + // not establish ordinal semantics: native derives this flag + // from pending IO-fences at host descriptor +0x48 (8837268..74, + // 8837eb0). The TSQ ownership notes decode the producer. + put_u32(raw, 0x215c, u32::from(submission_ordinal == 0)); + // +0x2160 is the packed 3D-channel SKSM completion scratch value: + // the physical queue timestamp in bits [39:0] and the channel's + // payload byte in bits [47:40]. G17PUserRenderStorage stages it + // from the retained fragment producer immediately before publish. + put_u32(raw, 0x2170, submission_ordinal as u32); + put_u64(raw, 0x21ce, runtime_control); + // Native HWDSID mask occupies +0x220d..+0x222c inclusive; + // +0x222d is a separate mode flag. An unaligned u64 at +0x222c + // reads 0x100 from that following flag, not from HWDSID bit 8. + // Preserve the template's zero mask; aperture IDs alone do not + // supply the explicit MDSID-list and accounting contract. + } + } + Ok(()) +} + +/// Copy the four embedded 3D register-entry arrays into a diagnostic mirror. +/// +/// The hardware-visible arrays remain descriptor-local. Native's addresses at +/// descriptor +0x7a0/+0xec0/+0x15e0/+0x1d00 are the self pointers in each +/// embedded array's +0x700 trailer, not pointers to a second allocation. This +/// mirror intentionally contains entries only and must never be bound to KSM. +pub(crate) fn stage_g17p_fragment_rce_programs( + fragment: &mut [u8], + rce_gpu_va: u64, + rce: &mut [u8], +) -> Result<(), RenderBuildError> { + if fragment.len() < FRAGMENT_DESCRIPTOR_SIZE + || rce.len() < G17P_FRAGMENT_RCE_STORAGE_SIZE + { + return Err(RenderBuildError::BufferTooSmall); + } + + let programs = [ + (0x07a0usize, 0x00a0usize, FRAGMENT_REGISTER_COUNT), + (0x0ec0usize, 0x07c0usize, PARTIAL_STORE_REGISTER_COUNT), + (0x15e0usize, 0x0ee0usize, PARTIAL_RESUME_REGISTER_COUNT), + (0x1d00usize, 0x1600usize, PARTIAL_LOAD_REGISTER_COUNT), + ]; + + rce[..G17P_FRAGMENT_RCE_STORAGE_SIZE].fill(0); + for (index, (_pointer_offset, source_offset, register_count)) in + programs.into_iter().enumerate() + { + let bytes = register_count + .checked_mul(REGISTER_BYTES) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let target_offset = index + .checked_mul(G17P_FRAGMENT_RCE_PROGRAM_STRIDE as usize) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let source_end = source_offset + .checked_add(bytes) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let target_end = target_offset + .checked_add(bytes) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + rce[target_offset..target_end] + .copy_from_slice(&fragment[source_offset..source_end]); + } + let _ = rce_gpu_va; + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct AddressedBytes { + pub(crate) address: u64, + pub(crate) bytes: [u8; N], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueuePublicationInput { + pub(crate) queue: u64, + pub(crate) pointers: u64, + pub(crate) item_ring: u64, + pub(crate) channel_ring: u64, + pub(crate) channel_producer: u64, + pub(crate) items: [u64; 3], + pub(crate) grid_index: u16, + pub(crate) slot_index: u16, + pub(crate) current_write: u32, + pub(crate) group_number: u32, + pub(crate) first_submit: bool, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueuePublication { + pub(crate) item_entries: [AddressedBytes<8>; 3], + pub(crate) event: AddressedBytes, + pub(crate) queue_write: AddressedBytes<4>, + pub(crate) ring_slot: AddressedBytes, + pub(crate) producer: AddressedBytes<4>, + pub(crate) write_after: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PairedQueuePublication { + pub(crate) tiling: QueuePublication, + pub(crate) fragment: QueuePublication, +} + +pub(crate) fn build_queue_publication( + kind: RenderDescriptorKind, + input: QueuePublicationInput, +) -> Result { + let write_after = input + .current_write + .checked_add(3) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + if input.grid_index > 0xff { + return Err(RenderBuildError::QueueGridOutOfRange); + } + if input.slot_index > 0xff { + return Err(RenderBuildError::RingSlotOutOfRange); + } + if input.group_number > (u32::MAX >> 8) { + return Err(RenderBuildError::ArithmeticOverflow); + } + + let item_offset = (input.current_write as u64) + .checked_mul(8) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let first_item = input + .item_ring + .checked_add(item_offset) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let item_entries = [ + AddressedBytes { + address: first_item, + bytes: input.items[0].to_le_bytes(), + }, + AddressedBytes { + address: first_item + .checked_add(8) + .ok_or(RenderBuildError::ArithmeticOverflow)?, + bytes: input.items[1].to_le_bytes(), + }, + AddressedBytes { + address: first_item + .checked_add(16) + .ok_or(RenderBuildError::ArithmeticOverflow)?, + bytes: input.items[2].to_le_bytes(), + }, + ]; + + let mut event = [0u8; EVENT_RECORD_SIZE]; + let counter = input.group_number << 8; + put_u32(&mut event, 0x00, EVENT_SELECTOR); + put_u32( + &mut event, + 0x04, + EVENT_SUBTYPE_BASE | input.grid_index as u32, + ); + put_u32(&mut event, 0x08, counter); + put_u32( + &mut event, + 0x10, + match kind { + RenderDescriptorKind::Tiling => 0, + RenderDescriptorKind::Fragment => 0x100, + }, + ); + + let mut ring_slot = [0u8; RING_SLOT_SIZE]; + put_u64(&mut ring_slot, 0x08, input.queue); + put_u32( + &mut ring_slot, + 0x10, + match kind { + RenderDescriptorKind::Tiling => 0, + RenderDescriptorKind::Fragment => 1, + }, + ); + let flags = (write_after & 0xffff) + | ((input.grid_index as u32) << 16) + | if input.first_submit { 1 << 24 } else { 0 }; + put_u32(&mut ring_slot, 0x14, flags); + + let queue_write_address = input + .pointers + .checked_add(QUEUE_WRITE_OFFSET) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let slot_offset = (input.slot_index as u64) + .checked_mul(RING_SLOT_SIZE as u64) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let ring_slot_address = input + .channel_ring + .checked_add(slot_offset) + .ok_or(RenderBuildError::ArithmeticOverflow)?; + let producer = (input.slot_index as u32 + 1) & 0xff; + Ok(QueuePublication { + item_entries, + event: AddressedBytes { + address: input.items[2], + bytes: event, + }, + queue_write: AddressedBytes { + address: queue_write_address, + bytes: write_after.to_le_bytes(), + }, + ring_slot: AddressedBytes { + address: ring_slot_address, + bytes: ring_slot, + }, + producer: AddressedBytes { + address: input.channel_producer, + bytes: producer.to_le_bytes(), + }, + write_after, + }) +} + +pub(crate) fn build_paired_queue_publication( + tiling: QueuePublicationInput, + fragment: QueuePublicationInput, +) -> Result { + Ok(PairedQueuePublication { + tiling: build_queue_publication(RenderDescriptorKind::Tiling, tiling)?, + fragment: build_queue_publication(RenderDescriptorKind::Fragment, fragment)?, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mcache_mapping_matches_native_g17p_3d_records() { + assert_eq!( + encode_g17p_mcache_mapping(G17PMcacheMapping { + address: 0xeeee_0000, + size_units: 0x4d000, + hwsid: 0x18, + selector: 3, + range_shift: 2, + }) + .unwrap(), + [ + 0x00, 0xdc, 0xdd, 0x01, 0x00, 0x06, 0x8c, 0x00, 0x00, 0xac, 0xe2, 0x01, + 0x00, 0x12, 0x00, 0x00, + ] + ); + assert_eq!( + encode_g17p_mcache_mapping(G17PMcacheMapping { + address: 0x100_0e83_8000, + size_units: 0x26a80, + hwsid: 0x18, + selector: 3, + range_shift: 1, + }) + .unwrap(), + [ + 0x00, 0x07, 0x1d, 0x00, 0x02, 0x06, 0x8c, 0x00, 0x80, 0x71, 0x1f, 0x00, + 0x02, 0x0a, 0x00, 0x00, + ] + ); + } + + fn fnv1a64(parts: &[&[u8]]) -> u64 { + let mut value = 0xcbf2_9ce4_8422_2325u64; + for part in parts { + for byte in *part { + value ^= *byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + } + value + } + + fn encoded_register_hash(registers: &[RegisterWrite]) -> u64 { + let mut raw = [0u8; FRAGMENT_REGISTER_COUNT * REGISTER_BYTES]; + encode_registers(&mut raw, 0, registers); + fnv1a64(&[&raw[..registers.len() * REGISTER_BYTES]]) + } + + fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) + } + + fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) + } + + fn objects() -> RenderDescriptorObjects { + RenderDescriptorObjects { + record_a: 0xffff_fc20_0150_0000, + shared: 0xffff_fc20_0160_0000, + record_b: 0xffff_fc20_0158_0000, + zero: 0xffff_fc20_0168_0000, + } + } + + #[test] + fn source_profile_pins_rsrc8_and_native_aligned_programs() { + let parameters = G17pRenderParameters::source_rsrc8(); + assert_eq!(parameters.store_pipeline_bind, 0x8); + assert_eq!(parameters.load_pipeline_bind, 0x0007_8000_0000_0040); + assert_eq!(parameters.store_pipeline, 0x0000_0100_0199_0640); + assert_eq!(parameters.load_pipeline, 0x0000_0100_0199_0240); + } + + #[test] + fn render_lifecycles_use_native_gid_predecessor_and_consecutive_currents() { + // AtomicU32(1): reserve-group returns predecessor 1 and advances the + // counter to 0x01000001. Native OSAddAtomic64 returns the old value, + // hence 3D current 0x01000001 and TA current 0x01000002. + assert_eq!( + g17p_render_lifecycle_pair(1, 0x0100_0001), + Some((0x0000_0001_0100_0001, 0x0000_0001_0100_0002)) + ); + // Exact first render pair from the working Metal capture. + assert_eq!( + g17p_render_lifecycle_pair(0x0700_c271, 0x0800_c294), + Some((0x0700_c271_0800_c294, 0x0700_c271_0800_c295)) + ); + assert_eq!(g17p_render_lifecycle_pair(0, 1), None); + assert_eq!(g17p_render_lifecycle_pair(1, u32::MAX), None); + + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.lifecycle = 0x1111_2222_3333_4444; + parameters.fragment_lifecycle = 0x5555_6666_7777_8888; + let ta = build_ta_registers(¶meters).unwrap(); + let fragment = build_fragment_registers(¶meters).unwrap(); + assert!(ta.iter().filter(|write| write.value == parameters.lifecycle).count() >= 3); + assert_eq!( + fragment + .iter() + .filter(|write| write.value == parameters.fragment_lifecycle) + .count(), + 3 + ); + let mut ta_descriptor = [0u8; TA_DESCRIPTOR_SIZE]; + build_ta_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut ta_descriptor, + ) + .unwrap(); + assert_eq!(get_u32(&ta_descriptor, 0x086e), 0x1111_2222); + assert_eq!(get_u64(&ta_descriptor, 0x08ce), 0x3333_4444); + + let mut descriptor = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_fragment_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut descriptor, + ) + .unwrap(); + assert_eq!(get_u32(&descriptor, 0x2108), 0x5555_6666); + assert_eq!(get_u64(&descriptor, 0x2168), 0x7777_8888); + } + + #[test] + fn vdm_user_offset_matches_g17p_compact_window() { + assert_eq!(vdm_user_offset(0x10_0186_0420).unwrap(), 0x0186_0420); + assert_eq!( + vdm_user_offset(0x4f_fff5_4400).unwrap(), + 0x3f_fff5_4400 + ); + assert_eq!( + vdm_user_offset(0xbf_fff5_4400), + Err(RenderBuildError::VdmAddressOutOfRange) + ); + } + + #[test] + fn native_ta_addresses_use_split_resource_and_user_domains() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.context_base = 0x70_0000_0000; + parameters.tilemap = 0x70_011e_0a00; + parameters.layermeta = 0x70_011e_2a00; + parameters.heapmeta = 0x70_011e_2a00; + parameters.tpc = 0x70_0129_0000; + parameters.deflake_3 = 0x10_0009_ca60; + parameters.deflake_2 = 0x10_0009_ca80; + parameters.deflake_1 = 0x10_0009_cd00; + + let registers = build_ta_registers(¶meters).unwrap(); + let value = |number| { + registers + .iter() + .find(|write| write.number == number) + .unwrap() + .value + }; + assert_eq!(value(0x1c039), 0x011e_0a00); + assert_eq!(value(0x1c031), 0x8000_0000_011e_2a00); + assert_eq!(value(0x1c0a1), 0x0129_0000); + assert_eq!(value(0x10111), 0x0009_cd00); + assert_eq!(value(0x10119), 0x0009_ca80); + assert_eq!(value(0x1c950), 0x0004_0000_0009_ca60); + + let mut descriptor = [0u8; TA_DESCRIPTOR_SIZE]; + build_ta_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata { + context_id: 1, + submission_ordinal: 0, + command_buffer_state_word: 0, + ta_hardware_buffer_id: 0, + submit_sequence: 1, + }, + &mut descriptor, + ) + .unwrap(); + assert_eq!(get_u32(&descriptor, 0x07d6), 0x0009_cd00); + assert_eq!(get_u64(&descriptor, 0x0780), parameters.tpc); + + parameters.deflake_1 = 0x11_0000_0000; + assert_eq!( + build_ta_registers(¶meters), + Err(RenderBuildError::UserAddressOutOfRange) + ); + } + + #[test] + fn register_programs_match_m1n1_9165484() { + let parameters = G17pRenderParameters::source_rsrc8(); + let tiling = build_ta_registers(¶meters).unwrap(); + let fragment = build_fragment_registers(¶meters).unwrap(); + assert_eq!(tiling.len(), 73); + assert_eq!(fragment.len(), 89); + assert_eq!(encoded_register_hash(&tiling), 0x29dc_2725_a3c2_90c9); + assert_eq!(encoded_register_hash(&fragment), 0xdb21_2bfe_1bbf_85c8); + assert_eq!(fragment[2], RegisterWrite::new(0x15379, 0x8)); + assert_eq!( + fragment[3], + RegisterWrite::new(0x15381, G17P_SOURCE_EOT_PIPELINE) + ); + assert_eq!( + fragment[4], + RegisterWrite::new(0x15369, G17P_LOAD_RESOURCE_SPEC) + ); + assert_eq!( + fragment[5], + RegisterWrite::new(0x15371, G17P_SOURCE_BG_PIPELINE) + ); + } + + #[test] + fn reused_cpu_staging_produces_identical_wire_descriptors() { + let parameters = G17pRenderParameters::source_rsrc8(); + for dirty in [0x00, 0x55, 0xff] { + let mut staging = [dirty; 0x8000]; + staging.fill(0); + let mut tiling = [0; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0; FRAGMENT_DESCRIPTOR_SIZE]; + let ta_metadata = RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling); + let fragment_metadata = RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment); + build_ta_descriptor(¶meters, objects(), ta_metadata, &mut tiling).unwrap(); + build_fragment_descriptor(¶meters, objects(), fragment_metadata, &mut fragment).unwrap(); + let (ta, fr) = staging.split_at_mut(0x4000); + build_ta_descriptor(¶meters, objects(), ta_metadata, ta).unwrap(); + build_fragment_descriptor(¶meters, objects(), fragment_metadata, fr).unwrap(); + assert_eq!(&ta[..TA_DESCRIPTOR_SIZE], &tiling); + assert_eq!(&fr[..FRAGMENT_DESCRIPTOR_SIZE], &fragment); + assert!(ta[TA_DESCRIPTOR_SIZE..].iter().all(|v| *v == 0)); + assert!(fr[FRAGMENT_DESCRIPTOR_SIZE..].iter().all(|v| *v == 0)); + } + } + + #[test] + fn descriptors_match_m1n1_9165484_and_duplicate_pipeline_fields() { + let parameters = G17pRenderParameters::source_rsrc8(); + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_ta_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut tiling, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut fragment, + ) + .unwrap(); + + assert_eq!(fnv1a64(&[&tiling]), 0x8902_c437_9d74_1cc7); + assert_eq!(fnv1a64(&[&fragment]), 0xdd6e_bcc7_77e7_834d); + assert_eq!(get_u32(&fragment, 0xb8), 0x15379); + assert_eq!(get_u64(&fragment, 0xbc), 0x8); + assert_eq!(get_u64(&fragment, 0xc8), G17P_SOURCE_EOT_PIPELINE); + assert_eq!(get_u64(&fragment, 0x1e78), G17P_LOAD_RESOURCE_SPEC); + assert_eq!(get_u64(&fragment, 0x1e80), G17P_SOURCE_BG_PIPELINE); + assert_eq!(get_u32(&fragment, 0x1f78), 0x8); + assert_eq!(get_u64(&fragment, 0x1f7c), G17P_SOURCE_EOT_PIPELINE); + } + + #[test] + fn pool_a_record_one_changes_only_the_two_stage_pointer_fields() { + let parameters = G17pRenderParameters::source_rsrc8(); + let baseline_objects = objects(); + let record_one_objects = baseline_objects.with_pool_a_record_index(1).unwrap(); + let mut baseline_ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut record_one_ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut baseline_3d = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + let mut record_one_3d = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_ta_descriptor( + ¶meters, + baseline_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut baseline_ta, + ) + .unwrap(); + build_ta_descriptor( + ¶meters, + record_one_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut record_one_ta, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + baseline_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut baseline_3d, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + record_one_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut record_one_3d, + ) + .unwrap(); + + assert_eq!(record_one_objects.record_a, baseline_objects.record_a + 0x100); + assert_eq!(get_u64(&baseline_ta, 0x10), baseline_objects.record_a); + assert_eq!(get_u64(&record_one_ta, 0x10), record_one_objects.record_a); + assert_eq!(get_u64(&baseline_3d, 0x20), baseline_objects.record_a); + assert_eq!(get_u64(&record_one_3d, 0x20), record_one_objects.record_a); + for (offset, (before, after)) in baseline_ta.iter().zip(record_one_ta.iter()).enumerate() { + if !(0x10..0x18).contains(&offset) { + assert_eq!(before, after, "unexpected TA descriptor change at {offset:#x}"); + } + } + for (offset, (before, after)) in baseline_3d.iter().zip(record_one_3d.iter()).enumerate() { + if !(0x20..0x28).contains(&offset) { + assert_eq!(before, after, "unexpected 3D descriptor change at {offset:#x}"); + } + } + } + + #[test] + fn pool_a_record_index_checks_overflow() { + let objects = RenderDescriptorObjects { + record_a: u64::MAX - 0xff, + ..objects() + }; + assert_eq!( + objects.with_pool_a_record_index(1), + Err(RenderBuildError::ArithmeticOverflow) + ); + } + + #[test] + fn fragment_hwpb_split_changes_only_fragment_plus_0x28() { + let parameters = G17pRenderParameters::source_rsrc8(); + let baseline_objects = objects(); + let hwpb = 0xffff_fc20_c099_8000; + let fragment_objects = baseline_objects.with_shared(hwpb); + let mut baseline_ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut split_ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut baseline_3d = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + let mut split_3d = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + + build_ta_descriptor( + ¶meters, + baseline_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut baseline_ta, + ) + .unwrap(); + build_ta_descriptor( + ¶meters, + baseline_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), + &mut split_ta, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + baseline_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut baseline_3d, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + fragment_objects, + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut split_3d, + ) + .unwrap(); + + assert_eq!(baseline_ta, split_ta); + assert_eq!(get_u64(&baseline_3d, 0x20), baseline_objects.record_a); + assert_eq!(get_u64(&split_3d, 0x20), baseline_objects.record_a); + assert_eq!(get_u64(&baseline_3d, 0x28), baseline_objects.shared); + assert_eq!(get_u64(&split_3d, 0x28), hwpb); + assert_eq!(get_u64(&baseline_3d, 0x30), baseline_objects.record_b); + assert_eq!(get_u64(&split_3d, 0x30), baseline_objects.record_b); + assert_eq!(get_u64(&baseline_3d, 0x38), baseline_objects.zero); + assert_eq!(get_u64(&split_3d, 0x38), baseline_objects.zero); + for (offset, (before, after)) in baseline_3d.iter().zip(split_3d.iter()).enumerate() { + if !(0x28..0x30).contains(&offset) { + assert_eq!( + before, after, + "unexpected 3D descriptor change at {offset:#x}" + ); + } + } + } + + #[test] + fn memory_backed_render_policy_is_independent_of_tib_size() { + for tib_blocks in [4, 8, 9, 12, 16] { + for process_empty_tiles in [false, true] { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.tib_blocks = tib_blocks; + parameters.process_empty_tiles = process_empty_tiles; + parameters.tile_config = 0x280 + | if process_empty_tiles { 0x1_0000 } else { 0 }; + let mut fragment = [0xa5u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_cold_opening_fragment_descriptor( + ¶meters, objects(), 1, 1, &mut fragment, + ) + .unwrap(); + assert_eq!(fragment[0x2100], 0); + assert_eq!(get_u64(&fragment, 0x1f38), tib_blocks); + assert_eq!(get_u64(&fragment, 0x1f58), parameters.tile_config); + // Keep real memory-backed partial phase programs, not the + // native memoryless count-two reset-only phase variant. + for (at, count) in [(0xec8, 16), (0x15e8, 23), (0x1d08, 10)] { + assert_eq!(get_u32(&fragment, at) & 0xffff, count); + } + apply_g17p_retained_descriptor_fields( + RenderDescriptorKind::Fragment, 0x7000098000, 0, + 1, 1, 1, false, false, &mut fragment, + ) + .unwrap(); + assert_eq!(fragment[0x2100], 0); + } + } + } + + #[test] + fn fragment_sample_count_matches_tile_configuration_on_every_submission() { + for (sample_log2, samples) in [(0, 1), (1, 2), (2, 4)] { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.utile_config = 0xa000 | sample_log2; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + + build_fragment_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut fragment, + ) + .unwrap(); + assert_eq!(get_u32(&fragment, 0x50), samples, "retained {samples}x"); + assert_eq!(get_u32(&fragment, 0xac), 0x10009); + assert_eq!(get_u64(&fragment, 0xb0) & 3, sample_log2); + + build_cold_opening_fragment_descriptor( + ¶meters, objects(), 1, 1, &mut fragment, + ) + .unwrap(); + assert_eq!(get_u32(&fragment, 0x50), samples, "first {samples}x"); + assert_eq!(get_u64(&fragment, 0xb0) & 3, sample_log2); + } + } + + #[test] + fn partial_render_uses_phase_specific_zls_control() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.depth_buffer = 0x5be8_2a00_00; + parameters.stencil_buffer = 0x5bea_4240_00; + parameters.depth_flags = 0x000c_0000; + parameters.partial_depth_flags = 0x000c_c000; + + let main = build_fragment_registers(¶meters).unwrap(); + assert_eq!( + main.iter().find(|reg| reg.number == 0x15319).unwrap().value, + 0x000c_0000, + ); + let store = partial_store_registers(¶meters); + let resume = partial_resume_registers(¶meters); + let load = partial_load_registers(¶meters); + for registers in [store.as_slice(), resume.as_slice(), load.as_slice()] { + assert_eq!( + registers.iter().find(|reg| reg.number == 0x15319).unwrap().value, + 0x000c_c000, + ); + } + } + + #[test] + fn cold_opening_descriptors_apply_only_the_native_first_work_overrides() { + let parameters = G17pRenderParameters::source_rsrc8(); + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_cold_opening_ta_descriptor(¶meters, objects(), 1, 0, 0, &mut tiling).unwrap(); + build_cold_opening_fragment_descriptor(¶meters, objects(), 1, 1, &mut fragment) + .unwrap(); + + assert_eq!(&tiling[0x38..0x3e], &[0x47, 0, 0x49, 0, 0x49, 0]); + assert_eq!(tiling[0x789], 0x08); + assert_eq!(get_u32(&tiling, 0x892), 1); + assert_eq!(&tiling[0x93e..0x940], &[0xd0, 0x91]); + + assert_eq!(get_u32(&fragment, 0x50), 1); + assert_eq!(&fragment[0x80..0x86], &[0x56, 0, 0x57, 0, 0x57, 0]); + assert_eq!(&fragment[0x88..0x8a], &[0x59, 0]); + assert_eq!(get_u32(&fragment, 0x90), 0); // final channel payload is staged later + assert_eq!(get_u32(&fragment, 0x2154), 1); + assert_eq!(get_u32(&fragment, 0x215c), 1); + assert_eq!(fragment[0x2100], 0); + assert_eq!(&fragment[0x2114..0x2134], &[0; 0x20]); + assert_eq!(get_u64(&fragment, 0x21d7), 0); + assert_eq!(fragment[0x222d], 1); + + } + + #[test] + fn cold_opening_ta_couples_the_hardware_buffer_id_without_clobbering_registers() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.ta_hardware_buffer_id = 5; + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + build_cold_opening_ta_descriptor( + ¶meters, + objects(), + 1, + 0, + parameters.ta_hardware_buffer_id, + &mut tiling, + ) + .unwrap(); + + assert_eq!(get_u32(&tiling, 0x18), 5); + // +0x300 is register 0x1c830 and +0x304 is its value word. + assert_eq!(get_u32(&tiling, 0x300), 0x1c830); + assert_eq!(get_u64(&tiling, 0x304), 5); + } + + #[test] + fn shared_usc_flist_id_drives_both_stage_registers_independently_of_pm_id() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.ta_hardware_buffer_id = 7; + parameters.usc_flist_hardware_buffer_id = 5; + // Every diagnostic bit set must not recouple the retired bit-16 + // capture literals to these source-owned registers. + parameters.native_ta_registers = u32::MAX; + + let tiling = build_ta_registers(¶meters).unwrap(); + let fragment = build_fragment_registers(¶meters).unwrap(); + assert_eq!( + tiling.iter().find(|write| write.number == 0x1c830).unwrap().value, + 7 + ); + assert_eq!( + tiling.iter().find(|write| write.number == 0x14308).unwrap().value, + 5 + ); + assert_eq!( + fragment.iter().find(|write| write.number == 0x14048).unwrap().value, + 5 + ); + assert_eq!( + fragment.iter().find(|write| write.number == 0x1c838).unwrap().value, + 7 + ); + } + + #[test] + fn descriptor_context_and_state_word_drive_every_native_work_key() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.ta_hardware_buffer_id = 5; + parameters.gpc_perf_state_map = 7; + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + let tiling_metadata = RenderDescriptorMetadata { + context_id: 9, + submission_ordinal: 4, + command_buffer_state_word: 0x40, + ta_hardware_buffer_id: parameters.ta_hardware_buffer_id, + submit_sequence: 7, + }; + let fragment_metadata = RenderDescriptorMetadata { + command_buffer_state_word: 0x41, + ..tiling_metadata + }; + + build_ta_descriptor(¶meters, objects(), tiling_metadata, &mut tiling).unwrap(); + build_fragment_descriptor(¶meters, objects(), fragment_metadata, &mut fragment) + .unwrap(); + + for (raw, offset, register, value) in [ + (&tiling[..], 0x300, 0x1c830, 5u64), + (&tiling[..], 0x36c, 0x10209, 0x900), + (&tiling[..], 0x378, 0x1c9f0, 0x900), + (&tiling[..], 0x384, 0x14320, 0x900), + (&fragment[..], 0x454, 0x1c838, 5u64), + (&fragment[..], 0x46c, 0x10211, 0x901), + (&fragment[..], 0x478, 0x10420, 0x901), + ] { + assert_eq!(get_u32(raw, offset), register); + assert_eq!(get_u64(raw, offset + 4), value); + } + assert_eq!(get_u32(&tiling, 0x0c), 9); + assert_eq!(get_u32(&fragment, 0x0c), 9); + } + + #[test] + fn tpc_extent_survives_cold_and_retained_descriptor_construction() { + for size in [0x800, 0x2000, 0x20000, 0x1000000] { + let mut p = G17pRenderParameters::source_rsrc8(); + p.tpc_size = size; + let mut ta = [0xa5; TA_DESCRIPTOR_SIZE]; + build_cold_opening_ta_descriptor(&p, objects(), 7, 0, 0, &mut ta).unwrap(); + for cold in [true, false] { + if !cold { + build_ta_descriptor(&p, objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), &mut ta).unwrap(); + } + apply_g17p_retained_descriptor_fields(RenderDescriptorKind::Tiling, + 0x7000090000, 0xfffffc20c0828000, 11, 12, 32, + cold, true, &mut ta).unwrap(); + assert_eq!(get_u64(&ta, 0x780), p.tpc); + assert_eq!(get_u64(&ta, 0x788), size); + } + } + } + + #[test] + fn rce_boundaries_survive_retained_descriptor_rebuilds() { + let u16_at = |raw: &[u8], at: usize| u16::from_le_bytes([raw[at], raw[at + 1]]); + for dimension in [32, 512, 1024] { + let mut p = G17pRenderParameters::source_rsrc8(); + p.width = dimension; + p.height = dimension; + let mut raw = [0xa5; FRAGMENT_DESCRIPTOR_SIZE]; + let mut ta = [0xa5; TA_DESCRIPTOR_SIZE]; + for ordinal in [0, 1, 31, 32, 97, 232, 513] { + if ordinal == 0 { + build_cold_opening_fragment_descriptor(&p, objects(), 7, 1, &mut raw).unwrap(); + build_cold_opening_ta_descriptor(&p, objects(), 7, 0, 0, &mut ta).unwrap(); + } else { + build_fragment_descriptor(&p, objects(), RenderDescriptorMetadata { + context_id: 7, submission_ordinal: ordinal, + ..RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment) + }, &mut raw).unwrap(); + build_ta_descriptor(&p, objects(), RenderDescriptorMetadata { + context_id: 7, submission_ordinal: ordinal, + ..RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling) + }, &mut ta).unwrap(); + } + apply_g17p_retained_descriptor_fields(RenderDescriptorKind::Fragment, + 0x7000098000, 0xfffffc20c0828000, 12, 12, ordinal, + ordinal == 0, true, &mut raw).unwrap(); + apply_g17p_retained_descriptor_fields(RenderDescriptorKind::Tiling, + 0x7000090000, 0xfffffc20c0828000, 11, 12, ordinal, + ordinal == 0, true, &mut ta).unwrap(); + assert_eq!([u16_at(&ta, 0x38), u16_at(&ta, 0x3a), + u16_at(&ta, 0x3c)], [71, 73, 73]); + assert_eq!(get_u32(&ta, 0x768) & 0xffff, 73); + for (index, register) in [(70, 0x01740), (71, 0x1c880), (72, 0x1c898)] { + assert_eq!(get_u32(&ta, 0x60 + index * 12), register); + } + // Independent native publication oracle: all 160 full FRG + // images carry these boundaries, including pending overlap. + assert_eq!([u16_at(&raw, 0x80), u16_at(&raw, 0x82), + u16_at(&raw, 0x84), u16_at(&raw, 0x88)], [86, 87, 87, 89]); + assert_eq!(get_u32(&raw, 0x7a8) & 0xffff, 89); + // Check which actual encoded instructions bracket the indices, + // so a changed recipe cannot silently retain stale boundaries. + for (index, register, value) in [(85, 0x01731, 1), + (86, 0x16020, 1), (87, 0x16020, 0), (88, 0x16068, 0x40000)] { + let at = 0xa0 + index * 12; + assert_eq!(get_u32(&raw, at), register); + assert_eq!(get_u64(&raw, at + 4), value); + } + } + } + } + + #[test] + fn allocation_epoch_survives_cold_and_retained_encoders_and_wrap() { + for epoch in [0, 1, 2, 3, 127, 255] { + for diagnostic in [false, true] { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.context_generation = epoch; + parameters.native_pm_bytes = diagnostic; + for cold in [false, true] { + let mut ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + if cold { + build_cold_opening_ta_descriptor(¶meters, objects(), 7, 0, 0, &mut ta).unwrap(); + build_cold_opening_fragment_descriptor(¶meters, objects(), 7, 1, &mut fragment).unwrap(); + } else { + build_ta_descriptor(¶meters, objects(), RenderDescriptorMetadata::source(RenderDescriptorKind::Tiling), &mut ta).unwrap(); + build_fragment_descriptor(¶meters, objects(), RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), &mut fragment).unwrap(); + } + assert_eq!(ta[0x94d], epoch); + assert_eq!(fragment[0x21e7], epoch); + // Retained record restaging changes submission identity, + // but must preserve the immutable execution-context epoch. + apply_g17p_retained_descriptor_fields(RenderDescriptorKind::Tiling, + 0x7000098000, 0xfffffc20c0828000, 0, 1, 513, false, true, &mut ta).unwrap(); + apply_g17p_retained_descriptor_fields(RenderDescriptorKind::Fragment, + 0x7000098000, 0xfffffc20c0828000, 1, 1, 513, false, true, &mut fragment).unwrap(); + assert_eq!(ta[0x94d], epoch); + assert_eq!(fragment[0x21e7], epoch); + } + } + } + } + + #[test] + fn completion_generation_matches_render_tags_across_reuse_and_wrap() { + let parameters = G17pRenderParameters::source_rsrc8(); + // Include a second client's cold submission and generations beyond the + // six-bit hardware tag wrap. The slot ordinal is intentionally unrelated. + for generation in [0, 1, 2, 62, 64, 65, 128, 0x12345678, u32::MAX - 1] { + for context_id in [1, 7] { + let metadata = RenderDescriptorMetadata { + context_id, + submission_ordinal: 2, + command_buffer_state_word: generation, + ..RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment) + }; + let mut ta = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + let mut event_control = [0xa5; 0x100]; + build_ta_descriptor(¶meters, objects(), metadata, &mut ta).unwrap(); + build_fragment_descriptor(¶meters, objects(), metadata, &mut fragment) + .unwrap(); + set_render_event_control_generation(&mut event_control, generation).unwrap(); + assert_eq!(get_u32(&event_control, 8), generation); + assert_eq!(&event_control[..8], &[0xa5; 8]); + assert!(event_control[12..].iter().all(|&v| v == 0xa5)); + let maintenance_key = ((context_id as u64) << 8) + | (get_u32(&event_control, 8) & 63) as u64; + for (raw, offset) in [(&ta[..], 0x36c), (&ta[..], 0x378), + (&ta[..], 0x384), (&fragment[..], 0x46c), + (&fragment[..], 0x478)] { + assert_eq!(get_u64(raw, offset + 4), maintenance_key); + } + } + } + assert_eq!( + set_render_event_control_generation(&mut [0; 11], 0), + Err(RenderBuildError::BufferTooSmall) + ); + } + + #[test] + fn fragment_parameter_buffer_owner_is_independent_of_perf_map_and_tail() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.native_ta_registers = u32::MAX; + + for (control, low, map, id) in [(0, 7, 3, 0), (1, 3, 7, 5), (1, 0, 1, 3)] { + parameters.ta_hardware_buffer_id = id; + parameters.gpc_perf_state_control = control; + parameters.gpc_perf_state_map_low = low; + parameters.gpc_perf_state_map = map; + let fragment = build_fragment_registers(¶meters).unwrap(); + let write = fragment + .iter() + .find(|write| write.number == 0x1c838) + .unwrap(); + assert_eq!(write.value, id as u64); + + let mut descriptor = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + build_fragment_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata::source(RenderDescriptorKind::Fragment), + &mut descriptor, + ) + .unwrap(); + assert_eq!(get_u32(&descriptor, 0x2104), control); + assert_eq!(get_u32(&descriptor, 0x210c), low); + assert_eq!(get_u32(&descriptor, 0x2110), u32::MAX); + assert_eq!(get_u32(&descriptor, 0x2124), 1); + assert_eq!(get_u32(&descriptor, 0x2128), 1); + assert_eq!(get_u32(&descriptor, 0x212c), 0); + assert_eq!(get_u32(&descriptor, 0x2130), 0); + } + } + + #[test] + fn priority_zero_ta_force_stamp_is_independent_of_physical_qid() { + for (queue_id, ordinal, cold, expected) in [ + (0, 0, true, 1), + (0, 1, false, 1), + (0, 2, false, 1), + (2, 0, true, 1), + (2, 1, false, 1), + (5, 0, true, 1), + (5, 1, false, 1), + ] { + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + apply_g17p_retained_descriptor_fields( + RenderDescriptorKind::Tiling, + 0x70_0001_8000, + 0xffff_fc20_c082_8000, + queue_id, + queue_id + 1, + ordinal, + cold, + true, + &mut tiling, + ) + .unwrap(); + assert_eq!(get_u32(&tiling, 0x8c2), expected); + assert_eq!(get_u32(&tiling, 0x8be), 0); + assert_eq!( + get_u64(&tiling, 0x8c6), + ((ordinal + 1) as u64) << 40 | (ordinal + 1) as u64 + ); + } + } + + #[test] + fn retained_descriptor_refresh_stages_native_external_rce_and_advances_item_two() { + let mut parameters = G17pRenderParameters::source_rsrc8(); + parameters.lifecycle = 0x1122_3344_5566_7788; + parameters.fragment_lifecycle = 0x99aa_bbcc_ddee_ff00; + let mut tiling = [0u8; TA_DESCRIPTOR_SIZE]; + let mut fragment = [0u8; FRAGMENT_DESCRIPTOR_SIZE]; + let tiling_va = 0x70_0001_8000; + let fragment_va = 0x70_0009_8000; + build_ta_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata { + context_id: 1, + submission_ordinal: 1, + command_buffer_state_word: 2, + ta_hardware_buffer_id: 0, + submit_sequence: 3, + }, + &mut tiling, + ) + .unwrap(); + build_fragment_descriptor( + ¶meters, + objects(), + RenderDescriptorMetadata { + context_id: 1, + submission_ordinal: 1, + command_buffer_state_word: 3, + ta_hardware_buffer_id: 0, + submit_sequence: 2, + }, + &mut fragment, + ) + .unwrap(); + apply_g17p_retained_descriptor_fields( + RenderDescriptorKind::Tiling, + tiling_va, + 0xffff_fc20_c082_8000, + 5, + 6, + 1, + false, + false, + &mut tiling, + ) + .unwrap(); + apply_g17p_retained_descriptor_fields( + RenderDescriptorKind::Fragment, + fragment_va, + 0xffff_fc20_c082_8000, + 6, + 6, + 1, + false, + false, + &mut fragment, + ) + .unwrap(); + let rce_va = 0x70_0028_cf20; + let mut rce = [0u8; G17P_FRAGMENT_RCE_STORAGE_SIZE]; + stage_g17p_fragment_rce_programs(&mut fragment, rce_va, &mut rce).unwrap(); + + assert_eq!(get_u64(&tiling, 0x760), tiling_va + 0x60); + assert_eq!(get_u32(&tiling, 0x79c), 6); + assert_eq!(get_u32(&tiling, 0x8ba), 5); + assert_eq!(get_u32(&tiling, 0x7a0), 0x200); + assert_eq!(get_u32(&tiling, 0x7a8), 2); + assert_eq!(get_u32(&tiling, 0x8d4), 0x1_0000); + assert_eq!(get_u64(&tiling, 0x934), 0xffff_fc20_c082_8000); + assert_eq!(get_u64(&fragment, 0x7a0), fragment_va + 0xa0); + assert_eq!(get_u64(&fragment, 0xec0), fragment_va + 0x7c0); + assert_eq!(get_u64(&fragment, 0x15e0), fragment_va + 0xee0); + assert_eq!(get_u64(&fragment, 0x1d00), fragment_va + 0x1600); + assert_eq!( + &rce[..FRAGMENT_REGISTER_COUNT * REGISTER_BYTES], + &fragment[0xa0..0xa0 + FRAGMENT_REGISTER_COUNT * REGISTER_BYTES], + ); + assert_eq!(get_u32(&tiling, 0x086e), 0x1122_3344); + assert_eq!(get_u64(&tiling, 0x08ce), 0x5566_7788); + assert_eq!(get_u32(&fragment, 0x2108), 0x99aa_bbcc); + assert_eq!(get_u64(&fragment, 0x2168), 0xddee_ff00); + assert_eq!( + &rce[0x720..0x720 + PARTIAL_STORE_REGISTER_COUNT * REGISTER_BYTES], + &fragment[0x7c0..0x7c0 + PARTIAL_STORE_REGISTER_COUNT * REGISTER_BYTES], + ); + assert_eq!(get_u32(&fragment, 0x2154), 6); + assert_eq!(get_u32(&fragment, 0x215c), 0); + assert_eq!(get_u32(&fragment, 0x2170), 1); + assert_eq!(get_u64(&fragment, 0x21ce), 0xffff_fc20_c082_8000); + } + + fn publication_input(kind: RenderDescriptorKind) -> QueuePublicationInput { + let fragment = matches!(kind, RenderDescriptorKind::Fragment); + let grid = fragment as u16; + QueuePublicationInput { + queue: 0xffff_fc20_c000_8000 + grid as u64 * 0xc0, + pointers: 0xffff_fc20_0001_0000 + grid as u64 * 0x80, + item_ring: 0xffff_fc20_c000_c000 + grid as u64 * 0x2870, + channel_ring: 0xffff_fc20_c010_0000 + grid as u64 * 0x4000, + channel_producer: 0xffff_fc20_0170_0008 + grid as u64 * 0x100, + items: [ + 0xffff_fc20_c020_0000 + grid as u64 * 0x1_0000, + 0xffff_fc20_c020_4000 + grid as u64 * 0x1_0000, + 0xffff_fc20_c020_8000 + grid as u64 * 0x1_0000, + ], + grid_index: grid, + slot_index: 0, + current_write: 0, + group_number: 1, + first_submit: true, + } + } + + fn publication_hash(publication: &QueuePublication) -> u64 { + let mut value = 0xcbf2_9ce4_8422_2325u64; + let mut add = |address: u64, bytes: &[u8]| { + for byte in address.to_le_bytes() { + value ^= byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + for byte in (bytes.len() as u32).to_le_bytes() { + value ^= byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + for byte in bytes { + value ^= *byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + }; + for entry in &publication.item_entries { + add(entry.address, &entry.bytes); + } + add(publication.event.address, &publication.event.bytes); + add( + publication.queue_write.address, + &publication.queue_write.bytes, + ); + add(publication.ring_slot.address, &publication.ring_slot.bytes); + add(publication.producer.address, &publication.producer.bytes); + value + } + + #[test] + fn paired_publication_matches_m1n1_9165484() { + let publication = build_paired_queue_publication( + publication_input(RenderDescriptorKind::Tiling), + publication_input(RenderDescriptorKind::Fragment), + ) + .unwrap(); + assert_eq!(publication.tiling.write_after, 3); + assert_eq!(publication.fragment.write_after, 3); + assert_eq!(publication_hash(&publication.tiling), 0x8d2e_1298_cc1d_39c7); + assert_eq!( + publication_hash(&publication.fragment), + 0x75e1_3fd1_c48e_51ae + ); + assert_eq!( + publication.tiling.ring_slot.bytes[0x14..0x18], + 0x0100_0003u32.to_le_bytes() + ); + assert_eq!( + publication.fragment.ring_slot.bytes[0x14..0x18], + 0x0101_0003u32.to_le_bytes() + ); + assert_eq!(publication.tiling.producer.bytes, 1u32.to_le_bytes()); + assert_eq!(publication.fragment.producer.bytes, 1u32.to_le_bytes()); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/render_passthrough.rs b/drivers/gpu/drm/asahi/g18/fw/render_passthrough.rs new file mode 100644 index 0000000000000..e219cee8d5dc6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/render_passthrough.rs @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Byte map from the native processRenderSetup slice, not a complete descriptor. +//! Source: the native render passthrough map, version 3. +//! Unknown bytes return None; callers must keep them open in the ledger. +//! Gears uses the common body but its parser envelope needs separate admission. +#![allow(dead_code)] +// (descriptor offset, segment offset, extent, byte mask); mask 0 means zero. +const FIELDS: &[(usize, usize, usize, u8)] = &[ + (0x440, 0x71c, 4, 255), + (0x4e0, 0x480, 8, 255), + (0x4e8, 0x478, 8, 255), + (0x4f0, 0x788, 8, 255), + (0x4f8, 0x8b0, 24, 255), + (0x510, 0x488, 8, 255), + (0x518, 0x398, 128, 255), + (0x598, 0x7f0, 128, 255), + (0x618, 0x418, 48, 255), + (0x648, 0x790, 48, 255), + (0x678, 0x490, 40, 255), + (0x6a0, 0x4d8, 8, 255), + (0x6a8, 0x870, 8, 255), + (0x6b0, 0x500, 8, 255), + (0x6b8, 0x528, 8, 255), + (0x6c0, 0x880, 8, 255), + (0x6c8, 0x4b8, 8, 255), + (0x6d0, 0x4e0, 8, 255), + (0x6d8, 0x890, 8, 255), + (0x6e0, 0x508, 8, 255), + (0x6e8, 0x530, 8, 255), + (0x6f0, 0x898, 8, 255), + (0x6f8, 0x4c0, 8, 255), + (0x700, 0x4e8, 8, 255), + (0x708, 0x510, 8, 255), + (0x710, 0x538, 8, 255), + (0x718, 0x4c8, 8, 255), + (0x720, 0x4f0, 8, 255), + (0x728, 0x878, 8, 255), + (0x730, 0x4d0, 8, 255), + (0x738, 0x4f8, 8, 255), + (0x740, 0x518, 8, 255), + (0x748, 0x540, 8, 255), + (0x750, 0x888, 8, 255), + (0x758, 0x520, 8, 255), + (0x760, 0x548, 8, 255), + (0x768, 0x570, 8, 255), + (0x770, 0x550, 32, 255), + (0x790, 0x458, 32, 255), + (0x7b0, 0x7d0, 16, 255), + (0x7c0, 0x448, 12, 255), + (0x7cc, 0x7c0, 12, 255), + (0x7d8, 0x700, 8, 255), + (0x7e8, 0x590, 8, 255), + (0x7f8, 0x598, 4, 255), + (0x818, 0x717, 1, 1), + (0x830, 0x578, 8, 255), + (0x838, 0x8d4, 1, 1), + (0x8a0, 0x8dc, 1, 1), + (0x8a4, 0x8d8, 4, 255), + (0x8a8, 0x8e0, 8, 255), + (0x8c0, 0x708, 4, 255), + (0x8c4, 0x70d, 1, 1), + (0x8c5, 0x8e8, 1, 1), + (0x8c6, 0x8a0, 5, 1), + (0x8cd, 0x710, 2, 1), + (0x8d0, 0x713, 1, 1), + (0x8d1, 0x716, 1, 1), + (0x8d2, 0x8a5, 2, 1), + (0x970, 0x8f8, 12, 255), + (0x981, 0x8ea, 1, 255), + (0x982, 0x714, 2, 1), + (0xac8, 0x8d0, 4, 255), + (0xaf0, 0x580, 4, 255), + (0xaf8, 0x588, 8, 255), + (0xb18, 0x738, 48, 255), + (0xb58, 0x768, 28, 255), + (0xfd8, 0x320, 4, 255), + (0x1058, 0xc8, 48, 255), + (0x1088, 0x2b8, 40, 255), + (0x10b0, 0x2e8, 20, 255), + (0x10c8, 0xf8, 112, 255), + (0x1138, 0x180, 12, 255), + (0x1158, 0x2ac, 12, 255), + (0x1178, 0x30e, 1, 1), + (0x1179, 0x0, 3, 0), + (0x1180, 0x168, 8, 255), + (0x11a1, 0x305, 3, 1), + (0x11a5, 0x309, 1, 1), + (0x11a6, 0x8e8, 1, 1), + (0x11a7, 0x30a, 2, 1), + (0x11aa, 0x311, 1, 1), + (0x12d4, 0x30c, 2, 1), + (0x1410, 0x300, 4, 255), + (0x1550, 0x170, 4, 255), + (0x1558, 0x178, 8, 255), + (0x1590, 0x348, 48, 255), + (0x15d0, 0x378, 28, 255), + (0x15f1, 0x313, 1, 1), + (0x1660, 0x312, 1, 1), +]; + +pub(crate) fn field(command: &[u8], descriptor: usize, width: usize) -> Option { + if !matches!(command.len(), 0xac0 | 0xad8) || width == 0 || width > 8 { return None; } + let mut value = 0u64; + for byte in 0..width { + let offset = descriptor.checked_add(byte)?; + let &(dest, source, count, mask) = FIELDS.iter().find(|&&(dest, _, count, _)| + offset >= dest && offset - dest < count)?; + let _ = count; + let v = if mask == 0 { 0 } else { *command.get(source + offset - dest)? & mask }; + value |= u64::from(v) << (byte * 8); + } + Some(value) +} + +/// Only the empirically mapped userspace bytes. Unknown prepare-time fields +/// are counted, returned as zero for dry-run output, and MUST block admission. +pub(crate) fn capture_fields(command: &[u8]) -> (crate::g18::fw::render_command::G18RenderCommandInputs, usize) { + use crate::g18::fw::render_command::*; + fn read(c: &[u8], offset: usize, bytes: usize, open: &mut usize) -> u64 { + match field(c, offset, bytes) { Some(v) => v, None => { *open += 1; 0 } } + } + let mut open = 0; + let mut inputs = G18RenderCommandInputs::default(); + inputs.ta.d12d0 = read(command, 0x12d0, 4, &mut open) as u32; + inputs.ta.d12c0 = read(command, 0x12c0, 4, &mut open) as u32; + inputs.ta.d12c8 = read(command, 0x12c8, 8, &mut open) as u64; + inputs.ta.d11b8 = read(command, 0x11b8, 8, &mut open) as u64; + inputs.ta.d11d8 = read(command, 0x11d8, 8, &mut open) as u64; + inputs.ta.d1068 = read(command, 0x1068, 8, &mut open) as u64; + inputs.ta.d1070 = read(command, 0x1070, 8, &mut open) as u64; + inputs.ta.d10f8 = read(command, 0x10f8, 8, &mut open) as u64; + inputs.ta.d08a0 = read(command, 0x8a0, 1, &mut open) as u8; + inputs.ta.d08a4 = read(command, 0x8a4, 4, &mut open) as u32; + inputs.ta.d08a8 = read(command, 0x8a8, 4, &mut open) as u32; + inputs.ta.d08ac = read(command, 0x8ac, 4, &mut open) as u32; + inputs.ta.d08b0 = read(command, 0x8b0, 8, &mut open) as u64; + inputs.ta.d1418 = core::array::from_fn(|i| read(command, 0x1418 + i * 8, 8, &mut open)); + inputs.ta.d1428 = read(command, 0x1428, 8, &mut open) as u64; + inputs.ta.d1430 = read(command, 0x1430, 8, &mut open) as u64; + inputs.ta.d1474 = read(command, 0x1474, 1, &mut open) as u8; + inputs.ta.d11a4 = read(command, 0x11a4, 1, &mut open) as u8; + inputs.ta.d11a7 = read(command, 0x11a7, 1, &mut open) as u8; + inputs.fragment.d158 = read(command, 0x158, 8, &mut open) as u64; + inputs.fragment.d150 = read(command, 0x150, 4, &mut open) as u32; + inputs.fragment.d980_is_2 = read(command, 0x980, 4, &mut open) == 2; + inputs.fragment.d08cf = read(command, 0x8cf, 1, &mut open) as u8; + inputs.fragment.dbbc = read(command, 0xbbc, 1, &mut open) as u8; + inputs.fragment.d08d8 = read(command, 0x8d8, 8, &mut open) as u64; + inputs.fragment.d08f0 = read(command, 0x8f0, 8, &mut open) as u64; + inputs.fragment.d08f8 = read(command, 0x8f8, 4, &mut open) as u32; + inputs.fragment.d0900 = read(command, 0x900, 4, &mut open) as u32; + inputs.fragment.d0908 = read(command, 0x908, 4, &mut open) as u32; + inputs.fragment.d0910 = core::array::from_fn(|i| read(command, 0x910 + i * 8, 8, &mut open)); + inputs.fragment.d0920 = read(command, 0x920, 8, &mut open) as u64; + inputs.fragment.d0930 = read(command, 0x930, 4, &mut open) as u32; + inputs.fragment.d04e0 = read(command, 0x4e0, 8, &mut open) as u64; + inputs.fragment.d04e8 = read(command, 0x4e8, 8, &mut open) as u64; + inputs.fragment.d04f0 = read(command, 0x4f0, 8, &mut open) as u64; + inputs.fragment.d0510 = read(command, 0x510, 8, &mut open) as u64; + inputs.fragment.d0518 = core::array::from_fn(|i| core::array::from_fn(|j| read(command, 0x518 + i * 16 + j * 8, 8, &mut open))); + inputs.fragment.d0690 = read(command, 0x690, 8, &mut open) as u64; + inputs.fragment.d0698 = read(command, 0x698, 8, &mut open) as u64; + inputs.fragment.d06a0 = core::array::from_fn(|i| read(command, 0x6a0 + i * 8, 8, &mut open)); + inputs.fragment.d06b0 = read(command, 0x6b0, 8, &mut open) as u64; + inputs.fragment.d06b8 = core::array::from_fn(|i| read(command, 0x6b8 + i * 8, 8, &mut open)); + inputs.fragment.d06d8 = read(command, 0x6d8, 8, &mut open) as u64; + inputs.fragment.d06f0 = read(command, 0x6f0, 8, &mut open) as u64; + inputs.fragment.d06f8 = read(command, 0x6f8, 8, &mut open) as u64; + inputs.fragment.d0708 = read(command, 0x708, 8, &mut open) as u64; + inputs.fragment.d0728 = read(command, 0x728, 8, &mut open) as u64; + inputs.fragment.d0730 = read(command, 0x730, 8, &mut open) as u64; + inputs.fragment.d0750 = read(command, 0x750, 8, &mut open) as u64; + inputs.fragment.d0758 = read(command, 0x758, 8, &mut open) as u64; + inputs.fragment.d0770 = core::array::from_fn(|i| read(command, 0x770 + i * 8, 8, &mut open)); + inputs.fragment.d0780 = core::array::from_fn(|i| read(command, 0x780 + i * 8, 8, &mut open)); + inputs.fragment.d0790 = core::array::from_fn(|i| read(command, 0x790 + i * 8, 8, &mut open)); + inputs.fragment.d07b0 = core::array::from_fn(|i| read(command, 0x7b0 + i * 8, 8, &mut open)); + inputs.fragment.d07c0 = read(command, 0x7c0, 8, &mut open) as u64; + inputs.fragment.d07c8 = read(command, 0x7c8, 4, &mut open) as u32; + inputs.fragment.d07cc = read(command, 0x7cc, 8, &mut open) as u64; + inputs.fragment.d07d4 = read(command, 0x7d4, 4, &mut open) as u32; + inputs.fragment.d07d8 = read(command, 0x7d8, 8, &mut open) as u64; + inputs.fragment.d07e8 = read(command, 0x7e8, 8, &mut open) as u64; + inputs.fragment.d07f8 = read(command, 0x7f8, 4, &mut open) as u32; + inputs.fragment.d04d0 = read(command, 0x4d0, 4, &mut open) as u32; + inputs.fragment.d08c4 = read(command, 0x8c4, 1, &mut open) as u8; + inputs.fragment.dc98 = read(command, 0xc98, 4, &mut open) as u32; + inputs.fragment.dc9c = read(command, 0xc9c, 1, &mut open) as u8; + inputs.fragment.d08a0 = read(command, 0x8a0, 1, &mut open) as u8; + inputs.fragment.d08a4 = read(command, 0x8a4, 4, &mut open) as u32; + inputs.fragment.d08a8 = read(command, 0x8a8, 4, &mut open) as u32; + inputs.fragment.d08ac = read(command, 0x8ac, 4, &mut open) as u32; + inputs.fragment.dad0 = core::array::from_fn(|i| read(command, 0xad0 + i * 8, 8, &mut open)); + inputs.fragment.dae0 = read(command, 0xae0, 8, &mut open) as u64; + inputs.fragment.dae8 = read(command, 0xae8, 8, &mut open) as u64; + inputs.fragment.d08b0 = core::array::from_fn(|i| read(command, 0x8b0 + i * 8, 8, &mut open)); + inputs.fragment.d08c5_flags = [0x8c5, 0x8c6, 0x8c7, 0x8cb].map(|o| read(command, o, 1, &mut open) as u8); + inputs.fragment.d08cd = read(command, 0x8cd, 1, &mut open) as u8; + (inputs, open) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/rtkit.rs b/drivers/gpu/drm/asahi/g18/fw/rtkit.rs new file mode 100644 index 0000000000000..e7fbe94188167 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/rtkit.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 RTKit endpoint numbers and the two firmware messages the live boot sends. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. + +pub(crate) const ENDPOINT_GFX_MESSAGES: u8 = 0x20; + +pub(crate) const ENDPOINT_GFX_INTERRUPTS: u8 = 0x21; + +const INITDATA_TAG: u64 = 0x0081_0000_0000_0000; + +const INITDATA_ADDRESS_MASK: u64 = 0x0000_0fff_ffff_ffff; + +const INITDATA_ADDRESS_PREFIX: u64 = 0xffff_f000_0000_0000; + +/// `AGXArmFirmware::powerTransitionEnd(AGFIFirmwareRole, bool)` closes the GPU +/// power-on transition by sending this note to EACH firmware role. +/// +/// KEXT_PROVEN, 25G83 @0xfffffe00088936a4. Both send sites are identical: +/// +/// ldr x0, [x19, #6616] / [x19, #6672] ; the two role objects +/// ldr x8, [obj + 0x8a8] ; sendAsyncNoteToFirmware +/// mov x1, #0x89000000000000 ; tag 0x89, payload 0 +/// mov w2, #0x1 ; gated=true +/// blraa x8, x16 +/// +/// MEASURED on hardware (m1n1 hypervisor capture): it is the LAST +/// EP-0x21 message of bring-up, sent once per role right after the initdata +/// handshake completes and 11.4 seconds before the first work doorbell: +/// +/// 57.95ms role0 -> ep=0x21 0x0089000000000000 +/// 59.31ms role1 -> ep=0x21 0x0089000000000000 +/// ... +/// 11425.12ms role0 -> ep=0x21 0x0083000000000000 <- first doorbell +/// +/// This driver has never sent it. Its EP-0x21 vocabulary was 0x81 initdata, +/// 0x83 doorbell, 0x84 device control and 0x86 recovery inspection only. +const POWER_TRANSITION_END_TAG: u64 = 0x0089_0000_0000_0000; + +/// Encode the role's power-transition-end note. The payload is zero in every +/// captured instance and the kext materialises the bare tag with a single +/// `movz x1, #0x89, lsl #48`. +pub(crate) const fn encode_power_transition_end() -> u64 { + POWER_TRANSITION_END_TAG +} + +/// Refusals made by the evidence-backed codec. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum WireError { + UnexpectedEndpointSet, + UnsupportedInterface, + AddressOutOfRange, + InvalidInitdataTag, + InvalidRuntimeType, + QueueIdOutOfRange, + UnmodeledBits, +} + +/// Encode the G17 init-data handoff exactly as the recovered host does. +/// +/// The host instruction truncates to 44 bits. Linux instead rejects an +/// address unless it is already in low-44-bit or firmware-canonical form. +pub(crate) const fn encode_initdata_doorbell(firmware_va: u64) -> Result { + let high = firmware_va & !INITDATA_ADDRESS_MASK; + if high != 0 && high != INITDATA_ADDRESS_PREFIX { + return Err(WireError::AddressOutOfRange); + } + + Ok(INITDATA_TAG | (firmware_va & INITDATA_ADDRESS_MASK)) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/scheduler.rs b/drivers/gpu/drm/asahi/g18/fw/scheduler.rs new file mode 100644 index 0000000000000..e4c793da78563 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/scheduler.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Command-specific G18 / 26A428 scheduler record layout. +//! +//! These are the 0x100-stride records referenced by compute command +0x10, +//! distinct from the queue's 0x200-stride scheduler entries. Native evidence: +//! AGXCommandBuffer::init 0x8be76b0..0x8be7a54 and +//! nextCommandBufferState 0x8be65bc..0x8be6604 / 0x8be668c..0x8be66f0. + +#![allow(dead_code)] + +pub(crate) const G18_SCHEDULER_RECORD_COUNT: usize = 36; +pub(crate) const G18_SCHEDULER_RECORD_STRIDE: usize = 0x100; +pub(crate) const G18_SCHEDULER_RECORD_EXTENT: usize = 0xc8; +pub(crate) const G18_SCHEDULER_RECORD_BYTES: usize = + G18_SCHEDULER_RECORD_COUNT * G18_SCHEDULER_RECORD_STRIDE; +pub(crate) const G18_SCHEDULER_COUNTER_BYTES: usize = G18_SCHEDULER_RECORD_COUNT * 4; +/// Native G18 channel initialization 0x8c45470/0x8c45488 sets device +0x6f8. +pub(crate) const G18_SCHEDULER_DEFAULT_THRESHOLD: u32 = 80; + +/// Initialize the full native-sized backing, before publishing any pointer. +/// The counter backing is a separately owned, zeroed allocation. +pub(crate) fn initialize_g18_scheduler_records( + out: &mut [u8; G18_SCHEDULER_RECORD_BYTES], + counter_base: u64, +) -> Option<()> { + if counter_base == 0 || counter_base & 3 != 0 { + return None; + } + counter_base.checked_add(G18_SCHEDULER_COUNTER_BYTES as u64)?; + out.fill(0); + for index in 0..G18_SCHEDULER_RECORD_COUNT { + let start = index * G18_SCHEDULER_RECORD_STRIDE; + out[start..start + 8].copy_from_slice(&(counter_base + index as u64 * 4).to_le_bytes()); + } + Some(()) +} + +/// Reset an unpublished record exactly as the native selected-slot reset. +/// Only the first 0xc8 bytes are reset; the native 0x100 stride tail survives. +/// The caller separately zeros the submitted counter. Never call this while +/// firmware can still own the record or its completion/list fields. +pub(crate) fn prepare_g18_scheduler_record( + out: &mut [u8; G18_SCHEDULER_RECORD_STRIDE], + submitted_counter: u64, + sequence: u32, + fallback_threshold: u32, + override_threshold: u32, + cap: u32, +) -> Option<()> { + if submitted_counter == 0 || submitted_counter & 3 != 0 { + return None; + } + submitted_counter.checked_add(4)?; + let threshold = if override_threshold == 0 { + fallback_threshold + } else { + override_threshold + }; + let threshold = if cap == 0 { + threshold + } else { + threshold.min(cap) + }; + out[..G18_SCHEDULER_RECORD_EXTENT].fill(0); + out[..8].copy_from_slice(&submitted_counter.to_le_bytes()); + out[8..12].copy_from_slice(&sequence.to_le_bytes()); + out[0x10..0x14].copy_from_slice(&threshold.to_le_bytes()); + Some(()) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/sksm.rs b/drivers/gpu/drm/asahi/g18/fw/sksm.rs new file mode 100644 index 0000000000000..d8c082230f433 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/sksm.rs @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8142/25G76 HAL300 SKSM entry and tag-14 serialization. +//! +//! The M5 host clears 0x200 bytes and writes two-bit dependency types at +//! +0x180. Neo's 0x180-byte entry cannot represent this data. These pure +//! builders do not register queues, publish producers or ring a doorbell. +//! Callers must retain every referenced object and implement the M5 lifetime +//! and completion protocol before any generated bytes reach firmware. + +pub(crate) const M5_SKSM_ENTRY_BYTES: usize = 0x200; +pub(crate) const M5_SKSM_ENTRY_COUNT: u32 = 256; + +/// Logical channel selector measured in native TA, 3D and CL tag-14 records. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum M5DataMaster { + Tiling = 0, + Fragment = 1, + Compute = 2, +} +const STAMP_MASK: u64 = (1 << 40) - 1; +const PACKED_ADDRESS_MASK: u64 = (1 << 38) - 1; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5SksmError { + QueueId, + Priority, + Stamp, + QueueFull, + Dependencies, + DependencyType, + Address, + Mcache, + Rce, + Qos, + AddCount, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmDependency { + pub(crate) queue_id: u8, + /// Already converted to the hardware's 32-bit generation / 8-bit index. + pub(crate) stamp: u64, + pub(crate) kind: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmRce { + pub(crate) address: u64, + pub(crate) tag: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmMcache { + pub(crate) address: u64, + pub(crate) index: u8, + pub(crate) count: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmQueue { + pub(crate) id: u8, + /// Host queue +0x1c, not the outer firmware channel's priority. + pub(crate) priority: u8, + pub(crate) current_stamp: u64, + pub(crate) submitted: u32, + pub(crate) previous_stamp: Option, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmOperands<'a> { + /// Explicit dependencies already resolved by their owning event machine. + pub(crate) barriers: &'a [M5SksmDependency], + /// Native convertEventMachineTSToHWTS(command) - 1, for first publication. + /// This must be grounded in the queue's initialized completion state. + pub(crate) first_predecessor: u64, + pub(crate) implicit_dependency_type: u8, + pub(crate) mcache: Option, + pub(crate) flag_bit1: bool, + pub(crate) flag_bit61: bool, + pub(crate) flag_bit63: bool, + pub(crate) payload: [u64; 2], + pub(crate) qos_byte_32: u8, + pub(crate) qos_class_40: u8, + pub(crate) qos_word_48: u16, + pub(crate) event_mask: [u64; 4], + pub(crate) rce_kind: u8, + pub(crate) rce: [M5SksmRce; 4], + /// Native encoder's scalar x5 argument, copied unmodified to +0x178. + pub(crate) auxiliary: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PreparedM5SksmEntry { + pub(crate) bytes: [u8; M5_SKSM_ENTRY_BYTES], + pub(crate) offset: u32, + /// May only replace host state after entry storage and a dmb ish. + pub(crate) queue_after: M5SksmQueue, +} + +fn check_queue(id: u8, priority: u8, stamp: u64) -> Result<(), M5SksmError> { + if id > 0x7f { + return Err(M5SksmError::QueueId); + } + if priority > 3 { + return Err(M5SksmError::Priority); + } + if stamp & !STAMP_MASK != 0 { + return Err(M5SksmError::Stamp); + } + Ok(()) +} + +fn packed_address(address: u64) -> Result { + if address & 31 != 0 || address >> 5 > PACKED_ADDRESS_MASK { + return Err(M5SksmError::Address); + } + Ok(address >> 5) +} + +fn put(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Recovered from M5 encodeKickEntry at 25G76 0xfffffe00089e750c and its +/// encodeDependencyList at 0xfffffe00089e781c. Geometry is deliberately fixed +/// to the independently captured 256 x 512-byte M5 queues. +pub(crate) fn prepare_m5_sksm_entry( + queue: M5SksmQueue, + operands: M5SksmOperands<'_>, +) -> Result { + check_queue(queue.id, queue.priority, queue.current_stamp)?; + if queue.submitted >= M5_SKSM_ENTRY_COUNT { + return Err(M5SksmError::QueueFull); + } + // Native always appends one implicit predecessor after explicit barriers. + if operands.barriers.len() >= 32 { + return Err(M5SksmError::Dependencies); + } + if operands.qos_class_40 > 31 { + return Err(M5SksmError::Qos); + } + if operands.rce_kind > 3 || operands.rce.iter().any(|r| r.tag > 0x7f) { + return Err(M5SksmError::Rce); + } + let mut bytes = [0; M5_SKSM_ENTRY_BYTES]; + let mut header = (queue.current_stamp << 2) + | ((queue.id as u64) << 42) + | ((queue.priority as u64) << 59) + | ((operands.flag_bit1 as u64) << 1) + | ((operands.flag_bit61 as u64) << 61) + | ((operands.flag_bit63 as u64) << 63); + if let Some(mcache) = operands.mcache { + if mcache.index > 63 || mcache.count == 0 || mcache.count > 64 { + return Err(M5SksmError::Mcache); + } + let packed = packed_address(mcache.address)?; + put( + &mut bytes, + 8, + (mcache.index as u64) | (packed << 6) | ((mcache.count as u64 - 1) << 54), + ); + header |= 1 << 58; + } + put(&mut bytes, 0, header); + put(&mut bytes, 0x10, operands.payload[0]); + put(&mut bytes, 0x18, operands.payload[1]); + let predecessor = M5SksmDependency { + queue_id: queue.id, + stamp: queue.previous_stamp.unwrap_or(operands.first_predecessor), + kind: operands.implicit_dependency_type, + }; + let mut kinds = 0; + for (i, dependency) in operands + .barriers + .iter() + .chain(core::iter::once(&predecessor)) + .enumerate() + { + check_queue(dependency.queue_id, 0, dependency.stamp)?; + if dependency.kind > 3 { + return Err(M5SksmError::DependencyType); + } + put( + &mut bytes, + 0x28 + i * 8, + dependency.stamp | ((dependency.queue_id as u64) << 40), + ); + kinds |= (dependency.kind as u64) << (i * 2); + } + let count = operands.barriers.len() + 1; + let valid = if count == 32 { + u32::MAX + } else { + (1u32 << count) - 1 + }; + put( + &mut bytes, + 0x20, + valid as u64 + | ((operands.qos_byte_32 as u64) << 32) + | ((operands.qos_class_40 as u64) << 40) + | ((operands.qos_word_48 as u64) << 48), + ); + put(&mut bytes, 0x180, kinds); + for (i, mask) in operands.event_mask.into_iter().enumerate() { + put(&mut bytes, 0x128 + i * 8, mask); + } + bytes[0x148] = operands.rce_kind; + for (i, rce) in operands.rce.into_iter().enumerate() { + put( + &mut bytes, + 0x150 + i * 8, + packed_address(rce.address)? | ((rce.tag as u64) << 43), + ); + } + put(&mut bytes, 0x178, operands.auxiliary); + Ok(PreparedM5SksmEntry { + bytes, + offset: (queue.current_stamp as u32 & 255) * M5_SKSM_ENTRY_BYTES as u32, + queue_after: M5SksmQueue { + current_stamp: (queue.current_stamp + 1) & STAMP_MASK, + submitted: queue.submitted + 1, + previous_stamp: Some(queue.current_stamp), + ..queue + }, + }) +} + +/// Native CL add-kicks record at the captured submitBuffer publication site. +/// This is a packed 21-byte command, unrelated to the outer 24-byte entry. +pub(crate) fn encode_m5_sksm_add_kicks( + queue_id: u8, + priority: u8, + first_stamp: u64, + count: u16, +) -> Result<[u8; 21], M5SksmError> { + encode_m5_sksm_add_kicks_for( + M5DataMaster::Compute, + queue_id, + priority, + first_stamp, + count, + ) +} + +/// Encode a retained queue's stage explicitly; graphics never uses CL's DM2. +pub(crate) fn encode_m5_sksm_add_kicks_for( + data_master: M5DataMaster, + queue_id: u8, + priority: u8, + first_stamp: u64, + count: u16, +) -> Result<[u8; 21], M5SksmError> { + check_queue(queue_id, priority, first_stamp)?; + if count == 0 || count > 256 { + return Err(M5SksmError::AddCount); + } + let mut bytes = [0; 21]; + bytes[..4].copy_from_slice(&14u32.to_le_bytes()); + bytes[4..6].copy_from_slice(&(queue_id as u16).to_le_bytes()); + bytes[6..8].copy_from_slice(&count.to_le_bytes()); + bytes[8] = priority; + bytes[9..17].copy_from_slice(&first_stamp.to_le_bytes()); + bytes[17..21].copy_from_slice(&(data_master as u32).to_le_bytes()); + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn native_graphics_add_kicks_records_match_both_stages() { + let ta = [ + 14, 0, 0, 0, 10, 0, 1, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + let fragment = [ + 14, 0, 0, 0, 11, 0, 1, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + ]; + assert_eq!( + encode_m5_sksm_add_kicks_for(M5DataMaster::Tiling, 10, 2, 3, 1).unwrap(), + ta + ); + assert_eq!( + encode_m5_sksm_add_kicks_for(M5DataMaster::Fragment, 11, 2, 3, 1).unwrap(), + fragment + ); + assert_eq!( + encode_m5_sksm_add_kicks(11, 2, 3, 1).unwrap(), + encode_m5_sksm_add_kicks_for(M5DataMaster::Compute, 11, 2, 3, 1).unwrap() + ); + } + + fn queue() -> M5SksmQueue { + M5SksmQueue { + id: 11, + priority: 2, + current_stamp: 0x10, + submitted: 0, + previous_stamp: Some(15), + } + } + + fn operands() -> M5SksmOperands<'static> { + M5SksmOperands { + barriers: &[], + first_predecessor: 0, + implicit_dependency_type: 1, + mcache: None, + flag_bit1: false, + flag_bit61: false, + flag_bit63: false, + payload: [0xffff_fc20_c040_0000, 0xffff_fc20_c000_0840], + qos_byte_32: 3, + qos_class_40: 8, + qos_word_48: 0xffff, + event_mask: [0, 0, 8, 0], + rce_kind: 0, + rce: [M5SksmRce { address: 0, tag: 0 }; 4], + auxiliary: 0x3f_ffff_ffff_ffff, + } + } + + fn word(bytes: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(bytes[offset..offset + 8].try_into().unwrap()) + } + + #[test] + fn dependency_types_survive_beyond_the_neo_entry_extent() { + let barriers = [M5SksmDependency { + queue_id: 7, + stamp: 21, + kind: 2, + }; 31]; + let mut op = operands(); + op.barriers = &barriers; + op.implicit_dependency_type = 3; + let p = prepare_m5_sksm_entry(queue(), op).unwrap(); + assert_eq!(word(&p.bytes, 0x20) as u32, u32::MAX); + assert_eq!(word(&p.bytes, 0x180), 0xeaaa_aaaa_aaaa_aaaa); + assert_eq!(word(&p.bytes, 0x120), (11 << 40) | 15); + assert!(p.bytes[0x188..].iter().all(|b| *b == 0)); + } + + #[test] + fn queue_generation_wrap_and_full_queue_are_separate() { + let mut q = queue(); + q.current_stamp = STAMP_MASK; + q.submitted = 255; + let p = prepare_m5_sksm_entry(q, operands()).unwrap(); + assert_eq!(p.offset, 255 * 512); + assert_eq!(p.queue_after.current_stamp, 0); + assert_eq!(p.queue_after.previous_stamp, Some(STAMP_MASK)); + assert_eq!( + prepare_m5_sksm_entry(p.queue_after, operands()), + Err(M5SksmError::QueueFull) + ); + } + + #[test] + fn invalid_fields_cannot_alias_another_queue_or_address() { + let mut q = queue(); + q.id = 128; + assert_eq!( + prepare_m5_sksm_entry(q, operands()), + Err(M5SksmError::QueueId) + ); + let mut op = operands(); + op.rce[0].address = 1 << 43; + assert_eq!( + prepare_m5_sksm_entry(queue(), op), + Err(M5SksmError::Address) + ); + op.rce[0].address = 1; + assert_eq!( + prepare_m5_sksm_entry(queue(), op), + Err(M5SksmError::Address) + ); + assert_eq!( + encode_m5_sksm_add_kicks(11, 2, 0, 0), + Err(M5SksmError::AddCount) + ); + assert_eq!( + encode_m5_sksm_add_kicks(11, 2, 1 << 40, 1), + Err(M5SksmError::Stamp) + ); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/store.rs b/drivers/gpu/drm/asahi/g18/fw/store.rs new file mode 100644 index 0000000000000..9364be13b9931 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/store.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! v91 little-endian field stores for descriptor and record encoders. +//! +//! The encoders' `copy_from_slice` field writes compile to byte stores under +//! this target's strict alignment (one STRB per byte, measured in the v90 +//! module's `encode_registers`). Most fields land in write-combined +//! firmware-visible memory, where every store is a separate transaction for +//! the combining buffers. With `g17p_host_path` bit11 (WORD_STORES) a field +//! whose address is naturally aligned for a wider access is written with the +//! widest aligned stores that exactly cover it; the bytes written are +//! identical. Unaligned fields keep byte stores. Aligned word stores are +//! single-copy atomic, which byte stores are not, so no reader can observe a +//! state the byte sequence could not also have produced. +#[cfg(not(test))] +#[inline(always)] +fn word_stores() -> bool { + crate::g18::compat::host_path::enabled(crate::g18::compat::host_path::WORD_STORES) +} +#[cfg(test)] +pub(crate) static TEST_WORD_STORES: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(true); +#[cfg(test)] +fn word_stores() -> bool { TEST_WORD_STORES.load(core::sync::atomic::Ordering::Relaxed) } + +#[inline(always)] +fn store_u16(dst: &mut [u8], value: u16) { + let at = dst.as_mut_ptr(); + if at as usize & 1 == 0 { + // SAFETY: `dst` is exactly two in-bounds bytes and `at` is 2-aligned. + unsafe { at.cast::().write(value.to_le()) } + } else { + dst.copy_from_slice(&value.to_le_bytes()); + } +} + +#[inline(always)] +fn store_u32(dst: &mut [u8], value: u32) { + let at = dst.as_mut_ptr(); + if at as usize & 3 == 0 { + // SAFETY: `dst` is exactly four in-bounds bytes and `at` is 4-aligned. + unsafe { at.cast::().write(value.to_le()) } + } else if at as usize & 1 == 0 { + let (low, high) = dst.split_at_mut(2); + store_u16(low, value as u16); + store_u16(high, (value >> 16) as u16); + } else { + dst.copy_from_slice(&value.to_le_bytes()); + } +} + +/// Store a little-endian u16 at `raw[offset..offset + 2]` (panics out of +/// bounds, like the slice copy it replaces). +#[inline(always)] +pub(crate) fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + let dst = &mut raw[offset..offset + 2]; + if word_stores() { store_u16(dst, value) } else { dst.copy_from_slice(&value.to_le_bytes()) } +} + +/// Store a little-endian u32 at `raw[offset..offset + 4]`. +#[inline(always)] +pub(crate) fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + let dst = &mut raw[offset..offset + 4]; + if word_stores() { store_u32(dst, value) } else { dst.copy_from_slice(&value.to_le_bytes()) } +} + +/// Store a little-endian u64 at `raw[offset..offset + 8]`. +#[inline(always)] +pub(crate) fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + let dst = &mut raw[offset..offset + 8]; + if !word_stores() { + dst.copy_from_slice(&value.to_le_bytes()); + return; + } + let at = dst.as_mut_ptr(); + if at as usize & 7 == 0 { + // SAFETY: `dst` is exactly eight in-bounds bytes and `at` is 8-aligned. + unsafe { at.cast::().write(value.to_le()) } + } else { + let (low, high) = dst.split_at_mut(4); + store_u32(low, value as u32); + store_u32(high, (value >> 32) as u32); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/submission.rs b/drivers/gpu/drm/asahi/g18/fw/submission.rs new file mode 100644 index 0000000000000..3099ef306fadb --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/submission.rs @@ -0,0 +1,9012 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! Byte-confirmed subset of the G17 GPU submission-transport producer ABI (the +//! "SKSM producer"). +//! +//! This module is a pure codec for the driver->firmware submission path — the +//! wire form that hands a compiled command buffer to the firmware for +//! execution. It allocates no memory, starts no ASC, sends no mailbox message, +//! and touches no MMIO. It models the EP 0x21 work-submission doorbell, the +//! channel/work-queue ring producer indices, the SKSM kick word/record, and the +//! host queue-timestamp plus `addKicks` MMIO producer. It also models the exact +//! host-side reset images for `_AGFIChannelState` and +//! `AGXUncachedFWChannelMem`, the A18 Pro SKSM hardware-bridge programming +//! equation, and its four completion-queue descriptors. The provenance ledger +//! explicitly separates byte-grounded G17 fields from inherited producer +//! records whose G17 consumer offsets still need first-hand confirmation. The +//! native G17 channel-command dispatcher is also modeled for the exact A18 +//! Pro/G17P identity: it reads a little-endian u32 tag at command offset zero, +//! keeps unresolved and nested SKU meanings out of the top-level enum, and +//! separates the host's `Nop=17` name from the firmware's fatal handling of +//! that value. +//! +//! Grounding is first-hand from the on-disk M5 role macho `GFX.a010.macho` +//! (`g17s a0, SKSM Enabled`), all three pinned A18 Pro `g17p` firmware variants +//! (`a000`, `b000`, and `b100`), and the pinned G17P/G17G/G17X host kexts. The +//! native G17P channel-state ABI is admitted only for the A18 Pro identity; +//! the M5-family identities remain unsupported. +//! Firmware addresses are VAs (0xfffffc0000000000 + fileoff - 0x4000). See +//! the G17 submission notes. +//! +//! Two-signal discipline, mirroring `g17_rtkit`/`g17_boot`: +//! [`GROUNDED_SUBMISSION_PRODUCER_AVAILABLE`] states that the *codec* is real +//! and cited; [`EXECUTABLE_SUBMISSION_TRANSPORT_AVAILABLE`] states whether a +//! submission can be driven to completion offline. It is `false`: the SKSM +//! completion path is a hardware kick FIFO the firmware reads through indexed +//! MMIO scratch registers. The host publication form, runtime scratch geometry, +//! and resource-0 ordered writer are implemented. J700 has completed the +//! channel-doorbell round trip with exact compute output across a retained CL2 +//! queue, and the measured first-partial path has executed graphics through +//! the classic channel transport. Linux still needs that contract wired into +//! its runtime and completion IRQ ingress. The native kext `addKicks` path is +//! tracked independently. See [`SUBMISSION_LIVE_GAPS`]. + +#[cfg(not(test))] +use crate::g18::compat::g17p::G17SksmScratchGeometry; +#[cfg(test)] +#[path = "g17_manager.rs"] +mod g17_manager; +#[cfg(test)] +use g17_manager::G17SksmScratchGeometry; + +/// The submission producer subset is byte-grounded against the M5 firmware. +/// See [`SUBMISSION_ABI_LEDGER`] for the inherited, still-unconfirmed pieces. +pub(crate) const GROUNDED_SUBMISSION_PRODUCER_AVAILABLE: bool = true; + +/// Whether a submission can be driven to completion. `false`: see +/// [`SUBMISSION_LIVE_GAPS`]. This keeps `SUBMISSION_TRANSPORT` out of +/// `g17_boot::pre_handoff_gate` admission until completion is implemented. +pub(crate) const EXECUTABLE_SUBMISSION_TRANSPORT_AVAILABLE: bool = + SUBMISSION_LIVE_GAPS.all_proven(); + +// --------------------------------------------------------------------------- +// 1. EP 0x21 TX work-submission doorbell — byte-identical to AGX2. +// Grounded from the firmware doorbell decoder at GFX 0xfffffc00000251cc / +// the TX case 0xfffffc0000025278 (see the research note §1). +// --------------------------------------------------------------------------- + +/// GPU doorbell endpoint (gfx_ints). Matches `gpu.rs` `EP_DOORBELL`. +pub(crate) const EP_DOORBELL: u8 = 0x21; + +/// TX (channel-kick) doorbell tag, `0x83 << 48`. Matches `gpu.rs` +/// `MSG_TX_DOORBELL`. +const MSG_TX_DOORBELL: u64 = 0x83 << 48; + +/// The firmware reads a 6-bit doorbell type from bits [53:48] +/// (`ubfx x8, x1, #48, #6` @ 0x251dc). +const DOORBELL_TYPE_SHIFT: u32 = 48; +const DOORBELL_TYPE_BITS: u32 = 6; +const DOORBELL_TYPE_MASK: u64 = (1 << DOORBELL_TYPE_BITS) - 1; + +/// `0x83 & 0x3f = 3`: the channel-kick (TX) doorbell case (`b.eq 0x25278`). +pub(crate) const DOORBELL_TYPE_TX: u8 = 3; +/// `0x84 & 0x3f = 4`: the firmware-control doorbell case (`b.eq 0x25268`). +pub(crate) const DOORBELL_TYPE_FWCTL: u8 = 4; +/// `0x85 & 0x3f = 5`: the halt doorbell case. +pub(crate) const DOORBELL_TYPE_HALT: u8 = 5; + +/// TX pipe field is `msg & 0x3` (`and w8, w1, #0x3` @ 0x25278). +const DOORBELL_PIPE_MASK: u64 = 0x3; +/// TX index field is bits [4:2] (`tst x1, #0x1c` @ 0x2527c; `index << 2`). +const DOORBELL_INDEX_SHIFT: u32 = 2; +const DOORBELL_INDEX_MASK: u64 = 0x7; + +/// Well-known TX-doorbell payloads used for firmware wakeups, matching +/// `gpu.rs` `DOORBELL_KICKFW` / `DOORBELL_DEVCTRL`. +pub(crate) const DOORBELL_KICKFW: u8 = 0x10; +pub(crate) const DOORBELL_DEVCTRL: u8 = 0x11; + +/// The three GPU submission pipes. The firmware treats `pipe_type == 3` as invalid +/// (`cmp w8, #0x3 ; b.eq 0x252f4` — no such pipe). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum PipeType { + Vertex = 0, + Fragment = 1, + Compute = 2, +} + +/// Errors from the submission codec. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum SubmissionError { + RenderBarrierFuture, + PriorBarrierFuture, + RenderBarrierSequenceOverflow, + RenderBarrierMissingToken, + RenderBarrierPredecessorFailed, + /// A doorbell index outside the 3-bit [4:2] field. + DoorbellIndexOutOfRange, + /// A doorbell whose type field is not the TX (channel-kick) type. + NotATxDoorbell, + /// A doorbell that decodes to the reserved pipe value 3. + ReservedPipe, + /// A kick word with bits set outside the modeled 47-bit form. + KickWordUnmodeledBits, + /// A queue id outside the 7-bit [46:40] field / 128-slot table. + QueueIdOutOfRange, + /// A SKSM timestamp outside the 40-bit [39:0] field. + KickTimestampOutOfRange, + /// The runtime queue geometry must contain between 1 and 256 stamps. + KickStampCountOutOfRange, + /// No SKSM queue geometry is proven for this target/backend pair. + KickQueueGeometryUnconfirmed, + /// The queue stride/count/alignment/extent/seed tuple is inconsistent. + KickQueueGeometryInvalid, + /// The timestamp's low-byte slot does not exist in this queue geometry. + KickStampIndexOutOfRange, + /// A zero-byte entry stride would alias every queue entry. + KickEntryStrideZero, + /// Multiplying the slot by the runtime entry stride overflowed u32. + KickEntryOffsetOverflow, + /// The queue already has `num_stamps` entries awaiting completion. + KickQueueSubmittedFull, + /// A completion observed no submitted entries or exceeded their count. + KickCompletionUnderflow, + /// A provider-owned pre-B2 state object is smaller than one 16 KiB page. + KickB2PrimaryStateBufferTooSmall, + /// The retained queue already published its one-time pre-B2 state. + KickB2PrimaryStateAlreadyPublished, + /// A retained queue static object is smaller than its native extent. + KickRetainedStorageBufferTooSmall, + /// A CL entry has one implicit dependency and at most 31 explicit ones. + KickDependencyCountOutOfRange, + /// A CL dependency names a queue outside the seven-bit entry field. + KickDependencyQueueIdOutOfRange, + /// The MCache aperture index or count cannot be represented in its field. + KickMcacheFieldOutOfRange, + /// The MCache aperture address is unaligned or exceeds its 38-bit field. + KickMcacheAddressUnencodable, + /// The RCE kind is outside the two-bit entry field. + KickRceKindOutOfRange, + /// An RCE tag is outside its seven-bit entry field. + KickRceTagOutOfRange, + /// An RCE address is unaligned or exceeds its 38-bit entry field. + KickRceAddressUnencodable, + /// A compute descriptor-relative RCE address overflowed u64. + ComputeRceAddressOverflow, + /// A compute event counter overflowed its 64-bit entry field. + ComputeEventCounterOverflow, + /// Required first-bind command publication facts are still unresolved. + FirstBindCommandPublicationIncomplete, + /// G17P target-1 requires both retained CPU and booted RTKit owners. + RunningClientPowerTransitionUnavailable, + /// The middle QoS field is wider than the five bits copied by the host. + KickQosClassOutOfRange, + /// The runtime MMIO aperture selector or base offset overflowed u32. + KickMmioOffsetOverflow, + /// The A18 Pro SKSM queue data-master field is limited to TA/3D/CL. + KickQueueDataMasterOutOfRange, + /// The A18 Pro SKSM completion-queue selector is outside the host range. + KickQueueCompletionSelectorOutOfRange, + /// The A18 Pro SKSM entry address cannot be represented by the host mask. + KickQueueEntryAddressUnencodable, + /// The A18 Pro completion descriptor ordinal is outside its four slots. + CompletionQueueOrdinalOutOfRange, + /// A present A18 Pro completion queue cannot have zero entries. + CompletionQueueCapacityZero, + /// A ring with zero entries cannot advance a producer index. + RingSizeZero, + /// A producer index must identify an entry inside its ring. + RingIndexOutOfRange, + /// Advancing the producer would make it collide with the consumer. + RingFull, + /// G17 exposes four priority groups in its runtime channel table. + ChannelPriorityOutOfRange, + /// This native channel layout is not grounded for the selected identity. + ChannelAbiUnconfirmed, + /// The top-level channel-command tag ABI is not grounded for this target. + CommandAbiUnconfirmed, + /// A top-level channel command must expose its complete four-byte tag. + CommandHeaderTooShort, + /// A first-render status pointer overflowed while deriving the FWCtl VAs. + PartialOpeningAddressOverflow, + /// A first-render scheduler or index mapping is not exactly one 16 KiB page. + PartialOpeningPageSize, + /// A first-render outer-channel counter has bits outside the 8-bit ring index. + PartialOpeningChannelCounterOutOfRange, + /// A first-render queue head does not fit the outer slot's 16-bit field. + PartialOpeningQueueHeadOutOfRange, + /// A cold-opening support object does not match its exact encoded extent. + PartialOpeningGraphObjectSize, + /// A first-partial descriptor cannot be represented by its context locators. + PartialOpeningDescriptorLocatorUnencodable, + /// The host/firmware meaning of this in-range top-level tag is unresolved. + CommandTagUnresolved, + /// This value is not a grounded G17P top-level command tag. + CommandTagInvalid, + /// The host names this tag, but the matched firmware handles it fatally. + CommandTagInvalidForFirmware, + /// A tag-14 AddKicks command must publish at least one new kick entry. + KsmAddKicksCountZero, + /// The tag-14 AddKicks count does not fit the record's 16-bit field. + KsmAddKicksCountOutOfRange, +} + +/// Encode a work-submission doorbell exactly as `gpu.rs::run_job` does: +/// `MSG_TX_DOORBELL | pipe_type | (index << 2)`. +/// +/// `index` is the queue priority (0..=7 fits the firmware's [4:2] field). +pub(crate) const fn encode_tx_doorbell(pipe: PipeType, index: u8) -> Result { + if index as u64 > DOORBELL_INDEX_MASK { + return Err(SubmissionError::DoorbellIndexOutOfRange); + } + Ok(MSG_TX_DOORBELL | (pipe as u64) | ((index as u64) << DOORBELL_INDEX_SHIFT)) +} + +/// Host state which decides the two distinct EP 0x21 notifications around a +/// direct G17P CL submit. Activation is first-or-changed-channel state; the +/// direct kick has its own feature, suppression, and command-enable gates. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PDirectClNotificationState { + pub(crate) cached_channel_identity: u64, + pub(crate) submitted_channel_identity: u64, + pub(crate) firmware_ready: bool, + pub(crate) direct_feature: bool, + pub(crate) submission_suppressed: bool, + pub(crate) command_submission_enabled: bool, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PDirectClNotifications { + pub(crate) activation: Option, + pub(crate) direct_kick: Option, +} + +/// Submission backend identity. This keeps the T8140 G17P HAL200 hybrid +/// distinct from both classic channels and the unimplemented HAL300 family +/// without changing mature generic paths. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17ClSubmissionBackend { + Classic, + G17PHal200Hybrid, + Hal300, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClBackendConfig { + pub(crate) backend: G17ClSubmissionBackend, + pub(crate) direct_feature: bool, + pub(crate) submission_suppressed: bool, + pub(crate) command_submission_enabled: bool, +} + +pub(crate) const T8140_G17P_CL_BACKEND: G17PClBackendConfig = G17PClBackendConfig { + backend: G17ClSubmissionBackend::G17PHal200Hybrid, + direct_feature: true, + submission_suppressed: false, + command_submission_enabled: true, +}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PContractPredicateState { + Pending, + ProvenPresent, +} + +/// Dedicated FlushID classification is not sufficient for release: an absent +/// feature needs a proven represented fallback, while a present feature needs +/// its numeric allocator, lifetime, and command operands implemented. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PFlushIdImplementationState { + Pending, + ProvenAbsentFallbackUnverified, + ProvenAbsentFallbackRepresented, + ProvenPresentUnimplemented, + ProvenPresentImplemented, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PCorrelationRingImplementationState { + Pending, + ProvenInactive, + ProvenRequiredImplemented, +} + +/// Required first-bind commands and publication facts. FlushID uses represented +/// fallback zero, the command-pointer ring is implemented, and the taken +/// correlation branch is represented by a host-only nonzero job token. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PFirstBindCommandPlan { + pub(crate) config_update_required: bool, + pub(crate) entry_signal_required: bool, + pub(crate) logical_flush_id_required: bool, + pub(crate) fallback_logical_flush_id: u32, + pub(crate) flush_id_serialized_in_tag15: bool, + pub(crate) flush_id_completion_release_required: bool, + pub(crate) dedicated_flush_id: G17PFlushIdImplementationState, + pub(crate) native_command_pointer_ring: G17PContractPredicateState, + pub(crate) optional_correlation_ring: G17PCorrelationRingImplementationState, +} + +impl G17PFirstBindCommandPlan { + pub(crate) const fn hardware_release_ready(self) -> bool { + self.config_update_required + && self.entry_signal_required + && self.logical_flush_id_required + && self.fallback_logical_flush_id == 0 + && !self.flush_id_serialized_in_tag15 + && !self.flush_id_completion_release_required + && matches!( + self.dedicated_flush_id, + G17PFlushIdImplementationState::ProvenAbsentFallbackRepresented + | G17PFlushIdImplementationState::ProvenPresentImplemented + ) + && matches!( + self.native_command_pointer_ring, + G17PContractPredicateState::ProvenPresent + ) + && matches!( + self.optional_correlation_ring, + G17PCorrelationRingImplementationState::ProvenInactive + | G17PCorrelationRingImplementationState::ProvenRequiredImplemented + ) + } +} + +pub(crate) const G17P_FIRST_BIND_COMMAND_PLAN: G17PFirstBindCommandPlan = + G17PFirstBindCommandPlan { + config_update_required: true, + entry_signal_required: true, + logical_flush_id_required: true, + fallback_logical_flush_id: 0, + flush_id_serialized_in_tag15: false, + flush_id_completion_release_required: false, + dedicated_flush_id: + G17PFlushIdImplementationState::ProvenAbsentFallbackRepresented, + native_command_pointer_ring: G17PContractPredicateState::ProvenPresent, + optional_correlation_ring: + G17PCorrelationRingImplementationState::ProvenRequiredImplemented, + }; + +/// Compile-time release gate consumed by the live first-add3 boundary. Keep +/// true only because the host-only correlation token, tag-15/tag-16 pointer +/// publisher, and FlushID fallback are all represented for this profile. +pub(crate) const G17P_FIRST_BIND_DIRECT_HARDWARE_RELEASE_READY: bool = + G17P_FIRST_BIND_COMMAND_PLAN.hardware_release_ready(); + +pub(crate) const fn require_g17p_first_bind_direct_hardware_release( + plan: G17PFirstBindCommandPlan, +) -> Result<(), SubmissionError> { + if plan.hardware_release_ready() { + Ok(()) + } else { + Err(SubmissionError::FirstBindCommandPublicationIncomplete) + } +} + +/// Native target-bound active state required before TA/3D/CL submission. +/// +/// D93 role 1 is managed and non-standalone. Linux retains its started CPU, +/// mailbox, and booted RTKit owners for the complete session, which is the +/// native target-1 state. Submit performs no management power request. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PRunningClientPowerContract { + pub(crate) role_index: u8, + pub(crate) service_id: u8, + pub(crate) target: u8, + pub(crate) management_request_used: bool, + pub(crate) managed: bool, + pub(crate) standalone: bool, +} + +pub(crate) const G17P_RUNNING_CLIENT_POWER_CONTRACT: G17PRunningClientPowerContract = + G17PRunningClientPowerContract { + role_index: 1, + service_id: 0x20, + target: 1, + management_request_used: false, + managed: true, + standalone: false, + }; + +pub(crate) const fn require_g17p_running_client_power_transition( + contract: G17PRunningClientPowerContract, + retained_cpu_owner: bool, + retained_rtkit_owner: bool, +) -> Result<(), SubmissionError> { + if contract.role_index == 1 + && contract.service_id == 0x20 + && contract.target == 1 + && !contract.management_request_used + && contract.managed + && !contract.standalone + && retained_cpu_owner + && retained_rtkit_owner + { + Ok(()) + } else { + Err(SubmissionError::RunningClientPowerTransitionUnavailable) + } +} + +/// Required lifetime of the eventual target-1 active lease. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PComputePowerLeaseStep { + AcquireBeforeReadiness, + HoldThroughClaim, + HoldThroughCompletion, + ReleaseAfterCompletion, +} + +pub(crate) const G17P_COMPUTE_POWER_LEASE_ORDER: [G17PComputePowerLeaseStep; 4] = [ + G17PComputePowerLeaseStep::AcquireBeforeReadiness, + G17PComputePowerLeaseStep::HoldThroughClaim, + G17PComputePowerLeaseStep::HoldThroughCompletion, + G17PComputePowerLeaseStep::ReleaseAfterCompletion, +]; + +pub(crate) const fn g17p_first_bind_tag15_required(queue_config_dirty: bool) -> bool { + queue_config_dirty +} + +pub(crate) const fn g17p_first_bind_tag16_required( + backend: G17PClBackendConfig, + queue_config_dirty: bool, +) -> bool { + matches!(backend.backend, G17ClSubmissionBackend::G17PHal200Hybrid) + && backend.direct_feature + && queue_config_dirty +} + +/// Host-only first/repeat state for the direct CL submission contract. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PDirectClLifecycleState { + pub(crate) queue_config_dirty: bool, + pub(crate) cached_channel_identity: u64, + /// Nonzero host-only descriptor sequence used for job ownership/fencing. + pub(crate) next_correlation_token: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PDirectClLifecycle { + pub(crate) publish_config_update: bool, + pub(crate) publish_entry_signal: bool, + pub(crate) correlation_token: u64, + pub(crate) notifications: PreparedG17PDirectClNotifications, + pub(crate) next: G17PDirectClLifecycleState, +} + +/// Derive first-bind and repeat transitions without selecting publication +/// storage. First bind publishes tags 15/16; a clean repeat omits both and the +/// unchanged channel identity omits activation, while direct kick remains. +pub(crate) const fn prepare_g17p_direct_cl_lifecycle( + current: G17PDirectClLifecycleState, + submitted_channel_identity: u64, + firmware_ready: bool, + backend: G17PClBackendConfig, +) -> Result { + if current.next_correlation_token == 0 { + return Err(SubmissionError::ComputeEventCounterOverflow); + } + let next_correlation_token = match current.next_correlation_token.checked_add(1) { + Some(token) => token, + None => return Err(SubmissionError::ComputeEventCounterOverflow), + }; + let publish_config_update = g17p_first_bind_tag15_required(current.queue_config_dirty); + let publish_entry_signal = + g17p_first_bind_tag16_required(backend, current.queue_config_dirty); + let notifications = match prepare_g17p_direct_cl_notifications( + G17PDirectClNotificationState { + cached_channel_identity: current.cached_channel_identity, + submitted_channel_identity, + firmware_ready, + direct_feature: backend.direct_feature, + submission_suppressed: backend.submission_suppressed, + command_submission_enabled: backend.command_submission_enabled, + }, + ) { + Ok(notifications) => notifications, + Err(error) => return Err(error), + }; + Ok(PreparedG17PDirectClLifecycle { + publish_config_update, + publish_entry_signal, + correlation_token: current.next_correlation_token, + notifications, + next: G17PDirectClLifecycleState { + queue_config_dirty: false, + cached_channel_identity: submitted_channel_identity, + next_correlation_token, + }, + }) +} + +/// First/new compute-channel activation. Priority 2 encodes as `0x...0a`; +/// this is independent of SKSM QID 4 and outer work-channel index 10. +pub(crate) const fn encode_g17p_cl_activation(priority: u8) -> Result { + encode_tx_doorbell(PipeType::Compute, priority) +} + +/// Direct CL `kickFirmware(4, 3, false)`, byte-proven as EP 0x21 `0x...10`. +pub(crate) const fn encode_g17p_direct_cl_kick() -> u64 { + MSG_TX_DOORBELL | DOORBELL_KICKFW as u64 +} + +/// Exact J700 native EP21 lifecycle around firmware startup and the first real +/// userspace CL. Startup publishes no work or record-less device-control +/// announcement; the first changed priority-2 CL emits activation then the +/// direct kick. +pub(crate) const G17P_NATIVE_PRE_USER_EP21_MESSAGES: [u64; 0] = []; +pub(crate) const G17P_NATIVE_FIRST_USER_EP21_MESSAGES: [u64; 2] = [ + 0x0083_0000_0000_000a, + 0x0083_0000_0000_0010, +]; + +/// Pure predicate/encoding boundary. A repeat submit omits activation when the +/// cached identity already matches, but an enabled, direct, unsuppressed submit +/// still emits the separate direct kick. +pub(crate) const fn prepare_g17p_direct_cl_notifications( + state: G17PDirectClNotificationState, +) -> Result { + let activation = if state.cached_channel_identity != state.submitted_channel_identity + && state.firmware_ready + { + match encode_g17p_cl_activation(2) { + Ok(message) => Some(message), + Err(error) => return Err(error), + } + } else { + None + }; + let direct_kick = if state.direct_feature + && !state.submission_suppressed + && state.command_submission_enabled + { + Some(encode_g17p_direct_cl_kick()) + } else { + None + }; + Ok(PreparedG17PDirectClNotifications { + activation, + direct_kick, + }) +} + +/// Extract the firmware's 6-bit doorbell type from bits [53:48]. +pub(crate) const fn doorbell_type(message: u64) -> u8 { + ((message >> DOORBELL_TYPE_SHIFT) & DOORBELL_TYPE_MASK) as u8 +} + +/// Decode a TX doorbell to `(pipe_type, index)`, applying the firmware's exact +/// masks and its invalid `pipe == 3` outcome. +pub(crate) const fn decode_tx_doorbell(message: u64) -> Result<(PipeType, u8), SubmissionError> { + if doorbell_type(message) != DOORBELL_TYPE_TX { + return Err(SubmissionError::NotATxDoorbell); + } + let pipe = match message & DOORBELL_PIPE_MASK { + 0 => PipeType::Vertex, + 1 => PipeType::Fragment, + 2 => PipeType::Compute, + _ => return Err(SubmissionError::ReservedPipe), + }; + let index = ((message >> DOORBELL_INDEX_SHIFT) & DOORBELL_INDEX_MASK) as u8; + Ok((pipe, index)) +} + +/// The firmware's pending-pipe bit for a submitted pipe: `1 << pipe_type` +/// (`mov w9,#1 ; lsl w8,w9,w8` in the pinned J700 A000 type-3 handler). This +/// path is distinct from the type-4 device-control bit 17 at scheduler root +/// `0xfffffc0000177190`. +pub(crate) const fn pending_pipe_bit(pipe: PipeType) -> u32 { + 1u32 << (pipe as u32) +} + +// --------------------------------------------------------------------------- +// 2. Legacy AGX2 channel state. These constants are retained only to make the +// inherited producer explicit; they are not a native-G17 admission. +// --------------------------------------------------------------------------- + +/// AGX2 FW-owned consumer index. +pub(crate) const AGX2_CHANNEL_STATE_READ_PTR_OFFSET: u32 = 0x00; +/// AGX2 driver-owned producer index. +pub(crate) const AGX2_CHANNEL_STATE_WRITE_PTR_OFFSET: u32 = 0x20; +/// Total AGX2 `ChannelState` size. +pub(crate) const AGX2_CHANNEL_STATE_SIZE: u32 = 0x40; + +// --------------------------------------------------------------------------- +// 2a. G17 runtime channel descriptors -- native AGX3 layout. +// +// Grounded first-hand from the G17S firmware recovery scanner at GFX +// 0xfffffc000000ff8c-0xfffffc00000103dc. runtime_pointers+0x20 holds four +// priority groups of three descriptors (vertex, fragment, compute). Each +// descriptor is 0x20 bytes: the firmware dereferences cursor pointers at +// +0x00/+0x08/+0x10 and obtains the command-entry array from +0x18. It +// advances through that array in 0x18-byte records. The final descriptor +// ends at runtime_pointers+0x1a0, immediately before the independent cursor +// pair at +0x1a0/+0x1a8. +// --------------------------------------------------------------------------- + +/// First G17 channel descriptor in `runtime_pointers`. +pub(crate) const G17_CHANNEL_DESCRIPTOR_TABLE_OFFSET: u32 = 0x20; +/// Vertex, fragment, and compute descriptors per priority group. +pub(crate) const G17_CHANNEL_DESCRIPTORS_PER_PRIORITY: u32 = 3; +/// Priority groups scanned by the G17 firmware. +pub(crate) const G17_CHANNEL_PRIORITY_COUNT: u8 = 4; +/// Total descriptor count: four priorities times three submission pipes. +pub(crate) const G17_CHANNEL_DESCRIPTOR_COUNT: u32 = 12; +/// Native G17 descriptor stride. AGX2 `ChannelRing` is only 0x10 bytes. +pub(crate) const G17_CHANNEL_DESCRIPTOR_STRIDE: u32 = 0x20; +/// G17 command-entry stride used by all three pipe rings. +pub(crate) const G17_CHANNEL_ENTRY_STRIDE: u32 = 0x18; + +/// Pointer fields inside one native G17 channel descriptor. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17ChannelDescriptorLayout { + /// Pointer to the cursor advanced after an entry has been dispatched. + pub(crate) dispatch_cursor: u32, + /// Pointer to the cursor advanced after the entry range is made coherent. + pub(crate) coherency_cursor: u32, + /// Pointer to the producer cursor sampled before scanning the ring. + pub(crate) producer_cursor: u32, + /// GPU pointer to the first 0x18-byte command entry. + pub(crate) entries: u32, +} + +pub(crate) const G17_CHANNEL_DESCRIPTOR_LAYOUT: G17ChannelDescriptorLayout = + G17ChannelDescriptorLayout { + dispatch_cursor: 0x00, + coherency_cursor: 0x08, + producer_cursor: 0x10, + entries: 0x18, + }; + +/// Return the runtime-pointer byte offset of a native G17 pipe descriptor. +/// +/// The table is priority-major and pipe-minor. Refuse priority values outside +/// the four groups that the firmware scans instead of aliasing the independent +/// cursor pair at `runtime_pointers+0x1a0`. +pub(crate) const fn g17_channel_descriptor_offset( + priority: u8, + pipe: PipeType, +) -> Result { + if priority >= G17_CHANNEL_PRIORITY_COUNT { + return Err(SubmissionError::ChannelPriorityOutOfRange); + } + + let index = priority as u32 * G17_CHANNEL_DESCRIPTORS_PER_PRIORITY + pipe as u32; + Ok(G17_CHANNEL_DESCRIPTOR_TABLE_OFFSET + index * G17_CHANNEL_DESCRIPTOR_STRIDE) +} + +// --------------------------------------------------------------------------- +// 2b. G17 host channel-reset images -- native AGX3 layout. +// +// Grounded first-hand from byte-identical `AGXChannel::resetChannelState` +// implementations in the pinned G17P host at 0xfffffe00089f2a60, G17G at +// 0xfffffe0008b225ac, and G17X at 0xfffffe0008ae6a6c. The host clears all +// 0x80 bytes of `_AGFIChannelState`, then writes the fields below. Its +// separate `AGXUncachedFWChannelMem` reset touches only seven u32 fields +// through +0x63; bytes between them are deliberately preserved here. +// +// These are independent host-reset records. They do not prove that the +// inherited AGX2 ChannelState/RingState names in sections 2/3 describe the +// G17 firmware consumer, so those ledger entries remain unsupported. +// --------------------------------------------------------------------------- + +/// Full byte extent cleared by the G17 host for `_AGFIChannelState`. +pub(crate) const G17_HOST_CHANNEL_STATE_RESET_SIZE: usize = 0x80; +/// Last byte touched by the G17 host's uncached-channel reset stores. +pub(crate) const G17_UNCACHED_CHANNEL_RESET_WRITE_EXTENT: usize = 0x64; +/// The older pinned host builds share this reset layout. The live M5 25G76 +/// driver instead resets 0xc0 bytes; see `encode_m5_25g76_channel_state_reset`. +pub(crate) const G17_HOST_CHANNEL_RESET_TARGETS_CONFIRMED: bool = true; + +/// Offsets written after the host clears `_AGFIChannelState`. +/// +/// Names are intentionally structural until the firmware consumer supplies +/// first-hand semantics for every field. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17HostChannelStateResetLayout { + pub(crate) gpu_va_00: usize, + pub(crate) gpu_va_08: usize, + pub(crate) gpu_va_10: usize, + pub(crate) zero_18: usize, + pub(crate) zero_1c: usize, + pub(crate) zero_20: usize, + pub(crate) sentinel_24: usize, + pub(crate) fixed_4_28: usize, + pub(crate) sentinel_40: usize, + pub(crate) word_44: usize, + pub(crate) zero_58: usize, + pub(crate) gpu_va_70: usize, +} + +pub(crate) const G17_HOST_CHANNEL_STATE_RESET_LAYOUT: G17HostChannelStateResetLayout = + G17HostChannelStateResetLayout { + gpu_va_00: 0x00, + gpu_va_08: 0x08, + gpu_va_10: 0x10, + zero_18: 0x18, + zero_1c: 0x1c, + zero_20: 0x20, + sentinel_24: 0x24, + fixed_4_28: 0x28, + sentinel_40: 0x40, + word_44: 0x44, + zero_58: 0x58, + gpu_va_70: 0x70, + }; + +/// Runtime values copied by the host into the structural G17 reset image. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17HostChannelStateResetValues { + pub(crate) gpu_va_00: u64, + pub(crate) gpu_va_08: u64, + pub(crate) gpu_va_10: u64, + pub(crate) word_44: u32, + pub(crate) gpu_va_70: u64, +} + +/// Fields touched in `AGXUncachedFWChannelMem` during a G17 channel reset. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17UncachedChannelResetLayout { + pub(crate) zero_00: usize, + pub(crate) zero_10: usize, + pub(crate) zero_20: usize, + pub(crate) zero_30: usize, + pub(crate) zero_40: usize, + pub(crate) sentinel_50: usize, + pub(crate) ring_entry_count_60: usize, +} + +pub(crate) const G17_UNCACHED_CHANNEL_RESET_LAYOUT: G17UncachedChannelResetLayout = + G17UncachedChannelResetLayout { + zero_00: 0x00, + zero_10: 0x10, + zero_20: 0x20, + zero_30: 0x30, + zero_40: 0x40, + sentinel_50: 0x50, + ring_entry_count_60: 0x60, + }; + +fn put_u32_le(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u16_le(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64_le(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Build the exact 0x80-byte `_AGFIChannelState` reset image. +pub(crate) fn encode_g17_host_channel_state_reset( + values: G17HostChannelStateResetValues, +) -> [u8; G17_HOST_CHANNEL_STATE_RESET_SIZE] { + let mut raw = [0u8; G17_HOST_CHANNEL_STATE_RESET_SIZE]; + let layout = G17_HOST_CHANNEL_STATE_RESET_LAYOUT; + + put_u64_le(&mut raw, layout.gpu_va_00, values.gpu_va_00); + put_u64_le(&mut raw, layout.gpu_va_08, values.gpu_va_08); + put_u64_le(&mut raw, layout.gpu_va_10, values.gpu_va_10); + put_u32_le(&mut raw, layout.zero_18, 0); + put_u32_le(&mut raw, layout.zero_1c, 0); + put_u32_le(&mut raw, layout.zero_20, 0); + put_u32_le(&mut raw, layout.sentinel_24, u32::MAX); + put_u32_le(&mut raw, layout.fixed_4_28, 4); + put_u32_le(&mut raw, layout.sentinel_40, u32::MAX); + put_u32_le(&mut raw, layout.word_44, values.word_44); + put_u32_le(&mut raw, layout.zero_58, 0); + put_u64_le(&mut raw, layout.gpu_va_70, values.gpu_va_70); + raw +} + +/// M5 G17G / macOS 25G76 resets six pairs of 128-bit vectors, covering 0xc0 +/// bytes. Its firmware consumes through +0xa7. The common 0x80-byte reset +/// would omit the moved fields and the firmware-visible tail. +pub(crate) const M5_25G76_CHANNEL_STATE_RESET_SIZE: usize = 0xc0; +pub(crate) const M5_25G76_CHANNEL_FIRMWARE_READ_EXTENT: usize = 0xa8; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ChannelStateResetValues { + pub(crate) uncached_state: u64, + pub(crate) command_pointer_ring: u64, + pub(crate) gpu_va_10: u64, + pub(crate) word_48: u32, + /// Opaque host field +0xf8 copied to the unaligned firmware field +0x9c. + pub(crate) gpu_va_9c: u64, +} + +/// Exact reset stores from M5 AGXChannel::resetChannelState at +/// 0xfffffe000897d36c..0xfffffe000897d410 (AGXG17G 353.14, 25G76). +/// Uncached stores are separately shared with apply_g17_uncached_channel_reset. +/// This codec alone does not admit M5 submission or a Linux GPU runtime. +pub(crate) fn encode_m5_25g76_channel_state_reset( + values: M5ChannelStateResetValues, +) -> [u8; M5_25G76_CHANNEL_STATE_RESET_SIZE] { + let mut raw = [0u8; M5_25G76_CHANNEL_STATE_RESET_SIZE]; + put_u64_le(&mut raw, 0x00, values.uncached_state); + put_u64_le(&mut raw, 0x08, values.command_pointer_ring); + put_u64_le(&mut raw, 0x10, values.gpu_va_10); + put_u32_le(&mut raw, 0x24, u32::MAX); + put_u32_le(&mut raw, 0x28, 4); + put_u32_le(&mut raw, 0x44, u32::MAX); + put_u32_le(&mut raw, 0x48, values.word_48); + put_u64_le(&mut raw, 0x9c, values.gpu_va_9c); + raw +} + +/// Apply only the seven uncached reset stores made by the G17 host. +/// +/// The host does not clear the intervening bytes in `resetChannelState`; a +/// caller-provided image therefore stays intact outside the proven stores. +pub(crate) fn apply_g17_uncached_channel_reset( + raw: &mut [u8; G17_UNCACHED_CHANNEL_RESET_WRITE_EXTENT], + ring_entry_count: u32, +) { + let layout = G17_UNCACHED_CHANNEL_RESET_LAYOUT; + + put_u32_le(raw, layout.zero_00, 0); + put_u32_le(raw, layout.zero_10, 0); + put_u32_le(raw, layout.zero_20, 0); + put_u32_le(raw, layout.zero_30, 0); + put_u32_le(raw, layout.zero_40, 0); + put_u32_le(raw, layout.sentinel_50, u32::MAX); + put_u32_le(raw, layout.ring_entry_count_60, ring_entry_count); +} + +// --------------------------------------------------------------------------- +// 2c. A18 Pro / G17P native channel state. +// +// This is deliberately identity-scoped. The producer is the pinned G17P +// host `AGXChannel::writeChannelCommandPointer` at +// 0xfffffe00089f5a88-0xfffffe00089f5b58. The matching firmware consumers +// are `armfw_g17p.a000` 0xfffffc0000012388-0xfffffc00000124f4 and the +// resume path 0xfffffc0000012514-0xfffffc0000012690. The matching b000 +// and b100 channel blocks are byte-identical to each other and implement +// the same accesses at 0xfffffc00000122f4-0xfffffc0000012460 and +// 0xfffffc00000124a4-0xfffffc0000012664. +// +// `_AGFIChannelState` is the cached indirection record: +0x00 points to +// the uncached state, +0x08 points to the u64 command-pointer ring, and +// +0x1c holds the cached coherency cursor. `AGXUncachedFWChannelMem` holds +// u32 cursors at +0x00/+0x30/+0x40/+0x50 and a u32 ring entry count at +// +0x60. This is not AGX2's read@0/write@0x20 `ChannelState`. +// --------------------------------------------------------------------------- + +/// Silicon/host identity used to admit native G17 submission ABIs. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17ChannelAbiTarget { + /// A18 Pro: G17P host and `g17p` firmware. + A18ProG17P, + /// M5: G17G host. No native channel consumer is admitted by this pass. + M5G17G, + /// M5 Pro/Max: G17X host with `g17s` firmware. Kept separate from G17P. + M5ProMaxG17X, +} + +/// Cached `_AGFIChannelState` fields consumed by the A18 Pro firmware. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PCachedChannelStateLayout { + pub(crate) uncached_state_gpu_va: u32, + pub(crate) command_pointer_ring_gpu_va: u32, + pub(crate) coherency_cursor: u32, + pub(crate) record_size: u32, +} + +/// Uncached channel fields shared by the A18 Pro host and firmware. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PUncachedChannelStateLayout { + /// Firmware dispatch/consumer cursor. + pub(crate) consumer: u32, + /// Cursor published after the command-pointer range is made coherent. + pub(crate) coherency_shadow: u32, + /// Host producer cursor, published after a `dmb ish`. + pub(crate) producer: u32, + /// Recovery target selected instead of the accelerator-entry snapshot. + pub(crate) recovery_target: u32, + /// Runtime ring depth in 8-byte command-pointer entries. + pub(crate) ring_entry_count: u32, + pub(crate) cursor_width: u32, + pub(crate) write_extent: u32, +} + +/// Complete admitted A18 Pro channel layout. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PChannelAbiLayout { + pub(crate) cached: G17PCachedChannelStateLayout, + pub(crate) uncached: G17PUncachedChannelStateLayout, + pub(crate) command_pointer_entry_bytes: u32, + pub(crate) accelerator_target_width: u32, +} + +pub(crate) const G17P_CHANNEL_ABI: G17PChannelAbiLayout = G17PChannelAbiLayout { + cached: G17PCachedChannelStateLayout { + uncached_state_gpu_va: 0x00, + command_pointer_ring_gpu_va: 0x08, + coherency_cursor: 0x1c, + record_size: 0x80, + }, + uncached: G17PUncachedChannelStateLayout { + consumer: 0x00, + coherency_shadow: 0x30, + producer: 0x40, + recovery_target: 0x50, + ring_entry_count: 0x60, + cursor_width: 4, + write_extent: 0x64, + }, + command_pointer_entry_bytes: 8, + accelerator_target_width: 2, +}; + +/// The host executes `dmb ish` after storing the u64 command pointer and +/// before publishing the next u32 producer cursor. +pub(crate) const G17P_CHANNEL_PRODUCER_REQUIRES_DMB_ISH: bool = true; + +/// Return the native channel ABI only for the exact identity whose producer +/// and consumer were both inspected. M5-family targets remain unsupported. +pub(crate) const fn g17_channel_abi( + target: G17ChannelAbiTarget, +) -> Result { + match target { + G17ChannelAbiTarget::A18ProG17P => Ok(G17P_CHANNEL_ABI), + G17ChannelAbiTarget::M5G17G | G17ChannelAbiTarget::M5ProMaxG17X => { + Err(SubmissionError::ChannelAbiUnconfirmed) + } + } +} + +/// A host write plan for one G17P command-pointer ring publication. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PChannelWritePlan { + /// Byte offset of the u64 slot written before the barrier. + pub(crate) entry_byte_offset: u64, + /// u32 producer value published after the barrier. + pub(crate) next_producer: u32, +} + +/// Validate and calculate the write performed by the G17P host producer. +/// +/// This pure helper does not execute the required barrier; callers must store +/// the u64 command GPU VA, execute `dmb ish`, then store `next_producer` to +/// uncached offset +0x40. +pub(crate) const fn prepare_g17p_channel_write( + target: G17ChannelAbiTarget, + producer: u32, + consumer: u32, + ring_entry_count: u32, +) -> Result { + match g17_channel_abi(target) { + Ok(_) => {} + Err(error) => return Err(error), + } + + if consumer >= ring_entry_count { + return Err(if ring_entry_count == 0 { + SubmissionError::RingSizeZero + } else { + SubmissionError::RingIndexOutOfRange + }); + } + + let next_producer = match ring_next(producer, ring_entry_count) { + Ok(next) => next, + Err(error) => return Err(error), + }; + if next_producer == consumer { + return Err(SubmissionError::RingFull); + } + + Ok(G17PChannelWritePlan { + entry_byte_offset: producer as u64 * G17P_CHANNEL_ABI.command_pointer_entry_bytes as u64, + next_producer, + }) +} + +// --------------------------------------------------------------------------- +// 2d. Proven G17P first-partial graphics publication. +// +// The measured J700 first-partial source contract executes graphics through +// the native partial graph. It publishes a one-record scheduler page and +// an eight-group primary index, presents five primary status fields before +// control 0x84, then exposes 3D before TA and sends one EP 0x21/0x83 kick +// for transport pair 0 on channel 0. This is the classic channel path and +// remains separate from the kext's SKSM queue-entry plus `addKicks` path. +// --------------------------------------------------------------------------- + +pub(crate) const G17P_PARTIAL_OPENING_PAGE_SIZE: usize = 0x4000; +/// Firmware-high zero page consumed at +0xe00 after the two instances exchange +/// their first peer messages. It is not named by an initdata descriptor, but +/// its fixed mapping is part of the native first-partial topology. +pub(crate) const G17P_PARTIAL_OPENING_COMPUTED_GPU_VA: u64 = 0xffff_fc20_015d_8000; +pub(crate) const G17P_PARTIAL_OPENING_SCHEDULER_GPU_VA: u64 = 0xffff_fc20_015e_0000; +pub(crate) const G17P_PARTIAL_OPENING_SCHEDULER_WORD: u64 = 0x0000_0020_0001_9000; +/// Firmware-high owner of the four-page primary-index run in the clean +/// first-partial capture. The render TTB0 aliases this at +/// [`G17P_PARTIAL_OPENING_PRIMARY_INDEX_GPU_VA`]. +pub(crate) const G17P_PARTIAL_OPENING_PRIMARY_INDEX_FIRMWARE_GPU_VA: u64 = + 0xffff_fc20_c084_8000; +pub(crate) const G17P_PARTIAL_OPENING_PRIMARY_INDEX_GPU_VA: u64 = 0x0000_0010_0019_0000; +pub(crate) const G17P_PARTIAL_OPENING_PRIMARY_INDEX_SIZE: usize = + 4 * G17P_PARTIAL_OPENING_PAGE_SIZE; +pub(crate) const G17P_PARTIAL_OPENING_FWCTL_OFFSET: u64 = 0x001c_0000; +pub(crate) const G17P_PARTIAL_OPENING_CONTROL_MESSAGE_SIZE: u64 = 0x40; +/// Legacy Linux render work-submission doorbell channel. +/// +/// `encode_tx_doorbell` (this file) gives the layout as +/// `MSG_TX_DOORBELL | pipe_type | (index << 2)`, `index` being the queue +/// priority. The three actual GPU pipes are: +/// +/// TA = Vertex(0) +/// 3D = Fragment(1) +/// compute = Compute(2) +/// +/// Linux historically sent only Vertex/priority 0 after publishing both render +/// queues. A gate probe proved that message wakes the cores, not that it +/// dispatches both pipes. Mac17P `AGXArmFirmware::submit3DChannel` at +/// `0x8a43aa0` sends Fragment first (tail `0x8a43c6c`), then +/// `submitTAChannel` at `0x8a43cdc` sends Vertex (tail `0x8a43e10`). The +/// byte-complete boot trace confirms ordinary userspace queues use priority 2; +/// the earlier priority-0 pairs are a distinct system bring-up class. +pub(crate) const G17P_PARTIAL_OPENING_WORK_DOORBELL_CHANNEL: u8 = 0; +pub(crate) const G17P_PARTIAL_OPENING_PAIRED_DOORBELL: u64 = + MSG_TX_DOORBELL | G17P_PARTIAL_OPENING_WORK_DOORBELL_CHANNEL as u64; +// Default-off build diagnostic: KRUSTFLAGS="--cfg g17p_render_priority_probe". +// Native ordinary user queues captured priority/subpriority 2/2. This does +// not establish a Linux UAPI mapping or make the profile a production fix. +pub(crate) const G17P_NATIVE_RENDER_DOORBELL_PRIORITY: u8 = + if cfg!(g17p_render_priority_probe) { 2 } else { 0 }; +// HardwareBufferID is independently allocated by AGXHardwareBufferIDManager, +// not derived from priority (native captures have priority 2 with IDs 5/6). +// Preserve the existing Linux identity for this diagnostic. +pub(crate) const G17P_NATIVE_RENDER_QOS_HARDWARE_BUFFER_ID: u8 = 0; +pub(crate) const G17P_NATIVE_RENDER_POLICY: u16 = + if G17P_NATIVE_RENDER_DOORBELL_PRIORITY == 0 { 1 } else { 2 }; +/// Mac17P submits 3D first: Fragment(1) | (priority 2 << 2) = 0x09. +pub(crate) const G17P_NATIVE_3D_DOORBELL: u64 = MSG_TX_DOORBELL + | PipeType::Fragment as u64 + | ((G17P_NATIVE_RENDER_DOORBELL_PRIORITY as u64) << DOORBELL_INDEX_SHIFT); +/// Mac17P then submits TA: Vertex(0) | (priority 2 << 2) = 0x08. +pub(crate) const G17P_NATIVE_TA_DOORBELL: u64 = MSG_TX_DOORBELL + | PipeType::Vertex as u64 + | ((G17P_NATIVE_RENDER_DOORBELL_PRIORITY as u64) << DOORBELL_INDEX_SHIFT); +pub(crate) const G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE: usize = 0x18; +pub(crate) const G17P_PARTIAL_OPENING_OUTER_SLOT_COUNT: u32 = 0x100; +/// Work-channel table index is `priority * 3 + pipe`, with pipe Vertex(TA)=0, +/// Fragment(3D)=1, Compute(CL)=2. That is exactly the grouping +/// [`g17_initdata::WORK_STATE_GRID_OFFSETS`] documents: indices 0,1,2 are +/// TA_0/3D_0/CL_0 and indices 6,7,8 are TA_2/3D_2/CL_2. +/// +/// The render must publish into the rings for the priority it actually +/// configures and announces. This profile follows native's initial priority-0 +/// qid-0/1 pair consistently through table selection, queue records, tags, +/// QoS and the paired doorbell. Later priority-2 userspace queues use indices +/// 6/7 and must likewise remain internally coherent. +/// +/// Compute is the worked example of the rule: index 8 is CL at priority 2, it +/// agrees with its own priority-2 doorbell 0x0a, and it is validated by 2000 +/// consecutive passing computes. +const fn g17p_work_channel_table_index(priority: u8, pipe: PipeType) -> u8 { + priority * 3 + pipe as u8 +} +pub(crate) const G17P_PARTIAL_OPENING_TILING_CHANNEL_TABLE_INDEX: u8 = + g17p_work_channel_table_index(G17P_NATIVE_RENDER_DOORBELL_PRIORITY, PipeType::Vertex); +pub(crate) const G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX: u8 = + g17p_work_channel_table_index(G17P_NATIVE_RENDER_DOORBELL_PRIORITY, PipeType::Fragment); +pub(crate) const G17P_COMPUTE_CHANNEL_TABLE_INDEX: u8 = + g17p_work_channel_table_index(2, PipeType::Compute); +// The compute index is the validated anchor for the rule above: it must stay 8. +const _: () = assert!(G17P_COMPUTE_CHANNEL_TABLE_INDEX == 8); +const _: () = assert!(G17P_PARTIAL_OPENING_TILING_CHANNEL_TABLE_INDEX == + G17P_NATIVE_RENDER_DOORBELL_PRIORITY * 3); +const _: () = assert!(G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX == + G17P_NATIVE_RENDER_DOORBELL_PRIORITY * 3 + 1); +/// Compute work-submission doorbell channel. VALIDATED by 2000 consecutive +/// passing computes. Decodes under `encode_tx_doorbell` as +/// `pipe Compute(2) | (priority 2 << 2)` = 0x0a, which is why it is not simply +/// the channel-table index (8). +pub(crate) const G17P_COMPUTE_WORK_DOORBELL_CHANNEL: u8 = 0x0a; +pub(crate) const G17P_COMPUTE_WORK_DOORBELL: u64 = + MSG_TX_DOORBELL | G17P_COMPUTE_WORK_DOORBELL_CHANNEL as u64; +pub(crate) const G17P_COLD_OPENING_RECORD_POOL_A_COUNT: usize = 35; +pub(crate) const G17P_COLD_OPENING_RECORD_POOL_A_STRIDE: usize = 0x100; +pub(crate) const G17P_COLD_OPENING_RECORD_POOL_A_SIZE: usize = + G17P_COLD_OPENING_RECORD_POOL_A_COUNT * G17P_COLD_OPENING_RECORD_POOL_A_STRIDE; +pub(crate) const G17P_COLD_OPENING_RECORD_POOL_B_SIZE: usize = 79 * 0x80; +pub(crate) const G17P_COLD_OPENING_SHARED_OBJECT_SIZE: usize = 0x88; +pub(crate) const G17P_COLD_OPENING_SHARED_CONTROL_SIZE: usize = 0x70; +pub(crate) const G17P_COLD_OPENING_SHARED_CONTROL_INNER_SIZE: usize = 4; +pub(crate) const G17P_COLD_OPENING_CHANNEL_CONTROL_SIZE: usize = 0x40; +pub(crate) const G17P_COLD_OPENING_OPTIONAL_SIZE: usize = 0xc0; +pub(crate) const G17P_COLD_OPENING_EVENT_SIZE: usize = 0x40; +pub(crate) const G17P_COLD_OPENING_QUEUE_CONTEXT_SIZE: usize = 0x4000; +pub(crate) const G17P_COLD_OPENING_TA_DESCRIPTOR_BASE: u64 = 0xffff_fc20_c001_8000; +pub(crate) const G17P_COLD_OPENING_3D_DESCRIPTOR_BASE: u64 = 0xffff_fc20_c00b_0000; +pub(crate) const G17P_COLD_OPENING_QUEUE_RECORD_SIZE: usize = 0xc0; +pub(crate) const G17P_COLD_OPENING_POINTER_BLOCK_SIZE: usize = 0x80; +pub(crate) const G17P_COLD_OPENING_OPERAND_DIRECTORY_GPU_VA: u64 = 0x70_0000_0000; +pub(crate) const G17P_COLD_OPENING_OPERAND_DIRECTORY_SIZE: usize = 4 * 0x4000; +pub(crate) const G17P_COLD_OPENING_OPERAND_TABLE_GPU_VA: u64 = 0x70_0020_8000; +pub(crate) const G17P_COLD_OPENING_OPERAND_BUFFER_BASE: u64 = 0x70_0022_0000; +pub(crate) const G17P_COLD_OPENING_OPERAND_BUFFER_COUNT: usize = 28; +pub(crate) const G17P_COLD_OPENING_OPERAND_BUFFER_SIZE: u64 = 0x10_0000; +pub(crate) const G17P_COLD_OPENING_OPERAND_BUFFER_STRIDE: u64 = 0x10_8000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PColdOpeningLeaf { + PrimaryIndex, + SecondaryIndex, + PoolASlots, + PoolBSlots, + SharedSlots, + Flag, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PColdOpeningStage { + Tiling, + Fragment, +} + +/// Relocatable leaf addresses used by the cold-opening pool and shared graph. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PColdOpeningGraphAddresses { + pub(crate) primary_index: u64, + pub(crate) secondary_index: u64, + pub(crate) pool_a_slots: u64, + pub(crate) pool_b_slots: u64, + pub(crate) shared_slots: u64, + pub(crate) flag: u64, +} + +/// Relocatable pointers carried by one cold-opening optional record. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PColdOpeningOptionalAddresses { + pub(crate) context_scratch: u64, + pub(crate) firmware_scratch: u64, + pub(crate) usc_priv_mem_flist: u64, + /// HardwareBufferID of `usc_priv_mem_flist`, copied to tag-15 +0x46. + pub(crate) usc_freelist_hardware_buffer_id: u32, + pub(crate) channel_control: u64, + pub(crate) tiling_shared_object: u64, + /// HWPB owner lease token (HardwareBufferBase +0x68), not an ordinal. + /// Reacquiring the TA HardwareBufferID from zero references assigns it. + pub(crate) parameter_buffer_token: u64, + /// PID of the process that owns the paired TA/3D channels. Native stores + /// the same u32 in `_AGFIChannelState +0x44` and ConfigUpdate +0x5a. + pub(crate) owner_pid: u32, + /// TA HardwareBufferID owned by the command being published. + /// + /// Native copies one allocator result to five firmware-visible sinks: TA + /// command +0x18, register 0x1c830, ConfigUpdate +0x7e, PBState +0x0c, and + /// the PBDesc ring index. Keeping the ConfigUpdate copy beside its object + /// pointer prevents those fields from being staged from different command + /// lifetimes. + pub(crate) ta_hardware_buffer_id: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PColdOpeningQueueAddresses { + pub(crate) pointers: u64, + pub(crate) item_ring: u64, + pub(crate) job_list: u64, + pub(crate) channel_control: u64, + /// PID of the process that owns this channel state. + pub(crate) owner_pid: u32, +} + +/// Root, queue, and scheduler identities at the proven first-work boundary. +/// The successful run used the two-entry macOS context table: slot/tag 0 and +/// slot/tag 1 both retain their low roots while their high roots are replaced +/// with distinct empty pages. Linux owns those pages, so their measured +/// physical addresses are intentionally not part of this contract. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PPartialOpeningContextContract { + pub(crate) context0_root_slot: u8, + pub(crate) context0_context_id: u8, + pub(crate) render_root_slot: u8, + pub(crate) render_context_id: u8, + pub(crate) context0_empty_high_root_slot: u8, + pub(crate) render_empty_high_root_slot: u8, + pub(crate) transport_pair: u8, + pub(crate) descriptor_pair: u8, + pub(crate) shared_queue_pair_namespace: u8, + pub(crate) optional_identity_context_id: u8, + pub(crate) descriptor_context_id: u8, + pub(crate) scheduler_class: u8, + pub(crate) optional_grids: [u8; 2], + pub(crate) queue_identity: u8, + pub(crate) optional_submission_ordinal: u16, + pub(crate) event_counter: u32, + pub(crate) work_doorbell_channel: u8, + pub(crate) context0_low_root_present_at_first_work: bool, + pub(crate) render_low_root_present_at_first_work: bool, + pub(crate) source_firmware_high_root_table_resident: bool, +} + +pub(crate) const G17P_PARTIAL_OPENING_CONTEXT: G17PPartialOpeningContextContract = + G17PPartialOpeningContextContract { + context0_root_slot: 0, + context0_context_id: 0, + render_root_slot: 1, + render_context_id: 1, + context0_empty_high_root_slot: 0, + render_empty_high_root_slot: 1, + transport_pair: 0, + descriptor_pair: 0, + shared_queue_pair_namespace: 0, + optional_identity_context_id: 1, + descriptor_context_id: 1, + scheduler_class: 2, + optional_grids: [0, 1], + queue_identity: 0x15, + optional_submission_ordinal: 0, + event_counter: 0x102, + work_doorbell_channel: G17P_PARTIAL_OPENING_WORK_DOORBELL_CHANNEL, + context0_low_root_present_at_first_work: true, + render_low_root_present_at_first_work: true, + source_firmware_high_root_table_resident: false, + }; + +/// One exact 64-bit write into the primary status-B object before control 0x84. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PPartialOpeningStatusWrite { + pub(crate) offset: u32, + pub(crate) value: u64, +} + +/// State required to prepare one half of the paired outer-channel publish. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PPartialOpeningOuterChannelState { + pub(crate) consumers: [u32; 2], + pub(crate) producer: u32, + pub(crate) queue_gpu_va: u64, + pub(crate) queue_write_index: u32, +} + +/// One complete 0x18-byte outer slot followed by its producer publication. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PPartialOpeningOuterSlotPlan { + pub(crate) channel_table_index: u8, + pub(crate) slot_index: u8, + pub(crate) slot_byte_offset: u32, + pub(crate) slot: [u8; G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE], + pub(crate) next_producer: u32, +} + +/// The two producer writes and the single mailbox kick for first graphics. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PPartialOpeningWorkPair { + pub(crate) fragment: G17PPartialOpeningOuterSlotPlan, + pub(crate) tiling: G17PPartialOpeningOuterSlotPlan, + pub(crate) paired_doorbell: u64, +} + +/// One proven CL_2 outer-ring publication and its matching RTKit doorbell. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PComputeChannelWork { + pub(crate) outer: G17PPartialOpeningOuterSlotPlan, + pub(crate) doorbell: u64, +} + +/// Side-effect order measured at the successful first-partial boundary. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PPartialOpeningPublicationStep { + PrimarySchedulerPage, + PrimaryIndexPage, + PreControlStatus, + ControlDone0x84, + FragmentState, + EmptyHighRoots, + FragmentProducer, + LateTilingState, + SystemBarrierBeforeTilingProducer, + TilingProducer, + SystemBarrierBeforePairedDoorbell, + PairedWorkDoorbell0x83, +} + +pub(crate) const G17P_PARTIAL_OPENING_PUBLICATION_ORDER: [G17PPartialOpeningPublicationStep; 12] = [ + G17PPartialOpeningPublicationStep::PrimarySchedulerPage, + G17PPartialOpeningPublicationStep::PrimaryIndexPage, + G17PPartialOpeningPublicationStep::PreControlStatus, + G17PPartialOpeningPublicationStep::ControlDone0x84, + G17PPartialOpeningPublicationStep::FragmentState, + G17PPartialOpeningPublicationStep::EmptyHighRoots, + G17PPartialOpeningPublicationStep::FragmentProducer, + G17PPartialOpeningPublicationStep::LateTilingState, + G17PPartialOpeningPublicationStep::SystemBarrierBeforeTilingProducer, + G17PPartialOpeningPublicationStep::TilingProducer, + G17PPartialOpeningPublicationStep::SystemBarrierBeforePairedDoorbell, + G17PPartialOpeningPublicationStep::PairedWorkDoorbell0x83, +]; + +/// Build the computed one-record scheduler page that decided graphics entry in +/// the successful post-control graft. +pub(crate) fn apply_g17p_partial_opening_scheduler_page( + page: &mut [u8], +) -> Result<(), SubmissionError> { + if page.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningPageSize); + } + page.fill(0); + put_u64_le(page, 0, G17P_PARTIAL_OPENING_SCHEDULER_WORD); + Ok(()) +} + +/// Array-returning form retained for host tests. Kernel mappings should use +/// [`apply_g17p_partial_opening_scheduler_page`] to avoid a 16 KiB temporary. +pub(crate) fn build_g17p_partial_opening_scheduler_page() -> [u8; G17P_PARTIAL_OPENING_PAGE_SIZE] { + let mut page = [0u8; G17P_PARTIAL_OPENING_PAGE_SIZE]; + let _ = apply_g17p_partial_opening_scheduler_page(&mut page); + page +} + +/// Build the eight-group primary index used by the proven opening. +pub(crate) fn apply_g17p_partial_opening_primary_index_page( + page: &mut [u8], +) -> Result<(), SubmissionError> { + if page.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningPageSize); + } + page.fill(0); + let mut index = 0usize; + for (start, groups) in [(0x11u32, 6u32), (0x3cu32, 2u32)] { + for group in 0..groups { + let base = start + group * 5; + for member in 0..4u32 { + put_u32_le(page, index * 4, base + member); + index += 1; + } + } + } + Ok(()) +} + +/// Array-returning form retained for host tests. Kernel mappings should use +/// [`apply_g17p_partial_opening_primary_index_page`] directly. +pub(crate) fn build_g17p_partial_opening_primary_index_page() -> [u8; G17P_PARTIAL_OPENING_PAGE_SIZE] +{ + let mut page = [0u8; G17P_PARTIAL_OPENING_PAGE_SIZE]; + let _ = apply_g17p_partial_opening_primary_index_page(&mut page); + page +} + +/// Write one of the six source-built leaf pages used by the cold opening. +pub(crate) fn apply_g17p_cold_opening_leaf_page( + leaf: G17PColdOpeningLeaf, + page: &mut [u8], +) -> Result<(), SubmissionError> { + if page.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + page.fill(0); + match leaf { + G17PColdOpeningLeaf::PrimaryIndex => { + apply_g17p_partial_opening_primary_index_page(page)?; + } + G17PColdOpeningLeaf::SecondaryIndex => { + let mut index = 0usize; + for (start, groups) in [(0x11u32, 6u32), (0x3cu32, 2u32)] { + for group in 0..groups { + put_u64_le(page, index * 8, (start + group * 5) as u64); + index += 1; + } + } + } + G17PColdOpeningLeaf::PoolASlots => put_u32_le(page, 0x04, 2), + G17PColdOpeningLeaf::PoolBSlots => {} + G17PColdOpeningLeaf::SharedSlots => { + put_u32_le(page, 0x00, 8); + put_u32_le(page, 0x04, 8); + put_u32_le(page, 0x60, 1); + } + G17PColdOpeningLeaf::Flag => put_u32_le(page, 0x00, 1), + } + Ok(()) +} + +/// Select the Pool-A record used by one render pass. Native G17P moves the cold +/// first pass from record 0 to the captured record 1. Retained native +/// scenes can reuse an index across multiple sequence values, so the traces do +/// not justify replacing Linux's retained `ordinal * 2` policy globally. +pub(crate) const fn g17p_render_pool_a_record_index( + submission_ordinal: u64, + native_first_record: bool, +) -> usize { + if native_first_record && submission_ordinal == 0 { + 1 + } else { + ((submission_ordinal % G17P_COLD_OPENING_RECORD_POOL_A_COUNT as u64) as usize * 2) + % G17P_COLD_OPENING_RECORD_POOL_A_COUNT + } +} + +/// Initialize only the Pool-A slot corresponding to the selected first +/// record. The slot page has no other modeled contents. +pub(crate) fn apply_g17p_cold_opening_pool_a_start_slot( + record_index: usize, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + if record_index >= G17P_COLD_OPENING_RECORD_POOL_A_COUNT { + return Err(SubmissionError::PartialOpeningQueueHeadOutOfRange); + } + out.fill(0); + // Native numbers the slot words from the start of the page: cold record + // index 1 points at page +0x04. The old extra +4 coupled index 1 to +0x08 + // and made the host graph one slot ahead of the valid first-render graph. + put_u32_le(out, record_index * 4, 1); + Ok(()) +} + +/// Write the 35-record Pool-A array with relocatable slot pointers and select +/// the record whose first-pass state word is initialized to 0x50. +pub(crate) fn apply_g17p_cold_opening_record_pool_a_at( + addresses: G17PColdOpeningGraphAddresses, + active_record_index: usize, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if active_record_index >= G17P_COLD_OPENING_RECORD_POOL_A_COUNT { + return Err(SubmissionError::PartialOpeningQueueHeadOutOfRange); + } + if out.len() != G17P_COLD_OPENING_RECORD_POOL_A_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + let slot_base = addresses.pool_a_slots; + for index in 0..G17P_COLD_OPENING_RECORD_POOL_A_COUNT { + let slot = slot_base + .checked_add((index * 4) as u64) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?; + let base = index * G17P_COLD_OPENING_RECORD_POOL_A_STRIDE; + put_u64_le(out, base, slot); + if index == active_record_index { + put_u32_le(out, base + 0x10, 0x50); + } + } + Ok(()) +} + +/// Established record-zero wrapper retained for callers and fixture tests. +pub(crate) fn apply_g17p_cold_opening_record_pool_a( + addresses: G17PColdOpeningGraphAddresses, + out: &mut [u8], +) -> Result<(), SubmissionError> { + apply_g17p_cold_opening_record_pool_a_at(addresses, 0, out) +} + +/// Write the 79-record Pool-B array in the cold pair-zero namespace. +pub(crate) fn apply_g17p_cold_opening_record_pool_b( + addresses: G17PColdOpeningGraphAddresses, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_RECORD_POOL_B_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + let slot_base = addresses + .pool_b_slots + .checked_add(4) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?; + let shared = addresses + .shared_slots + .checked_add(0x40) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?; + for index in 0..79usize { + let base = index * 0x80; + put_u32_le(out, base, 0x80004 + index as u32 * 4); + put_u32_le(out, base + 0x04, 0x10); + put_u64_le( + out, + base + 0x08, + slot_base + .checked_add((index * 4) as u64) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?, + ); + let phase = index % 36; + let cycle = if phase == 35 { + 0x178000 + } else { + 0x178020 + phase as u32 * 0x20 + }; + put_u32_le(out, base + 0x28, cycle); + put_u64_le(out, base + 0x40, shared); + if index == 0 { + put_u32_le(out, base + 0x4c, 1); + } + } + Ok(()) +} + +/// Write the legacy packed HWPB-shaped object used by TA and by the +/// established Fragment path. Native Fragment can instead name the dedicated +/// PM HWPB state; TA keeps this distinct object. +pub(crate) fn apply_g17p_cold_opening_shared_object( + addresses: G17PColdOpeningGraphAddresses, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_SHARED_OBJECT_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + for (offset, address) in [ + (0x20, addresses.primary_index), + (0x44, addresses.secondary_index), + (0x4c, addresses.shared_slots), + (0x64, addresses.flag), + ] { + put_u64_le(out, offset, address); + } + for (offset, value) in [ + (0x28, 0x0019_0000), + (0x2c, 0x10), + (0x30, 0x0001_0000), + (0x34, 0x20), + (0x38, 0x0c18), + (0x3c, 8), + (0x54, 0x1f), + (0x58, 0x0002_0000), + (0x7c, 0x3060), + (0x80, 0x1020), + (0x84, 0x0018_0000), + ] { + put_u32_le(out, offset, value); + } + Ok(()) +} + +/// Write the 0x70-byte class-2 control object used before cold first work. +pub(crate) fn apply_g17p_cold_opening_shared_control( + operand_table: u64, + inner_state: u64, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_SHARED_CONTROL_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u32_le(out, 0x00, 1); + put_u32_le(out, 0x10, 2); + put_u64_le(out, 0x18, 0x0004_0000_0000_0070); + put_u64_le(out, 0x20, 0x0000_1900_0000_0000); + put_u64_le(out, 0x28, 0x0000_1900_0000_0000); + put_u64_le(out, 0x30, operand_table); + put_u32_le(out, 0x40, 4); + put_u32_le(out, 0x48, 0xe0); + put_u64_le(out, 0x4c, inner_state); + put_u32_le(out, 0x60, 3); + Ok(()) +} + +pub(crate) fn apply_g17p_cold_opening_shared_control_inner( + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_SHARED_CONTROL_INNER_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + put_u32_le(out, 0, 2); + Ok(()) +} + +/// Write the fresh channel-control record used by both cold queues. +pub(crate) fn apply_g17p_cold_opening_channel_control( + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_CHANNEL_CONTROL_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u64_le(out, 0x00, 0x0000_0100_0000_ffff); + put_u64_le(out, 0x20, 0x0002_0000_0000_0000); + put_u64_le(out, 0x30, 0x0000_0000_ff00_0000); + Ok(()) +} + +/// Write one exact grid-0/1 optional record from the cold partial opening. +pub(crate) fn apply_g17p_cold_opening_optional( + stage: G17PColdOpeningStage, + addresses: G17PColdOpeningOptionalAddresses, + queue_id: u16, + partner_queue_id: u16, + qos_hardware_buffer_id: u16, + install: u16, + context_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_OPTIONAL_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u32_le(out, 0, 0x0f); + for (offset, address) in [ + (0x08, addresses.context_scratch), + (0x10, addresses.firmware_scratch), + (0x36, addresses.usc_priv_mem_flist), + (0x4a, addresses.channel_control), + ] { + put_u64_le(out, offset, address); + } + let fields: &[(usize, u16)] = match stage { + // +0x56 is the AGXQOSQueue HardwareBufferID. Native gives each + // retained TA/3D pair one ID (both stages share it), while different + // pairs carry different IDs. It must match both the qid*8 QoS record + // and the KSM dependency-control qword. + // Byte-diffed against live ConfigUpdate records from two independent + // Metal render passes on the target (macOS 2026-09-02, + // `records-gfx.txt` and `everyobj-gfx.txt`): + // + // +0x66 = 1 in both. + // +0x46 is written below as a u32 from the allocated FList hardware + // buffer id; it reads 0 in one capture and 2 in the other, + // so it is dynamic, not a constant. + // +0x62 read 1 in one capture and 0 in the other, so it is dynamic, + // not a constant; 1 is what this driver always sent and what + // the majority of captured records carry. An earlier claim + // that native always has 0 came from a single record and is + // contradicted by the wider capture. + G17PColdOpeningStage::Tiling => &[ + (0x18, queue_id), + (0x1a, install), + (0x1e, G17P_NATIVE_RENDER_DOORBELL_PRIORITY as u16), + (0x26, 1), + (0x52, 1), + (0x56, qos_hardware_buffer_id), + (0x5e, G17P_NATIVE_RENDER_POLICY), + (0x62, 1), + (0x66, 1), + // +0x82 is the TA's FORWARD REFERENCE TO ITS 3D PARTNER, not a + // flag. MEASURED ON macOS 2026-09-02 (recs2-run1.txt): across two + // processes and six passes, a native TA ConfigUpdate carries the + // partner FRAGMENT queue's id here -- 0x10 while its own +0x18 is + // 0x0f -- and the matching 3D record fills +0x76..+0x85 with 0xff, + // the "no forward reference" sentinel this encoder already writes + // for the Fragment stage. The constant 1 published here before + // named queue 1, which is neither stage. + (0x82, partner_queue_id), + ], + G17PColdOpeningStage::Fragment => &[ + (0x18, queue_id), + (0x1a, install), + (0x1e, G17P_NATIVE_RENDER_DOORBELL_PRIORITY as u16), + (0x22, 1), + (0x26, 1), + (0x52, 1), + (0x56, qos_hardware_buffer_id), + (0x5e, G17P_NATIVE_RENDER_POLICY), + (0x62, 1), + (0x66, 1), + ], + }; + for (offset, value) in fields { + put_u16_le(out, *offset, *value); + } + // DIAGNOSTIC. +0x22 is the ConfigUpdate's data-master field, hardcoded to + // 1 for the fragment. The tiler (DM0) and compute (DM2) both dispatch and + // execute; the fragment is retired with KTrace 0x43 status 4 without ever + // running, with an identically shaped descriptor, a satisfied barrier and + // geometry already tiled. Overriding this isolates the data master itself. + let override_dm = match stage { + G17PColdOpeningStage::Fragment => crate::g18::fw::config::RENDER_3D_DM, + G17PColdOpeningStage::Tiling => crate::g18::fw::config::RENDER_TA_DM, + }; + if override_dm != 0xff { + put_u16_le(out, 0x22, override_dm as u16); + } + // AGXChannel::resetChannelState copies channel+0x4c (the owning process + // PID) to state+0x44. ConfigUpdate carries that same u32 at unaligned + // +0x5a; the working Metal capture has 0x625 in both locations. + put_u32_le(out, 0x5a, addresses.owner_pid); + // Native copies the HardwareBufferID allocated for the FList named at + // +0x36 into this tag-15 field. Keep the object pointer and its table slot + // sourced from the same retained owner. + put_u32_le(out, 0x46, addresses.usc_freelist_hardware_buffer_id); + // +0x32 is the parameter-buffer binding context when +0x6e is nonzero. + // It is a distinct kext allocation/source from TA command +0x0c even when + // both happen to be context 1 in our single-client bring-up path. + put_u16_le(out, 0x32, context_id); + match stage { + G17PColdOpeningStage::Tiling => { + put_u64_le(out, 0x6e, addresses.tiling_shared_object); + put_u64_le(out, 0x76, addresses.parameter_buffer_token); + // AGXTAChannelSKSM::submitBuffer copies descriptor +0x1248 here + // with a 32-bit store. The same source feeds TA command +0x18, + // register 0x1c830, PBState +0x0c, and the PBDesc ring; firmware + // uses it to index the 0x50-byte PB binding table. + put_u32_le(out, 0x7e, addresses.ta_hardware_buffer_id); + } + G17PColdOpeningStage::Fragment => out[0x76..0x86].fill(0xff), + } + Ok(()) +} + +/// Write one cold event record. Both halves use the measured counter 0x102. +pub(crate) fn apply_g17p_cold_opening_event( + stage: G17PColdOpeningStage, + queue_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_EVENT_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u32_le(out, 0x00, 0x0e); + // +0x04 is [u16 queue_id][u16 1]: the firmware's own 0x41 tag14-addkicks + // receipt reads the QID with `ldrh w21, [x19, #4]`, and the native capture + // carries 0x11/0x12 there for TA/3D. + put_u32_le(out, 0x04, 0x0001_0000 | queue_id as u32); + put_u32_le( + out, + 0x08, + 0x100 | u32::from(G17P_NATIVE_RENDER_DOORBELL_PRIORITY), + ); + if stage == G17PColdOpeningStage::Fragment { + put_u32_le(out, 0x10, 0x100); + } + Ok(()) +} + +/// Refresh the canonical optional record for a later logical submission. +/// The queue-context record itself remains in locator slot zero. +/// +/// Publishes the per-slot ordinal as the FList generation at `+0x3e`, which +/// is the same number as the device-global FList sequence for as long as one +/// client renders. `apply_g17p_retained_optional_with_flist_generation` takes +/// the device-global value explicitly. +pub(crate) fn apply_g17p_retained_optional( + stage: G17PColdOpeningStage, + addresses: G17PColdOpeningOptionalAddresses, + submission_ordinal: u64, + bias_generation: bool, + queue_id: u16, + partner_queue_id: u16, + qos_hardware_buffer_id: u16, + install: u16, + context_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + apply_g17p_retained_optional_with_flist_generation( + stage, + addresses, + submission_ordinal, + g17p_render_flist_generation(submission_ordinal), + bias_generation, + queue_id, + partner_queue_id, + qos_hardware_buffer_id, + install, + context_id, + out, + ) +} + +/// `apply_g17p_retained_optional` with the USC FList generation supplied by +/// the caller. +/// +/// The FList is one device-global pool, so its generation advances with every +/// render on the device, while `submission_ordinal` counts only this slot's +/// renders. With one client the two coincide; the moment a second slot +/// publishes, its first tag-15 would otherwise announce generation 0 against a +/// pool the firmware last saw at N. +pub(crate) fn apply_g17p_retained_optional_with_flist_generation( + stage: G17PColdOpeningStage, + addresses: G17PColdOpeningOptionalAddresses, + submission_ordinal: u64, + flist_generation: u64, + bias_generation: bool, + queue_id: u16, + partner_queue_id: u16, + qos_hardware_buffer_id: u16, + install: u16, + context_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + apply_g17p_retained_optional_with_flist_generation_and_context_update( + stage, + addresses, + submission_ordinal, + flist_generation, + bias_generation, + queue_id, + partner_queue_id, + qos_hardware_buffer_id, + install, + context_id, + submission_ordinal == 0, + out, + ) +} + +/// Retained ConfigUpdate with the GART-context branch supplied independently +/// from queue installation and QoS setup. +/// +/// Native derives +0x62 from the descriptor's GART allocation generation +/// versus the channel's cached context. A graph rebuilt for another VM must +/// therefore request the context update even when its physical QID is already +/// installed and its global submission ordinal is nonzero. Conversely, a +/// same-context restage must leave it clear. +pub(crate) fn apply_g17p_retained_optional_with_flist_generation_and_context_update( + stage: G17PColdOpeningStage, + addresses: G17PColdOpeningOptionalAddresses, + submission_ordinal: u64, + flist_generation: u64, + bias_generation: bool, + queue_id: u16, + partner_queue_id: u16, + qos_hardware_buffer_id: u16, + install: u16, + context_id: u16, + context_update: bool, + out: &mut [u8], +) -> Result<(), SubmissionError> { + // Payload bytes wrap independently of the full retained owner generation. + apply_g17p_cold_opening_optional( + stage, + addresses, + queue_id, + partner_queue_id, + qos_hardware_buffer_id, + install, + context_id, + out, + )?; + if submission_ordinal != 0 { + for offset in [0x1a, 0x52] { + put_u16_le(out, offset, 0); + } + } + put_u16_le(out, 0x62, u16::from(context_update)); + // +0x2a is read by the firmware as a whole u64 (`ldur x27, [x19, #42]` in + // the tag-15 ConfigUpdate handler) and stored into its per-queue table at + // `0xfffffc0000125df0 + qid * 8`. Every consumer of that table uses + // `lsr x9, x11, #40` + `uxtb` -- bits 47:40, which is byte 0x2f, the high + // byte of the u16 written at +0x2e, i.e. `ordinal & 0xff`. And each one + // compares it against a cached byte and SKIPS when they are equal: + // + // ldr x11, [table + qid*8] + // lsr x9, x11, #40 + // ldrb w10, [x19, #40] + // cmp w10, w9, uxtb + // b.eq ; unchanged -> do nothing + // strb w9, [x19, #40] + // + // It is a change-detected generation. The raw ordinal is 0 on the first + // render submit, which matches the firmware's initial cached 0, so the + // very first submission of a boot reads as "nothing changed" and the + // firmware declines to act -- with no record emitted either way, which is + // why no capture ever showed it. + // + // This is the same defect already fixed at +0x3e in this function, whose + // comment records that native never publishes zero there. Apply the same + // bias so the first submit publishes a generation the firmware sees as new. + let generation = if bias_generation { + (submission_ordinal as u16).wrapping_add(1) + } else { + submission_ordinal as u16 + }; + put_u16_le(out, 0x2a, generation); + put_u16_le(out, 0x2e, generation << 8); + // +0x3e is the AGXUSCPrivMemFList generation. Native copies FList+0xb0 + // here and copies the same value into opcode-0x20 +0x0c during + // AsyncGrowEngine::updateSubmitInfo. Keep both Linux producers on the + // shared helper below: changing this generation between registration and + // the tag-15 kick makes B1 release and re-register the just-published USC + // allocation before fragment engine service. + // The kext uses a qword store at 8a98c48; +0x46 begins the independent + // hardware-buffer ID and must remain untouched by this generation. + put_u64_le(out, 0x3e, flist_generation); + // Keep the pair-owned HardwareBufferID identical in ConfigUpdate, the + // QID-indexed QoS row, and the KSM dependency-control qword. + put_u16_le(out, 0x56, qos_hardware_buffer_id); + // TA +0x76 is not the submission ordinal. Native copies the u64 at + // AGXHardwareBufferBase +0x68 here. The cold encoder above copies the + // captured PB owner lease token; leave it intact when refreshing queue + // generations. In particular, do not overwrite +0x76/+0x7e with ordinal. + Ok(()) +} + +/// Host-owned equivalent of AGXUSCPrivMemFList +0xb0 for a retained render +/// queue. Valid native cold records start at zero; each later retained restage +/// advances the same generation once. +pub(crate) const fn g17p_render_flist_generation(submission_ordinal: u64) -> u64 { + submission_ordinal as u64 +} + +/// Refresh the canonical event record while retaining its physical address. +/// +/// Tag-14 +0x09 is an unaligned u64 containing the 40-bit KSM timestamp. +/// 25G83 A000 reads it with `ldur x3, [x19, #9]` at 0x11730. Small cold +/// captures do not establish a 24-bit field. +0x08 remains queue priority; +/// the independently owned data-master field starts at +0x11. +pub(crate) fn apply_g17p_retained_event_with_stamp( + stage: G17PColdOpeningStage, + entry_stamp: u64, + queue_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if entry_stamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + apply_g17p_cold_opening_event(stage, queue_id, out)?; + put_u64_le(out, 0x09, entry_stamp); + Ok(()) +} + +/// Refresh the canonical event record while retaining its physical address. +/// +/// Without the SKSM half there is no KSM producer to read a stamp from, so +/// the stamp is derived as `submission_ordinal + 1` -- which is exactly what +/// the producer would have yielded for the first `stamp_count` submissions, +/// since the geometry seeds `current_timestamp` at 1 and advances it by one +/// per kick. +pub(crate) fn apply_g17p_retained_event( + stage: G17PColdOpeningStage, + submission_ordinal: u64, + queue_id: u16, + out: &mut [u8], +) -> Result<(), SubmissionError> { + let group_number = submission_ordinal + .checked_add(1) + .ok_or(SubmissionError::PartialOpeningQueueHeadOutOfRange)?; + apply_g17p_retained_event_with_stamp(stage, group_number & KICK_TS_MASK, queue_id, out) +} + +/// Write the cold class-2 queue record for the owning process. +pub(crate) fn apply_g17p_cold_opening_queue_record( + addresses: G17PColdOpeningQueueAddresses, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_QUEUE_RECORD_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u64_le(out, 0x00, addresses.pointers); + put_u64_le(out, 0x08, addresses.item_ring); + // +0x10. Native's per-queue ring records carry the SAME pointer here for + // both render stages (ringrec-gfx.txt: 0xfffffc2000000060 in the TA record + // at fwva 0xfffffc20c0000b40 and in the 3D record at 0xfffffc20c0000c00), + // and it is the a5 argument of every AGXChannel::init. That looked like a + // shared firmware constant we were failing to write. It is NOT. + // + // KEXT_PROVEN (25G83): cmdq+0x680 is an AGXTimeStampQueue whose +0x28 is + // its firmware VA; AGXCommandQueue::allocate3DWorkQueueInner passes that + // as a5 to the channel init, AGXChannel::init stores it at chan+0x98 and + // resetChannelState copies it to state+0x10. The 0x18-byte + // _AGFITimeStampQueue slots are host-allocated out of a firmware-shared + // stack, and 0x60 is simply slot 4. The value repeats across boots only + // because the Metal test app is always the fifth client. Writing that + // literal from Linux would have published a pointer into someone else's + // allocation. + // + // FW_PROVEN: nothing in the DM-ring consumption path (FUN_261a4 / + // FUN_12638) nor in the job builder FUN_23004 reads state+0x10 at all. It + // is a per-command-queue timestamp list head, used only by + // AGFAcceleratorProcessTimeStampQueue on the completion side. It cannot + // stop consumption or launch, so it is not our bug. Our per-graph value + // stays. + put_u64_le(out, 0x10, addresses.job_list); + put_u32_le(out, 0x24, u32::MAX); + // The live native render path reads state +0x28 through getPriority(), + // copies it to 3D setup +0xe5c, and validates it against KSM queue +0x1c. + // The captured ordinary Metal command carries 2. Channel +0x1c0 is only + // context-priority bookkeeping and is not the live-state value. + let priority = u32::from(G17P_NATIVE_RENDER_DOORBELL_PRIORITY); + put_u32_le(out, 0x28, priority); + put_u32_le(out, 0x2c, priority); + if priority == 0 { + out[0x32..0x38].fill(0xff); + put_u32_le(out, 0x38, 1); + } else { + out[0x36..0x38].fill(0xff); + put_u32_le(out, 0x38, 0); + } + // MEASURED ON macOS 2026-09-02 (`ringrec-gfx.txt`): the live per-queue + // record of a real Metal render pass, read through the kernel alias the + // AGXChannel keeps at +0x58 for the firmware VA at +0x80, is + // + // +0x24 0xffffffff +0x28 2 +0x2c 2 +0x36 0xffff + // +0x40 2 +0x44 0xffffffff +0x48 + // +0x9c + // + // This driver had that {2, 0xffffffff, pid} block four bytes early, at + // +0x3c/+0x40/+0x44, and the pointer at +0x70. The pointer had once been + // at +0x9c and was moved on the theory that the object is 0x80 bytes; the + // ring slots name records on an exact 0xc0 stride (base + qid * 0xc0), so + // 0x9c is inside the object and was right all along. + put_u32_le(out, 0x40, u32::from(G17P_NATIVE_RENDER_POLICY)); + put_u32_le(out, 0x44, u32::MAX); + // Native reset copies the channel owner's process PID here. The same + // value is published in ConfigUpdate +0x5a. + put_u32_le(out, 0x48, addresses.owner_pid); + put_u64_le(out, 0x9c, addresses.channel_control); + Ok(()) +} + +pub(crate) fn apply_g17p_cold_opening_pointer_block(out: &mut [u8]) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_POINTER_BLOCK_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + put_u32_le(out, 0x50, u32::MAX); + put_u64_le(out, 0x60, 0x500); + Ok(()) +} + +/// Apply the measured first-publication bytes after descriptor construction. +pub(crate) fn apply_g17p_cold_opening_descriptor_overrides( + stage: G17PColdOpeningStage, + descriptor: &mut [u8], +) -> Result<(), SubmissionError> { + if descriptor.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + let fields: &[(usize, u8)] = match stage { + G17PColdOpeningStage::Tiling => &[ + (0x38, 0x47), + (0x3a, 0x49), + (0x3c, 0x49), + (0x0789, 0x08), + (0x093e, 0xd0), + (0x093f, 0x91), + ], + G17PColdOpeningStage::Fragment => &[ + (0x50, 0x01), + (0x80, 0x56), + (0x82, 0x57), + (0x84, 0x57), + (0x88, 0x59), + (0x90, 0x01), + (0x215c, 0x00), + (0x222d, 0x00), + ], + }; + for (offset, value) in fields { + descriptor[*offset] = *value; + } + put_u32_le(descriptor, 0x0c, 1); + Ok(()) +} + +/// Keep the fixed operand table zero at the proven cold first-work boundary. +pub(crate) fn apply_g17p_cold_opening_operand_table(out: &mut [u8]) -> Result<(), SubmissionError> { + if out.len() != G17P_PARTIAL_OPENING_PAGE_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + Ok(()) +} + +/// Write the optional 28-entry table used by a later populated control state. +pub(crate) fn apply_g17p_populated_operand_table(out: &mut [u8]) -> Result<(), SubmissionError> { + apply_g17p_cold_opening_operand_table(out)?; + for index in 0..G17P_COLD_OPENING_OPERAND_BUFFER_COUNT { + let address = G17P_COLD_OPENING_OPERAND_BUFFER_BASE + .checked_add(index as u64 * G17P_COLD_OPENING_OPERAND_BUFFER_STRIDE) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?; + put_u64_le(out, index * 0x40, address | (1u64 << 60)); + } + Ok(()) +} + +/// Keep the four-page operand directory zero through cold first work. +pub(crate) fn apply_g17p_cold_opening_operand_directory( + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_OPERAND_DIRECTORY_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + Ok(()) +} + +/// Write the optional populated directory for the 28 operand buffers. +pub(crate) fn apply_g17p_populated_operand_directory( + out: &mut [u8], +) -> Result<(), SubmissionError> { + apply_g17p_cold_opening_operand_directory(out)?; + let pages_per_buffer = G17P_COLD_OPENING_OPERAND_BUFFER_SIZE / 0x1000; + let mut qword = 0usize; + for buffer in 0..G17P_COLD_OPENING_OPERAND_BUFFER_COUNT { + let base = G17P_COLD_OPENING_OPERAND_BUFFER_BASE + .checked_add(buffer as u64 * G17P_COLD_OPENING_OPERAND_BUFFER_STRIDE) + .ok_or(SubmissionError::PartialOpeningAddressOverflow)?; + for page in 0..pages_per_buffer { + put_u64_le(out, qword * 8, base + page * 0x1000); + qword += 1; + } + } + Ok(()) +} + +/// Write the queue-context page for the clean first-partial TA/3D pair. +/// +/// This is the new local grid-0/1 generation selected by +/// `G17P_PARTIAL_OPENING_GRAPH=1`. Its namespace tag and fragment locators are +/// distinct from the untagged full-opening pair-zero context. +pub(crate) fn apply_g17p_cold_opening_queue_context( + stage: G17PColdOpeningStage, + descriptor: u64, + queue: u64, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != G17P_COLD_OPENING_QUEUE_CONTEXT_SIZE { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + out.fill(0); + let (descriptor_base, words, locators): (u64, &[(usize, u64)], &[(usize, u64)]) = match stage { + G17PColdOpeningStage::Tiling => ( + G17P_COLD_OPENING_TA_DESCRIPTOR_BASE, + &[ + (0x200, 0x1000_0000_0000_0004), + (0x220, 0xffff_0c00_0000_0001), + (0x378, 0x003f_ffff_ffff_ffff), + ], + &[(0x350, 0x0002_3803_8000_0003)], + ), + G17PColdOpeningStage::Fragment => ( + G17P_COLD_OPENING_3D_DESCRIPTOR_BASE, + &[ + (0x200, 0x1000_0400_0000_0004), + (0x220, 0xffff_1800_0000_0003), + (0x228, 0x0000_0000_0000_0001), + (0x230, 0x0000_0100_0000_0000), + (0x378, 0x003f_ffff_ffff_ffff), + ], + &[ + (0x350, 0x0002_b003_8000_4c05), + (0x358, 0x0000_8003_8000_4c3e), + (0x360, 0x0000_b803_8000_4c77), + (0x368, 0x0000_5003_8000_4cb0), + ], + ), + }; + let descriptor_delta = descriptor + .checked_sub(descriptor_base) + .ok_or(SubmissionError::PartialOpeningDescriptorLocatorUnencodable)?; + if descriptor_delta & 0x1f != 0 { + return Err(SubmissionError::PartialOpeningDescriptorLocatorUnencodable); + } + let locator_delta = descriptor_delta / 0x20; + for (offset, value) in words { + put_u64_le(out, *offset, *value); + } + for (offset, value) in locators { + put_u64_le( + out, + *offset, + value + .checked_add(locator_delta) + .ok_or(SubmissionError::PartialOpeningDescriptorLocatorUnencodable)?, + ); + } + put_u64_le(out, 0x210, descriptor); + put_u64_le(out, 0x218, queue); + Ok(()) +} + +/// Derive the five stable status-B writes presented before control 0x84. +pub(crate) const fn prepare_g17p_partial_opening_pre_0x84_status( + kernel_va_base: u64, +) -> Result<[G17PPartialOpeningStatusWrite; 5], SubmissionError> { + let fwctl_va = match kernel_va_base.checked_add(G17P_PARTIAL_OPENING_FWCTL_OFFSET) { + Some(value) => value, + None => return Err(SubmissionError::PartialOpeningAddressOverflow), + }; + let fwctl_next = match fwctl_va.checked_add(G17P_PARTIAL_OPENING_CONTROL_MESSAGE_SIZE) { + Some(value) => value, + None => return Err(SubmissionError::PartialOpeningAddressOverflow), + }; + Ok([ + G17PPartialOpeningStatusWrite { + offset: 0x4018, + value: 0x0005_0601_0000_0000, + }, + G17PPartialOpeningStatusWrite { + offset: 0x4020, + value: 61, + }, + G17PPartialOpeningStatusWrite { + offset: 0x40b0, + value: 5, + }, + G17PPartialOpeningStatusWrite { + offset: 0x48e0, + value: fwctl_va, + }, + G17PPartialOpeningStatusWrite { + offset: 0x48e8, + value: fwctl_next, + }, + ]) +} + +fn prepare_g17p_partial_opening_outer_slot( + state: G17PPartialOpeningOuterChannelState, + channel_table_index: u8, + queue_grid_index: u8, // the QID this slot dispatches to + kind: u32, + channel_started: bool, +) -> Result { + if state.producer >= G17P_PARTIAL_OPENING_OUTER_SLOT_COUNT + || state + .consumers + .iter() + .any(|consumer| *consumer >= G17P_PARTIAL_OPENING_OUTER_SLOT_COUNT) + { + return Err(SubmissionError::PartialOpeningChannelCounterOutOfRange); + } + if state.queue_write_index > u16::MAX as u32 { + return Err(SubmissionError::PartialOpeningQueueHeadOutOfRange); + } + + let next_producer = (state.producer + 1) & 0xff; + if state.consumers.contains(&next_producer) { + return Err(SubmissionError::RingFull); + } + + let mut slot = [0u8; G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE]; + put_u64_le(&mut slot, 0x08, state.queue_gpu_va); + put_u32_le(&mut slot, 0x10, kind); + // +0x14 is [u16 write_index][u8 queue_id][u8 1]. The middle byte is the + // QUEUE ID, not a separate "grid index" -- macOS's own + // AGFIAcceleratorDataMasterEntry slot carries 0x11 (TA) and 0x12 (3D) + // there, matching the QIDs its tag-15 publishes, and the rest of the + // 0x18-byte entry matches this encoder field for field: + // + // +0x00 zero +0x00 zero + // +0x08 queue record VA +0x08 state.queue_gpu_va + // +0x10 data master (TA 0 / 3D 1) +0x10 kind (PipeType Vertex/Fragment) + // +0x14 0x01_11_0003 / 0x01_12_0003 write_index 3, qid, bit24 set + // + // This byte must agree with the QID published in tag-15, tag-14 and the + // SKSM registration. It follows the retained queue object even when a + // recovered compute session advances past its first QID. + // + // +0x17 is NOT a constant. `AGXArmFirmware::encodeAcceleratorRingCommand` + // fills this entry as `+0x08 chanstate fwva, +0x10 type, +0x14 info[0], + // +0x16 qid, +0x17 !chan[0x3c]` -- the last byte is the logical NOT of a + // channel state word, so it is 0 once that word is set. MEASURED on macOS + // 2026-09-02: a live Metal render's data-master ring entries carry 0x00 + // there for BOTH stages (`00 3c 00 10 .. 39 01 08 00` for 3D and + // `.. 43 01 07 00` for TA), across two independent captures with different + // queue ids. This driver hardcoded bit 24, i.e. it told firmware the + // channel state word was clear on every single submission. + put_u32_le( + &mut slot, + 0x14, + state.queue_write_index + | ((queue_grid_index as u32) << 16) + | (if channel_started { 0 } else { 1 << 24 }), + ); + + Ok(G17PPartialOpeningOuterSlotPlan { + channel_table_index, + slot_index: state.producer as u8, + slot_byte_offset: state.producer * G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE as u32, + slot, + next_producer, + }) +} + +pub(crate) fn prepare_g17p_compute_channel_work( + state: G17PPartialOpeningOuterChannelState, + queue_id: u8, + channel_started: bool, +) -> Result { + // Match AGXArmFirmware::encodeAcceleratorRingCommand for every pipe: + // +0x17 is !AGXChannel[0x3c], so it is one only for the first publication + // of a physical channel and zero after that channel has started. + let outer = prepare_g17p_partial_opening_outer_slot( + state, + G17P_COMPUTE_CHANNEL_TABLE_INDEX, + queue_id, + PipeType::Compute as u32, + channel_started, + )?; + Ok(PreparedG17PComputeChannelWork { + outer, + doorbell: G17P_COMPUTE_WORK_DOORBELL, + }) +} + +/// Couple the fragment-first/tiling-second outer slots to their one channel-0 +/// 0x83 notification. These are the initdata work rings at table indices 1 and +/// 0, not the kext's separate 8-byte command-pointer rings. The caller performs +/// the barriers in [`G17P_PARTIAL_OPENING_PUBLICATION_ORDER`]. +pub(crate) fn prepare_g17p_partial_opening_work_pair( + fragment: G17PPartialOpeningOuterChannelState, + tiling: G17PPartialOpeningOuterChannelState, + fragment_queue_id: u8, + tiling_queue_id: u8, + first_submit: bool, +) -> Result { + // `+0x17` is `!AGXChannel[0x3c]`: one on the first publication of a + // physical queue and zero after it has started. Every valid native queue + // follows 1 -> 0; a module-wide constant encoded one side incorrectly. + let render_channel_started = !first_submit; + let fragment = match prepare_g17p_partial_opening_outer_slot( + fragment, + G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX, + fragment_queue_id, + 1, + render_channel_started, + ) { + Ok(value) => value, + Err(error) => return Err(error), + }; + let tiling = match prepare_g17p_partial_opening_outer_slot( + tiling, + G17P_PARTIAL_OPENING_TILING_CHANNEL_TABLE_INDEX, + tiling_queue_id, + 0, + render_channel_started, + ) { + Ok(value) => value, + Err(error) => return Err(error), + }; + Ok(PreparedG17PPartialOpeningWorkPair { + fragment, + tiling, + paired_doorbell: G17P_PARTIAL_OPENING_PAIRED_DOORBELL, + }) +} + +/// Layout-only forms used to make accidental field drift a compile-time +/// error. The intervening words remain unnamed because their semantics are +/// outside the admitted producer/consumer subset. +#[allow(dead_code)] +#[repr(C)] +struct G17PCachedChannelStateLayoutAssert { + uncached_state_gpu_va: u64, + command_pointer_ring_gpu_va: u64, + _unknown_10: u64, + _unknown_18: u32, + coherency_cursor: u32, + _unknown_20: [u8; 0x60], +} + +#[allow(dead_code)] +#[repr(C)] +struct G17PUncachedChannelStateLayoutAssert { + consumer: u32, + _unknown_04: [u8; 0x0c], + _unknown_10: u32, + _unknown_14: [u8; 0x0c], + _unknown_20: u32, + _unknown_24: [u8; 0x0c], + coherency_shadow: u32, + _unknown_34: [u8; 0x0c], + producer: u32, + _unknown_44: [u8; 0x0c], + recovery_target: u32, + _unknown_54: [u8; 0x0c], + ring_entry_count: u32, +} + +const _: () = { + assert!(core::mem::size_of::() == 0x80); + assert!( + core::mem::offset_of!(G17PCachedChannelStateLayoutAssert, uncached_state_gpu_va) == 0x00 + ); + assert!( + core::mem::offset_of!( + G17PCachedChannelStateLayoutAssert, + command_pointer_ring_gpu_va + ) == 0x08 + ); + assert!(core::mem::offset_of!(G17PCachedChannelStateLayoutAssert, coherency_cursor) == 0x1c); + + assert!(core::mem::size_of::() == 0x64); + assert!(core::mem::offset_of!(G17PUncachedChannelStateLayoutAssert, consumer) == 0x00); + assert!(core::mem::offset_of!(G17PUncachedChannelStateLayoutAssert, coherency_shadow) == 0x30); + assert!(core::mem::offset_of!(G17PUncachedChannelStateLayoutAssert, producer) == 0x40); + assert!(core::mem::offset_of!(G17PUncachedChannelStateLayoutAssert, recovery_target) == 0x50); + assert!(core::mem::offset_of!(G17PUncachedChannelStateLayoutAssert, ring_entry_count) == 0x60); +}; + +// --------------------------------------------------------------------------- +// 3. Legacy AGX2 work-queue RingState. The A18 Pro ARM firmware directly +// confirms only done@0; the SKSM hardware consumer keeps the remaining +// rptr/wptr/size offsets unconfirmed, so this record is not admitted as a +// native G17P layout. +// --------------------------------------------------------------------------- + +pub(crate) const AGX2_RING_STATE_GPU_DONEPTR_OFFSET: u32 = 0x00; +pub(crate) const AGX2_RING_STATE_GPU_RPTR_OFFSET: u32 = 0x30; +pub(crate) const AGX2_RING_STATE_CPU_WPTR_OFFSET: u32 = 0x40; +pub(crate) const AGX2_RING_STATE_RB_SIZE_OFFSET: u32 = 0x50; + +/// AGX2 work-queue entries are u64 command GPU virtual addresses. +pub(crate) const AGX2_RING_ENTRY_BYTES: u32 = 8; + +/// The shipping producer's maximum outstanding event slots +/// (`workqueue.rs::MAX_JOB_SLOTS`). This is not the command-ring depth: the +/// current Linux ring has `WQ_SIZE == 0x500`. Its numerical equality with the +/// 7-bit QID maximum is only a coincidence and carries no ABI meaning. +pub(crate) const HOST_MAX_OUTSTANDING_JOB_SLOTS: u32 = 127; + +/// Advance a producer/consumer index within a ring of `size` entries. +/// +/// Return an error for a zero-sized ring or a producer outside the ring. A +/// valid `index` is always below `u32::MAX`, so `index + 1` cannot overflow +/// after these checks. +pub(crate) const fn ring_next(index: u32, size: u32) -> Result { + if size == 0 { + return Err(SubmissionError::RingSizeZero); + } + if index >= size { + return Err(SubmissionError::RingIndexOutOfRange); + } + Ok((index + 1) % size) +} + +// --------------------------------------------------------------------------- +// 4. SKSM kick word + per-QID kick-record table (AGX3-specific). +// Grounded from GFX 0x10c88-0x10d2c. The canonical kick-word *encoder* lives +// in `g17_rtkit::KickWord`; these constants are the same firmware bytes, +// re-cited so this module stands alone. `g17_boot` cross-checks them against +// `g17_rtkit` to prove no drift. +// --------------------------------------------------------------------------- + +/// TS occupies bits [39:0] (`and x0,x5,#0xffffffffff` at 25G83 A000 0xfd18). 40-bit, per +/// the firmware print `"TS=0x%010llx"`. +pub(crate) const KICK_TS_BITS: u32 = 40; +pub(crate) const KICK_TS_MASK: u64 = (1u64 << KICK_TS_BITS) - 1; +/// QID occupies bits [46:40] (`ubfx x1,x5,#40,#7` @ 0x10c90). +pub(crate) const KICK_QID_SHIFT: u32 = 40; +pub(crate) const KICK_QID_BITS: u32 = 7; +pub(crate) const KICK_QID_MASK: u8 = (1u8 << KICK_QID_BITS) - 1; +/// Bit 46 also selects one of two 64-bit valid-queue mask words +/// (`ubfx x6,x5,#46,#1` @ 0x10c9c). +pub(crate) const KICK_SELECTOR_BIT: u32 = 46; +/// Bits [63:47] are unmodeled and must be zero. +pub(crate) const KICK_RESERVED_MASK: u64 = !((1u64 << (KICK_SELECTOR_BIT + 1)) - 1); + +/// Per-QID kick-record table base offset from the valid-mask base +/// (`add x2,x22,#0x10` @ 0x10c94). +pub(crate) const KICK_RECORD_TABLE_BASE: u32 = 0x10; +/// Per-QID record stride (`add x2,x2,x1,lsl#5` @ 0x10c98). +pub(crate) const KICK_RECORD_STRIDE: u32 = 0x20; +/// 128 queue slots: 7-bit QID, two 64-bit valid-mask words selected by bit 46. +pub(crate) const KICK_RECORD_COUNT: u32 = 128; + +/// Fields of one 0x20-byte per-QID kick record (written across 0x10cb0-0x10d2c). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct KickRecordLayout { + /// `+0x00` u64: first/min TS seen for this QID. + pub(crate) ts_first: u32, + /// `+0x08` u64: last/max TS seen for this QID. + pub(crate) ts_last: u32, + /// `+0x10` u64: pending kick-slot bitmask. + pub(crate) slot_mask: u32, + /// `+0x18` u32: valid/first flag. + pub(crate) valid_flag: u32, + /// `+0x1c` u32: winning kick-slot index. + pub(crate) slot_index: u32, +} + +/// The exact offsets, so a change to the firmware layout fails a test. +pub(crate) const KICK_RECORD_LAYOUT: KickRecordLayout = KickRecordLayout { + ts_first: 0x00, + ts_last: 0x08, + slot_mask: 0x10, + valid_flag: 0x18, + slot_index: 0x1c, +}; + +/// Byte offset of one firmware kick record: `0x10 + QID * 0x20`. Mirrors +/// `g17_rtkit::kick_queue_record_offset`. +pub(crate) const fn kick_record_offset(queue_id: u8) -> Result { + if queue_id > KICK_QID_MASK { + Err(SubmissionError::QueueIdOutOfRange) + } else { + Ok(KICK_RECORD_TABLE_BASE + queue_id as u32 * KICK_RECORD_STRIDE) + } +} + +/// The firmware-visible fields of a decoded SKSM kick word. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SksmKickWord { + pub(crate) timestamp: u64, + pub(crate) queue_id: u8, + pub(crate) mask_selector: u8, +} + +/// Encode the exact raw SKSM kick fields, leaving bits [63:47] clear. +pub(crate) const fn encode_kick_word(timestamp: u64, queue_id: u8) -> Result { + if timestamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + if queue_id > KICK_QID_MASK { + return Err(SubmissionError::QueueIdOutOfRange); + } + Ok(timestamp | ((queue_id as u64) << KICK_QID_SHIFT)) +} + +/// Decode a kick word with the firmware's exact bitfields, refusing the +/// unmodeled upper bits. Kept parallel to `g17_rtkit::KickWord::decode_strict`. +pub(crate) const fn decode_kick_word(word: u64) -> Result { + if word & KICK_RESERVED_MASK != 0 { + return Err(SubmissionError::KickWordUnmodeledBits); + } + Ok(SksmKickWord { + timestamp: word & KICK_TS_MASK, + queue_id: ((word >> KICK_QID_SHIFT) as u8) & KICK_QID_MASK, + mask_selector: ((word >> KICK_SELECTOR_BIT) & 1) as u8, + }) +} + +// --------------------------------------------------------------------------- +// 5. Host SKSM kick-queue producer transaction (AGX3-specific). +// +// Grounded from G17S KSMKickQueue::encodeKickEntry at host text +// 0xfffffe0008b51f74-0xfffffe0008b521c4. The current timestamp at queue +// +0x48 selects `entry_base + low8(timestamp) * entry_stride`; the current +// 40-bit timestamp is returned for the doorbell, then the low-byte slot is +// advanced modulo runtime `num_stamps`. Wrapping the slot to zero increments +// the 32-bit stamp in bits [39:8]. This coupling is load-bearing: choosing a +// different entry slot from the one encoded in the doorbell desynchronizes +// the queue and can reach the firmware's out-of-order fatal path. +// --------------------------------------------------------------------------- + +/// Stamp sequence in bits [39:8] of a host SKSM timestamp. +pub(crate) const KICK_STAMP_SHIFT: u32 = 8; +pub(crate) const KICK_STAMP_MASK: u64 = u32::MAX as u64; +/// Entry/stamp slot in bits [7:0] of a host SKSM timestamp. +pub(crate) const KICK_STAMP_INDEX_MASK: u64 = 0xff; +/// The low-byte slot representation permits at most 256 runtime stamps. +pub(crate) const KICK_NUM_STAMPS_MAX: u16 = 256; + +/// Decoded host-side timestamp state before a kick is published. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct KickTimestamp { + pub(crate) stamp: u32, + pub(crate) stamp_index: u8, +} + +/// Pure result of preparing one host SKSM kick-queue producer transaction. +/// +/// The caller must still encode the complete target-specific entry, execute +/// the recovered `dmb ish`, update the submitted counter, and perform the live +/// target-specific `addKicks` MMIO publication. This structure only makes the +/// entry selection, queue identity, and next producer timestamp indivisible. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedKickSubmission { + /// Slot selected from the current timestamp's low byte. + pub(crate) entry_index: u8, + /// Byte offset from the runtime queue entry base. + pub(crate) entry_offset: u32, + /// Current 40-bit timestamp consumed by this entry. + pub(crate) timestamp: u64, + /// Queue whose target-specific `addKicks` tag must be published later. + pub(crate) queue_id: u8, + /// Timestamp stored at queue +0x48 for the following submission. + pub(crate) next_timestamp: u64, +} + +/// Decode the exact host timestamp split: stamp[31:0] followed by slot[7:0]. +pub(crate) const fn decode_kick_timestamp( + timestamp: u64, +) -> Result { + if timestamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + + Ok(KickTimestamp { + stamp: ((timestamp >> KICK_STAMP_SHIFT) & KICK_STAMP_MASK) as u32, + stamp_index: (timestamp & KICK_STAMP_INDEX_MASK) as u8, + }) +} + +/// Encode the host timestamp layout without a queue ID. +pub(crate) const fn encode_kick_timestamp(timestamp: KickTimestamp) -> u64 { + ((timestamp.stamp as u64) << KICK_STAMP_SHIFT) | timestamp.stamp_index as u64 +} + +/// Low-byte payload of the retained render channel's next submission. +/// +/// In the inspected 25G83 kext, FRG submit increments channel +0x204 +/// (8837ae0..e8); getter 88b4ee8 returns its low byte. The counter follows +/// channel-object lifetime, independently of its physical timestamp base. +/// Linux retains one ordinal per physical pair across its storage handoffs, +/// advancing once per publication; outstanding submissions need not retire +/// before this counter advances. The byte wraps independently of the 40-bit +/// SKSM timestamp and is neither a QID nor an address-space tag. +pub(crate) const fn g17p_serialized_render_scratch_payload(submission_ordinal: u64) -> u8 { + submission_ordinal.wrapping_add(1) as u8 +} + +/// Pack `AGX3DCommandDescriptor::getSKSMCompletionScratchRegValue()`. +/// +/// The kext stores this value in both its host descriptor and the emitted GPU +/// command at +0x2160. Bits [39:0] are the command's SKSM queue timestamp and +/// bits [47:40] are the 3D-channel scratch payload. The native full-command +/// witness `0x0000010000000f76` therefore decodes as payload one and timestamp +/// `0xf76`; it is categorically not a USC FList pointer. +pub(crate) const fn encode_g17p_3d_completion_scratch( + timestamp: u64, + payload: u8, +) -> Result { + if timestamp & !KICK_TS_MASK != 0 { + Err(SubmissionError::KickTimestampOutOfRange) + } else { + Ok(timestamp | ((payload as u64) << KICK_QID_SHIFT)) + } +} + +/// ConfigUpdate +0x26 gates the u64 completion seed at +0x2a (B1 +/// 11434..11470). Busy channels preserve firmware's evolving scratch value. +/// Native emits both fields with u32/u64 stores; validate before mutation. +pub(crate) fn apply_g17p_idle_scratch_seed( + out: &mut [u8], seed: Option, +) -> Result<(), SubmissionError> { + if out.len() < 0x32 { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + put_u32_le(out, 0x26, u32::from(seed.is_some())); + put_u64_le(out, 0x2a, seed.unwrap_or(0)); + Ok(()) +} + +/// Prepare the exact entry-slot/doorbell/next-timestamp tuple for one kick. +/// +/// `num_stamps` and `entry_stride` are intentionally runtime inputs: Apple's +/// host loads both from the per-queue geometry rather than hard-coding them. +/// Invalid geometry returns an error before an entry address or doorbell is +/// produced. No shared memory, counter, barrier, MMIO, or endpoint state is +/// mutated here. +pub(crate) const fn prepare_kick_submission( + current_timestamp: u64, + queue_id: u8, + num_stamps: u16, + entry_stride: u32, +) -> Result { + let current = match decode_kick_timestamp(current_timestamp) { + Ok(timestamp) => timestamp, + Err(err) => return Err(err), + }; + + if num_stamps == 0 || num_stamps > KICK_NUM_STAMPS_MAX { + return Err(SubmissionError::KickStampCountOutOfRange); + } + if current.stamp_index as u16 >= num_stamps { + return Err(SubmissionError::KickStampIndexOutOfRange); + } + if entry_stride == 0 { + return Err(SubmissionError::KickEntryStrideZero); + } + + let index = current.stamp_index as u32; + if index != 0 && entry_stride > u32::MAX / index { + return Err(SubmissionError::KickEntryOffsetOverflow); + } + let entry_offset = index * entry_stride; + + if queue_id > KICK_QID_MASK as u8 { + return Err(SubmissionError::QueueIdOutOfRange); + } + + let next_index = ((current.stamp_index as u16 + 1) % num_stamps) as u8; + let next_stamp = if next_index == 0 { + current.stamp.wrapping_add(1) + } else { + current.stamp + }; + let next_timestamp = encode_kick_timestamp(KickTimestamp { + stamp: next_stamp, + stamp_index: next_index, + }); + + Ok(PreparedKickSubmission { + entry_index: current.stamp_index, + entry_offset, + timestamp: current_timestamp, + queue_id, + next_timestamp, + }) +} + +// --------------------------------------------------------------------------- +// 5a. A18 Pro / G17P CL kick-entry body and host queue-state transition. +// +// `AGXKSMKickQueueG16::encodeCLKickEntry` at +// 0xfffffe0008a56848-0xfffffe0008a56a64 gathers the CL descriptor inputs. +// Its shared encoder at 0xfffffe0008a56a68-0xfffffe0008a56eb4 writes the +// complete 0x180-byte body described below. The recovered G17S encoder at +// 0xfffffe0008b51f74-0xfffffe0008b521c4 independently confirms the slot, +// timestamp, barrier, and counter tail; scratch tags remain target-specific. +// +// The body is the first 0x180 bytes of each 0x200-spaced queue-context +// record captured by m1n1. The host zeroes the runtime stride before these +// stores, so every byte not named here stays zero. +// --------------------------------------------------------------------------- + +pub(crate) const G17P_CL_KICK_ENTRY_SIZE: usize = 0x180; +pub(crate) const G17P_CL_DEPENDENCIES_MAX: usize = 32; +pub(crate) const G17P_CL_EXPLICIT_DEPENDENCIES_MAX: usize = G17P_CL_DEPENDENCIES_MAX - 1; +const G17P_CL_PACKED_ADDRESS_MASK: u64 = (1u64 << 38) - 1; + +/// Proven allocation and timestamp geometry for one target/backend pair. +/// +/// Only A18 Pro's G17P HAL200 hybrid has an admitted tuple. Classic and +/// HAL300 remain deliberately absent until their own host contracts are +/// recovered. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17SksmQueueGeometry { + target: G17ChannelAbiTarget, + backend: G17ClSubmissionBackend, + allocation_alignment: u64, + entry_stride: u32, + stamp_count: u16, + timestamp_seed: u64, + backing_size: usize, +} + +impl G17SksmQueueGeometry { + pub(crate) const fn validate(self) -> Result<(), SubmissionError> { + if !matches!( + (self.target, self.backend), + ( + G17ChannelAbiTarget::A18ProG17P, + G17ClSubmissionBackend::G17PHal200Hybrid + ) + ) { + return Err(SubmissionError::KickQueueGeometryUnconfirmed); + } + if self.allocation_alignment == 0 + || self.allocation_alignment & (self.allocation_alignment - 1) != 0 + || self.entry_stride < G17P_CL_KICK_ENTRY_SIZE as u32 + || self.stamp_count == 0 + || self.stamp_count > KICK_NUM_STAMPS_MAX + || self.timestamp_seed & !KICK_TS_MASK != 0 + || (self.timestamp_seed & KICK_STAMP_INDEX_MASK) >= self.stamp_count as u64 + || self.backing_size as u64 + != self.entry_stride as u64 * self.stamp_count as u64 + || self.backing_size as u64 % self.allocation_alignment != 0 + { + return Err(SubmissionError::KickQueueGeometryInvalid); + } + Ok(()) + } + + pub(crate) const fn allocation_alignment(self) -> u64 { + self.allocation_alignment + } + + pub(crate) const fn mapping_alignment(self, minimum: u64) -> u64 { + if self.allocation_alignment > minimum { + self.allocation_alignment + } else { + minimum + } + } + + pub(crate) const fn entry_stride(self) -> u32 { + self.entry_stride + } + + pub(crate) const fn stamp_count(self) -> u16 { + self.stamp_count + } + + pub(crate) const fn timestamp_seed(self) -> u64 { + self.timestamp_seed + } + + pub(crate) const fn backing_size(self) -> usize { + self.backing_size + } +} + +pub(crate) const T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY: G17SksmQueueGeometry = + G17SksmQueueGeometry { + target: G17ChannelAbiTarget::A18ProG17P, + backend: G17ClSubmissionBackend::G17PHal200Hybrid, + allocation_alignment: 0x100, + entry_stride: 0x200, + stamp_count: 0x100, + timestamp_seed: 1, + backing_size: 0x20000, + }; + +pub(crate) const fn g17_sksm_queue_geometry( + target: G17ChannelAbiTarget, + backend: G17ClSubmissionBackend, +) -> Option { + match (target, backend) { + (G17ChannelAbiTarget::A18ProG17P, G17ClSubmissionBackend::G17PHal200Hybrid) => { + Some(T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY) + } + _ => None, + } +} + +const _: () = assert!(T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.entry_stride as usize + * T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.stamp_count as usize + == T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.backing_size); +const _: () = assert!(T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.entry_stride as usize + >= G17P_CL_KICK_ENTRY_SIZE); +const _: () = assert!(T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.allocation_alignment == 0x100); +const _: () = assert!(T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.timestamp_seed == 1); + +/// Host-object fields used while selecting, publishing, and retiring one CL +/// kick entry. These are CPU-object offsets, not firmware queue-record offsets. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClKickQueueStateLayout { + pub(crate) queue_id: usize, + pub(crate) num_stamps: usize, + pub(crate) entry_stride: usize, + pub(crate) completion_selector: usize, + pub(crate) submitted: usize, + pub(crate) current_timestamp: usize, + pub(crate) last_add_kicks_timestamp: usize, + pub(crate) entry_cpu_base: usize, + pub(crate) entry_gpu_base: usize, + pub(crate) base_timestamp: usize, + pub(crate) previous_valid: usize, + pub(crate) previous_timestamp: usize, +} + +pub(crate) const G17P_CL_KICK_QUEUE_STATE_LAYOUT: G17PClKickQueueStateLayout = + G17PClKickQueueStateLayout { + queue_id: 0x0c, + num_stamps: 0x10, + entry_stride: 0x14, + completion_selector: 0x1c, + submitted: 0x44, + current_timestamp: 0x48, + last_add_kicks_timestamp: 0x50, + entry_cpu_base: 0x78, + entry_gpu_base: 0x80, + base_timestamp: 0x90, + previous_valid: 0x98, + previous_timestamp: 0xa0, + }; + +/// Fixed byte positions inside the 0x180-byte G17P CL entry body. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClKickEntryLayout { + pub(crate) header: usize, + pub(crate) mcache: usize, + pub(crate) payload: [usize; 2], + pub(crate) dependency_control: usize, + pub(crate) dependencies: usize, + pub(crate) event_mask: [usize; 4], + pub(crate) rce_kind: usize, + pub(crate) rce_bindings: [usize; 4], + pub(crate) zero_170: usize, + pub(crate) auxiliary: usize, +} + +pub(crate) const G17P_CL_KICK_ENTRY_LAYOUT: G17PClKickEntryLayout = G17PClKickEntryLayout { + header: 0x00, + mcache: 0x08, + payload: [0x10, 0x18], + dependency_control: 0x20, + dependencies: 0x28, + event_mask: [0x128, 0x130, 0x138, 0x140], + rce_kind: 0x148, + rce_bindings: [0x150, 0x158, 0x160, 0x168], + zero_170: 0x170, + auxiliary: 0x178, +}; + +/// One explicit barrier dependency after its descriptor-selected shared stamp +/// has been loaded. `event_word` is barrier-record word zero; record word one +/// selects the shared-stamp source and has no independent wire field. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClBarrierDependency { + /// Already converted native predecessor stamp. Never reinterpret it using + /// the consumer queue's base timestamp or ring geometry. + pub(crate) native_timestamp: Option, + pub(crate) event_word: u32, + pub(crate) shared_stamp: u64, + pub(crate) queue_id: u8, +} + +/// Represent a render dependency using the native event word plus epoch. +/// The event value advances by 0x100 and wraps as on M1/M2. Its high timestamp +/// bits are carried separately; dropping them makes a dependency at 2^24 +/// name the wrong kick even if AddKicks itself preserves all forty bits. +pub(crate) fn g17p_render_barrier_from_timestamp( + timestamp: u64, + queue_id: u8, +) -> Result { + if timestamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + if queue_id > KICK_QID_MASK { + return Err(SubmissionError::KickDependencyQueueIdOutOfRange); + } + Ok(G17PClBarrierDependency { + native_timestamp: None, + event_word: (timestamp as u32).wrapping_shl(8), + shared_stamp: timestamp >> 24, + queue_id, + }) +} + +/// Logical render prefix, resolved at ioctl admission before asynchronous work. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] +pub(crate) struct G17PRenderOrder { + pub(crate) sequence: u64, + pub(crate) wait_through: Option, +} + +/// Zero snapshots all preceding ioctls; N adds the first N renders of this +/// ioctl. NONE remains independent. Sequence zero is reserved for internal work. +pub(crate) fn g17p_render_order( + ioctl_base: u64, preceding: u64, barrier: u16, +) -> Result { + let sequence = ioctl_base.checked_add(preceding) + .and_then(|n| n.checked_add(1)) + .ok_or(SubmissionError::RenderBarrierSequenceOverflow)?; + let wait_through = if barrier == u16::MAX { + None + } else { + if u64::from(barrier) > preceding { + return Err(SubmissionError::RenderBarrierFuture); + } + Some(ioctl_base.checked_add(u64::from(barrier)) + .ok_or(SubmissionError::RenderBarrierSequenceOverflow)?) + }; + Ok(G17PRenderOrder { sequence, wait_through }) +} + +/// Add an exact historical prefix without broadening it to the ioctl base. +/// The base is frozen for the whole ioctl, before any of its jobs publish; +/// even an earlier command in this ioctl cannot satisfy a historical ordinal. +/// Zero is absent, including when the legacy header is NONE. +pub(crate) fn g17p_render_order_with_prior( + ioctl_base: u64, preceding: u64, barrier: u16, prior: u64, +) -> Result { + if prior > ioctl_base { + return Err(SubmissionError::PriorBarrierFuture); + } + let mut order = g17p_render_order(ioctl_base, preceding, barrier)?; + if prior != 0 { + order.wait_through = Some(order.wait_through.map_or(prior, |p| p.max(prior))); + } + Ok(order) +} + +/// Resolve a still-owned predecessor. A successful retirement needs no GPU +/// dependency; errors and an unassigned token must never satisfy the prefix. +/// The caller filters owner identity and supplies every live predecessor in +/// the prefix under the same runtime lock as publication and retirement. +pub(crate) fn g17p_live_render_dependency( + order: G17PRenderOrder, predecessor_sequence: u64, + predecessor_status: i32, native_word: u64, +) -> Result, SubmissionError> { + if !g17p_render_prefix_contains(order.wait_through, predecessor_sequence) { + return Ok(None); + } + if predecessor_status < 0 { + return Err(SubmissionError::RenderBarrierPredecessorFailed); + } + if predecessor_status > 0 { return Ok(None); } + if native_word == u64::MAX { + return Err(SubmissionError::RenderBarrierMissingToken); + } + // Bit 46 is QID bit 6: the firmware takes the QID as bits [46:40] (ubfx + // x1,x5,#40,#7) and the same bit as its valid-queue mask-word selector + // (ubfx x6,x5,#46,#1). A predecessor on QID 64..127 sets it; its token is + // fully modeled and the dependency encoder takes all seven QID bits. + // decode_kick_word still refuses bits [63:47]. Refusing the selector made + // every barrier on a 64..127 fragment QID fail the dependent render with + // EIO (the dead-pair DeviceLost). + let token = decode_kick_word(native_word)?; + let mut dependency = g17p_render_barrier_from_timestamp(token.timestamp, token.queue_id)?; + dependency.native_timestamp = Some(token.timestamp); + Ok(Some(dependency)) +} + +pub(crate) fn g17p_render_prefix_contains(prefix: Option, sequence: u64) -> bool { + sequence != 0 && prefix.is_some_and(|limit| sequence <= limit) +} + +/// Collapse a validated logical prefix to its newest kick per hardware QID. +/// Each QID's implicit previous-kick parent supplies the transitive order. +/// Use logical sequence, not wrapped firmware timestamps or scan order, to +/// choose the newest. Callers must check every covered predecessor for errors +/// before inserting it, including predecessors this frontier will subsume. +pub(crate) fn g17p_insert_prefix_dependency( + dependencies: &mut [G17PClBarrierDependency], + sequences: &mut [u64], count: &mut usize, + sequence: u64, dependency: G17PClBarrierDependency, +) -> Result<(), SubmissionError> { + if *count > dependencies.len() || *count > sequences.len() { + return Err(SubmissionError::KickDependencyCountOutOfRange); + } + if let Some(index) = dependencies[..*count].iter() + .position(|prior| prior.queue_id == dependency.queue_id) + { + if sequence > sequences[index] { + dependencies[index] = dependency; + sequences[index] = sequence; + } + return Ok(()); + } + if *count == dependencies.len() || *count == sequences.len() { + return Err(SubmissionError::KickDependencyCountOutOfRange); + } + dependencies[*count] = dependency; + sequences[*count] = sequence; + *count += 1; + Ok(()) +} + +/// Optional MCache aperture descriptor copied into entry +0x08. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClMcacheAperture { + pub(crate) address: u64, + pub(crate) index: u8, + pub(crate) count: u8, +} + +/// One of the four address/tag bindings written at +0x150..+0x168. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClRceBinding { + pub(crate) address: u64, + pub(crate) tag: u8, +} + +/// Command-relative CL fields witnessed on consecutive output-positive +/// compute entries. The descriptor slot stride remains a storage decision; +/// both RCE addresses are derived from the current slot's low alias. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PComputeClCommandState { + pub(crate) event_mask: [u64; 4], + pub(crate) rce_bindings: [G17PClRceBinding; 4], +} + +/// Host-written prefix of the conditional tag-14 fallback command. +pub(crate) const G17P_COMPUTE_PRIMARY_EVENT_SIZE: usize = 0x15; +/// Host-written prefix of the descriptor-selected tag-16 command. +pub(crate) const G17P_COMPUTE_OPTIONAL_EVENT_SIZE: usize = 0x12; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PComputePrimaryEvent { + pub(crate) queue_id: u16, + pub(crate) descriptor_generation: u16, + pub(crate) data_master: u8, + pub(crate) old_timestamp: u64, +} + +/// Descriptor accounting and bytes prepared only for the tag-14 fallback. +/// Direct-feature G17P skips this transport command. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PComputePrimaryEvent { + pub(crate) descriptor_generation: u32, + pub(crate) bytes: [u8; G17P_COMPUTE_PRIMARY_EVENT_SIZE], +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PComputeOptionalEvent { + pub(crate) queue_id: u16, + pub(crate) old_timestamp: u64, +} + +/// Encode the tag-14 fallback command used when the direct feature is absent. +pub(crate) fn encode_g17p_compute_primary_event( + event: G17PComputePrimaryEvent, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() < G17P_COMPUTE_PRIMARY_EVENT_SIZE { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + out[..G17P_COMPUTE_PRIMARY_EVENT_SIZE].fill(0); + put_u32_le(out, 0x00, 0x0e); + put_u16_le(out, 0x04, event.queue_id); + put_u16_le(out, 0x06, event.descriptor_generation); + out[0x08] = event.data_master; + put_u64_le(out, 0x09, event.old_timestamp); + put_u32_le(out, 0x11, 2); + Ok(()) +} + +/// Advance the host descriptor's submitted-entry count for one CL entry. +pub(crate) fn advance_g17p_compute_descriptor_generation( + generation_before: u32, + nop_count: u32, +) -> Result { + generation_before + .checked_add(nop_count) + .and_then(|generation| generation.checked_add(1)) + .ok_or(SubmissionError::ComputeEventCounterOverflow) +} + +/// Prepare the tag-14 fallback bytes. Required NOPs are accounted before the +/// one CL entry; the direct-feature path advances the same generation but +/// does not call this fallback encoder or publish its pointer. +pub(crate) fn prepare_g17p_compute_primary_event( + generation_before: u32, + nop_count: u32, + queue_id: u8, + data_master: u8, + old_timestamp: u64, +) -> Result { + let descriptor_generation = + advance_g17p_compute_descriptor_generation(generation_before, nop_count)?; + let event_generation = u16::try_from(descriptor_generation) + .map_err(|_| SubmissionError::ComputeEventCounterOverflow)?; + let mut bytes = [0u8; G17P_COMPUTE_PRIMARY_EVENT_SIZE]; + encode_g17p_compute_primary_event( + G17PComputePrimaryEvent { + queue_id: u16::from(queue_id), + descriptor_generation: event_generation, + data_master, + old_timestamp, + }, + &mut bytes, + )?; + Ok(PreparedG17PComputePrimaryEvent { + descriptor_generation, + bytes, + }) +} + +/// Host-written extent of the tag-14 `KsmKickQueueAddKicks` channel command. +/// +/// `AGXCLChannelSKSM::submitBuffer` writes exactly these 0x15 bytes inline at +/// mac17p `0xfffffe00089fa39c-0xfffffe00089fa3cc`, then publishes the record's +/// GPU address through `AGXChannel::writeChannelCommandPointer`. +pub(crate) const G17P_KSM_ADD_KICKS_COMMAND_SIZE: usize = 0x15; +/// Bytes the matched J700 A000 handler cache-invalidates before reading the +/// record (`mov w1, #0x18` at `0xfffffc0000011714`). The three bytes past the +/// host-written extent are never loaded, but must not be stale. +pub(crate) const G17P_KSM_ADD_KICKS_COMMAND_FETCH_SIZE: usize = 0x18; + +/// Fields the native host copies into one tag-14 AddKicks record. +/// +/// Native sources, all from `submitBuffer`: +/// * `queue_id` = `AGXKSMKickQueue[0x0c]`, the QID. +/// * `add_count` = `AGXCLCommandDescriptor[0x2e4]`, the number of kick +/// entries this submission published (NOPs plus the one CL entry). It is the +/// same value the suppressed MMIO variant passes to +/// `AGXKSMKickQueueG16::addKicks`. +/// * `queue_priority`= `AGXKSMKickQueue[0x1c]`, i.e. `AGXChannel::getPriority()`. +/// This queue's value is the retained completion selector. +/// * `entry_stamp` = the queue's producer stamp **before** the CL entry +/// advanced it, returned by `AGXKSMKickQueueG16::encodeKickEntry` +/// (`ldr x0, [x19, #0x48]` at `0xfffffe0008a56e00`). Its low byte is the ring +/// entry index; A000 reads it only for its trace event. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmAddKicksCommand { + pub(crate) queue_id: u8, + pub(crate) add_count: u32, + pub(crate) queue_priority: u8, + pub(crate) entry_stamp: u64, +} + +/// Bytes for one tag-14 record, ready to copy into retained queue storage. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PKsmAddKicks { + pub(crate) bytes: [u8; G17P_KSM_ADD_KICKS_COMMAND_SIZE], +} + +/// Build the tag-14 `KsmKickQueueAddKicks` channel command. +/// +/// The record replaces the host SKSM-port `addKicks` pair on J700, where every +/// host access to `sgx 0x0..0x2ffff` takes an asynchronous SError. The matched +/// A000 firmware dispatches type 14 to `0xfffffc0000011710`, which writes the +/// QID to `sgx+0x103a8` and this record's `add_count` to `sgx+0x21018` through +/// the firmware's own mapping of the block. +/// +/// The A000 handler loads only `+0x04`, `+0x06` and `+0x09`; `+0x08` and `+0x11` +/// are written for host parity. +pub(crate) fn prepare_g17p_ksm_add_kicks_command( + command: G17PKsmAddKicksCommand, +) -> Result { + if command.queue_id > KICK_QID_MASK { + return Err(SubmissionError::QueueIdOutOfRange); + } + if command.queue_priority > 3 { + return Err(SubmissionError::KickQueueCompletionSelectorOutOfRange); + } + if command.entry_stamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + if command.add_count == 0 { + return Err(SubmissionError::KsmAddKicksCountZero); + } + let add_count = match u16::try_from(command.add_count) { + Ok(add_count) => add_count, + Err(_) => return Err(SubmissionError::KsmAddKicksCountOutOfRange), + }; + + let mut bytes = [0u8; G17P_KSM_ADD_KICKS_COMMAND_SIZE]; + put_u32_le( + &mut bytes, + 0x00, + G17PTopLevelCommandType::KsmKickQueueAddKicks as u32, + ); + put_u16_le(&mut bytes, 0x04, u16::from(command.queue_id)); + put_u16_le(&mut bytes, 0x06, add_count); + bytes[0x08] = command.queue_priority; + put_u64_le(&mut bytes, 0x09, command.entry_stamp); + put_u32_le(&mut bytes, 0x11, G17P_COMPUTE_DATA_MASTER); + Ok(PreparedG17PKsmAddKicks { bytes }) +} + +/// Encode tag 16 only when the host descriptor carries its optional pointer. +/// The simple direct first add3 profile does not publish this record. +pub(crate) fn encode_g17p_compute_optional_event( + event: G17PComputeOptionalEvent, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() < G17P_COMPUTE_OPTIONAL_EVENT_SIZE { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + out[..G17P_COMPUTE_OPTIONAL_EVENT_SIZE].fill(0); + put_u32_le(out, 0x00, 0x10); + put_u16_le(out, 0x04, event.queue_id); + put_u64_le(out, 0x06, event.old_timestamp); + put_u32_le(out, 0x0e, 0xfe); + Ok(()) +} + +pub(crate) fn prepare_g17p_compute_cl_command_state( + descriptor_low: u64, + submission_ordinal: u64, + primary_checkpoint: u8, +) -> Result { + let next = submission_ordinal + .checked_add(1) + .ok_or(SubmissionError::ComputeEventCounterOverflow)?; + let event_previous = u64::from(submission_ordinal) + .checked_mul(2) + .ok_or(SubmissionError::ComputeEventCounterOverflow)?; + let event_current = u64::from(next) + .checked_mul(4) + .ok_or(SubmissionError::ComputeEventCounterOverflow)?; + let rce_a = descriptor_low + .checked_add(0x40) + .ok_or(SubmissionError::ComputeRceAddressOverflow)?; + let rce_b = descriptor_low + .checked_add(0x760) + .ok_or(SubmissionError::ComputeRceAddressOverflow)?; + let empty = G17PClRceBinding { address: 0, tag: 0 }; + Ok(G17PComputeClCommandState { + event_mask: [0, event_previous, event_current, 0], + rce_bindings: [ + G17PClRceBinding { + address: rce_a, + tag: primary_checkpoint, + }, + G17PClRceBinding { + address: rce_b, + tag: 0x04, + }, + empty, + empty, + ], + }) +} + +/// Select the descriptor-embedded RegisterArrays consumed by a native render +/// kick. 3D's four complete arrays start at command +0xa0 and follow at the +/// 0x720-byte stride present in every working capture. +pub(crate) fn prepare_g17p_render_cl_command_state( + descriptor_low: u64, + tiling: bool, +) -> Result { + const FRAGMENT_RCE_PROGRAM_STRIDE: u64 = 0x720; + const FRAGMENT_RCE_PROGRAM_OFFSET: u64 = 0xa0; + let binding = |offset: u64, tag: u8| { + descriptor_low + .checked_add(offset) + .map(|address| G17PClRceBinding { address, tag }) + .ok_or(SubmissionError::ComputeRceAddressOverflow) + }; + let empty = G17PClRceBinding { address: 0, tag: 0 }; + + let rce_bindings = if tiling { + [binding(0x60, 0x47)?, empty, empty, empty] + } else { + let fragment_rce = descriptor_low + .checked_add(FRAGMENT_RCE_PROGRAM_OFFSET) + .ok_or(SubmissionError::ComputeRceAddressOverflow)?; + let external_binding = |index: u64, tag: u8| { + fragment_rce + .checked_add(index * FRAGMENT_RCE_PROGRAM_STRIDE) + .map(|address| G17PClRceBinding { address, tag }) + .ok_or(SubmissionError::ComputeRceAddressOverflow) + }; + let full = [ + external_binding(0, 0x56)?, + external_binding(1, 0x10)?, + external_binding(2, 0x17)?, + external_binding(3, 0x0a)?, + ]; + match crate::g18::fw::config::RENDER_3D_RCE { + 0 => [empty, empty, empty, empty], + 1 => [full[0], empty, empty, empty], + 2 => [full[0], full[1], empty, empty], + 3 => [full[0], full[1], full[2], empty], + _ => full, + } + }; + + Ok(G17PComputeClCommandState { + event_mask: [0; 4], + rce_bindings, + }) +} + +/// Physical-queue QoS fields packed above the 32-bit dependency mask at entry +/// +0x20. These remain with the registered SKSM queue when later descriptors +/// publish new per-command event masks. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClQosConfig { + pub(crate) queue_byte_32: u8, + pub(crate) class_40: u8, + pub(crate) word_48: u16, +} + +/// Physical queue configuration and mutable producer state read before one CL +/// entry. The QoS fields belong to this retained queue, while the event mask in +/// [`G17PClKickEntryOperands`] belongs to the command being submitted. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClKickProducerState { + pub(crate) queue_id: u8, + pub(crate) geometry: G17SksmQueueGeometry, + pub(crate) completion_selector: u8, + pub(crate) qos: G17PClQosConfig, + pub(crate) submitted: u32, + pub(crate) current_timestamp: u64, + /// Last timestamp reconciled through `submitCLChannel` addKicks. + pub(crate) last_add_kicks_timestamp: u64, + pub(crate) base_timestamp: u64, + pub(crate) previous_valid: bool, + pub(crate) previous_timestamp: u64, +} + +impl G17SksmQueueGeometry { + pub(crate) const fn initial_producer_state( + self, + queue_id: u8, + completion_selector: u8, + qos: G17PClQosConfig, + ) -> Result { + match self.validate() { + Ok(()) => {} + Err(error) => return Err(error), + } + if queue_id > KICK_QID_MASK { + return Err(SubmissionError::QueueIdOutOfRange); + } + if completion_selector > 3 { + return Err(SubmissionError::KickQueueCompletionSelectorOutOfRange); + } + Ok(G17PClKickProducerState { + queue_id, + geometry: self, + completion_selector, + qos, + submitted: 0, + current_timestamp: self.timestamp_seed, + last_add_kicks_timestamp: self.timestamp_seed, + base_timestamp: self.timestamp_seed, + previous_valid: false, + previous_timestamp: 0, + }) + } +} + +/// Descriptor and call-site operands consumed by `encodeCLKickEntry`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClKickEntryOperands<'a> { + /// Descriptor +0x4c bit 0; true sets entry-header bit 1. + pub(crate) descriptor_flag_4c: bool, + /// Descriptor +0x5c8 bit 0; true sets entry-header bit 61. + pub(crate) descriptor_flag_5c8: bool, + /// Result of `convertEventMachineTSToHWTS` for a queue with no predecessor. + pub(crate) converted_command_timestamp: u64, + pub(crate) barriers: &'a [G17PClBarrierDependency], + pub(crate) mcache: Option, + /// The two qwords reached through `AGXKSMKickQueueEntryPayload`. + pub(crate) payload: [u64; 2], + /// Per-command mask copied independently to +0x128..+0x140. + pub(crate) event_mask: [u64; 4], + pub(crate) rce_kind: u8, + pub(crate) rce_bindings: [G17PClRceBinding; 4], + /// Descriptor +0x178 when nonzero, otherwise the descriptor-indexed host + /// table value selected by +0x61c. + pub(crate) auxiliary: u64, +} + +/// Pure result of one entry encode and the host state values published after +/// the entry barrier. No shared memory or MMIO is touched by this function. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PClKickEntry { + /// Bytes the live publisher must clear before copying `entry`. + pub(crate) zero_length: u32, + pub(crate) entry: [u8; G17P_CL_KICK_ENTRY_SIZE], + pub(crate) entry_index: u8, + pub(crate) entry_offset: u32, + pub(crate) current_timestamp: u64, + pub(crate) next_timestamp: u64, + pub(crate) submitted_before: u32, + pub(crate) submitted_after: u32, + pub(crate) previous_timestamp_after: u64, + /// Producer state to use for the next entry on this physical queue. + pub(crate) producer_after: G17PClKickProducerState, +} + +/// Ordering at the side-effect boundary of the host encoder. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PClKickPublicationStep { + ZeroEntryStride, + EntryBody, + InnerShareableBarrier, + NextTimestamp, + SubmittedAtomicAdd, + PreviousValid, + PreviousTimestamp, +} + +pub(crate) const G17P_CL_KICK_PUBLICATION_ORDER: [G17PClKickPublicationStep; 7] = [ + G17PClKickPublicationStep::ZeroEntryStride, + G17PClKickPublicationStep::EntryBody, + G17PClKickPublicationStep::InnerShareableBarrier, + G17PClKickPublicationStep::NextTimestamp, + G17PClKickPublicationStep::SubmittedAtomicAdd, + G17PClKickPublicationStep::PreviousValid, + G17PClKickPublicationStep::PreviousTimestamp, +]; + +/// G17P direct first-submit side-effect ledger. The tag/QID/SKSM/channel and +/// mailbox ordering is binary-verified against the native submit path. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PFirstSubmitPublicationStep { + AcquireGfx1RunningPower, + PublishB2, + PublishConfigUpdatePointer, + ConfigureQid4Hardware, + ZeroSksmStride, + WriteSksmEntry, + EntryBarrier, + PublishSksmProducer, + AdvanceDescriptorGeneration, + RecordHostCorrelationToken, + PublishEntrySignalPointer, + AddKicks, + PublishCl2, + SendActivation, + DirectKickBarrier, + SendDirectKick, +} + +pub(crate) const G17P_FIRST_SUBMIT_PUBLICATION_ORDER: [G17PFirstSubmitPublicationStep; 16] = [ + G17PFirstSubmitPublicationStep::AcquireGfx1RunningPower, + G17PFirstSubmitPublicationStep::PublishB2, + G17PFirstSubmitPublicationStep::PublishConfigUpdatePointer, + G17PFirstSubmitPublicationStep::ConfigureQid4Hardware, + G17PFirstSubmitPublicationStep::ZeroSksmStride, + G17PFirstSubmitPublicationStep::WriteSksmEntry, + G17PFirstSubmitPublicationStep::EntryBarrier, + G17PFirstSubmitPublicationStep::PublishSksmProducer, + G17PFirstSubmitPublicationStep::AdvanceDescriptorGeneration, + G17PFirstSubmitPublicationStep::RecordHostCorrelationToken, + G17PFirstSubmitPublicationStep::PublishEntrySignalPointer, + G17PFirstSubmitPublicationStep::AddKicks, + G17PFirstSubmitPublicationStep::PublishCl2, + G17PFirstSubmitPublicationStep::SendActivation, + G17PFirstSubmitPublicationStep::DirectKickBarrier, + G17PFirstSubmitPublicationStep::SendDirectKick, +]; + +fn encode_g17p_cl_dependency(timestamp: u64, queue_id: u8) -> Result { + if timestamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + if queue_id > KICK_QID_MASK as u8 { + return Err(SubmissionError::KickDependencyQueueIdOutOfRange); + } + Ok(timestamp | ((queue_id as u64) << KICK_QID_SHIFT)) +} + +fn encode_g17p_cl_barrier_dependency( + queue: G17PClKickProducerState, + barrier: G17PClBarrierDependency, +) -> Result { + if barrier.queue_id > KICK_QID_MASK as u8 { + return Err(SubmissionError::KickDependencyQueueIdOutOfRange); + } + + if let Some(timestamp) = barrier.native_timestamp { + return encode_g17p_cl_dependency(timestamp, barrier.queue_id); + } + + let stamp_count = queue.geometry.stamp_count() as u64; + let event_timestamp = (((barrier.event_word as u64) >> 8) + | barrier.shared_stamp.wrapping_shl(24)) + .wrapping_sub(1); + let event_stamp = event_timestamp / stamp_count; + let event_index = event_timestamp % stamp_count; + let index_sum = (queue.base_timestamp & KICK_STAMP_INDEX_MASK) + event_index; + let index_carry = index_sum / stamp_count; + let stamp = ((queue.base_timestamp >> KICK_STAMP_SHIFT) as u32) + .wrapping_add(event_stamp as u32) + .wrapping_add(index_carry as u32); + let index = (index_sum % stamp_count) as u8; + + encode_g17p_cl_dependency( + encode_kick_timestamp(KickTimestamp { + stamp, + stamp_index: index, + }), + barrier.queue_id, + ) +} + +fn encode_g17p_cl_mcache(aperture: G17PClMcacheAperture) -> Result { + if aperture.index > 0x3f || aperture.count == 0 || aperture.count > 0x40 { + return Err(SubmissionError::KickMcacheFieldOutOfRange); + } + if aperture.address & 0x1f != 0 || aperture.address >> 5 > G17P_CL_PACKED_ADDRESS_MASK { + return Err(SubmissionError::KickMcacheAddressUnencodable); + } + + Ok((aperture.index as u64) + | ((aperture.address >> 5) << 6) + | (((aperture.count as u64 - 1) & 0x3f) << 54)) +} + +fn encode_g17p_cl_rce(binding: G17PClRceBinding) -> Result { + if binding.tag > KICK_QID_MASK as u8 { + return Err(SubmissionError::KickRceTagOutOfRange); + } + if binding.address & 0x1f != 0 || binding.address >> 5 > G17P_CL_PACKED_ADDRESS_MASK { + return Err(SubmissionError::KickRceAddressUnencodable); + } + + Ok((binding.address >> 5) | ((binding.tag as u64) << 43)) +} + +/// Encode the complete G17P CL entry body and calculate the exact host state +/// transition that follows the recovered `dmb ish`. +pub(crate) fn prepare_g17p_cl_kick_entry( + queue: G17PClKickProducerState, + operands: G17PClKickEntryOperands<'_>, +) -> Result { + queue.geometry.validate()?; + if queue.completion_selector > 3 { + return Err(SubmissionError::KickQueueCompletionSelectorOutOfRange); + } + if queue.base_timestamp & !KICK_TS_MASK != 0 { + return Err(SubmissionError::KickTimestampOutOfRange); + } + if operands.barriers.len() > G17P_CL_EXPLICIT_DEPENDENCIES_MAX { + return Err(SubmissionError::KickDependencyCountOutOfRange); + } + if queue.qos.class_40 > 0x1f { + return Err(SubmissionError::KickQosClassOutOfRange); + } + // Native writes `(RCEInfo + 0x00) & 3` here. RCEInfo +0x04 is the active + // RegisterArray count (1 for TA, 4 for 3D), not this field. + if operands.rce_kind > 3 { + return Err(SubmissionError::KickRceKindOutOfRange); + } + + let prepared = prepare_kick_submission( + queue.current_timestamp, + queue.queue_id, + queue.geometry.stamp_count(), + queue.geometry.entry_stride(), + )?; + if queue.submitted >= queue.geometry.stamp_count() as u32 { + return Err(SubmissionError::KickQueueSubmittedFull); + } + + let mut entry = [0u8; G17P_CL_KICK_ENTRY_SIZE]; + let layout = G17P_CL_KICK_ENTRY_LAYOUT; + + let mut header = queue.current_timestamp << 2; + header |= (queue.queue_id as u64) << 42; + header |= (queue.completion_selector as u64) << 59; + if operands.descriptor_flag_4c { + header |= 1 << 1; + } + if let Some(aperture) = operands.mcache { + header |= 1 << 58; + put_u64_le(&mut entry, layout.mcache, encode_g17p_cl_mcache(aperture)?); + } + if operands.descriptor_flag_5c8 { + header |= 1 << 61; + } + put_u64_le(&mut entry, layout.header, header); + + for (offset, value) in layout.payload.into_iter().zip(operands.payload) { + put_u64_le(&mut entry, offset, value); + } + + for (index, barrier) in operands.barriers.iter().copied().enumerate() { + put_u64_le( + &mut entry, + layout.dependencies + index * 8, + encode_g17p_cl_barrier_dependency(queue, barrier)?, + ); + } + + let implicit_timestamp = if queue.previous_valid { + queue.previous_timestamp + } else { + operands.converted_command_timestamp.wrapping_sub(1) + }; + let implicit_index = operands.barriers.len(); + // DIAGNOSTIC. On a first submit this declares a self-dependency on stamp 0 + // (`converted_command_timestamp` is 1, minus one). Measured on hardware at + // the stall: the TA entry's dependency word is 0x0000050000000000, i.e. + // {qid 5, stamp 0}, while the hardware's completed register for that queue, + // `0x21028[5]`, reads 0x862cde4910 -- not zero, and not the firmware's + // "never completed = base-1 = 0" convention. Firmware assumes one value and + // the hardware register holds another, so this dependency may be + // unsatisfiable from cold. Omitting it isolates whether the SKSM front-end + // is waiting on it. + let omit_implicit = !queue.previous_valid + && crate::g18::fw::config::NO_IMPLICIT_DEP != 0; + if !omit_implicit { + put_u64_le( + &mut entry, + layout.dependencies + implicit_index * 8, + encode_g17p_cl_dependency(implicit_timestamp, queue.queue_id)?, + ); + } + + let dependency_count = implicit_index + usize::from(!omit_implicit); + let dependency_mask = if dependency_count == G17P_CL_DEPENDENCIES_MAX { + u32::MAX + } else if dependency_count == 0 { + 0 + } else { + (1u32 << dependency_count) - 1 + }; + let dependency_control = dependency_mask as u64 + | ((queue.qos.queue_byte_32 as u64) << 32) + | ((queue.qos.class_40 as u64) << 40) + | ((queue.qos.word_48 as u64) << 48); + put_u64_le(&mut entry, layout.dependency_control, dependency_control); + + for (offset, value) in layout.event_mask.into_iter().zip(operands.event_mask) { + put_u64_le(&mut entry, offset, value); + } + entry[layout.rce_kind] = operands.rce_kind; + for (offset, binding) in layout.rce_bindings.into_iter().zip(operands.rce_bindings) { + put_u64_le(&mut entry, offset, encode_g17p_cl_rce(binding)?); + } + put_u64_le(&mut entry, layout.auxiliary, operands.auxiliary); + + let mut producer_after = queue; + producer_after.current_timestamp = prepared.next_timestamp; + producer_after.submitted = queue.submitted.wrapping_add(1); + producer_after.previous_valid = true; + producer_after.previous_timestamp = prepared.timestamp; + + Ok(PreparedG17PClKickEntry { + zero_length: queue.geometry.entry_stride(), + entry, + entry_index: prepared.entry_index, + entry_offset: prepared.entry_offset, + current_timestamp: prepared.timestamp, + next_timestamp: prepared.next_timestamp, + submitted_before: queue.submitted, + submitted_after: producer_after.submitted, + previous_timestamp_after: prepared.timestamp, + producer_after, + }) +} + +/// Apply the completed-kick count used by `AGXCLCommandDescriptor::complete`. +/// The live callback performs an acquire-release atomic subtraction at queue +/// +0x44. This pure form keeps every other producer field unchanged. +pub(crate) fn complete_g17p_cl_kicks( + mut producer: G17PClKickProducerState, + completed_count: u32, +) -> Result { + if completed_count == 0 { + return Ok(producer); + } + if producer.submitted == 0 || completed_count > producer.submitted { + return Err(SubmissionError::KickCompletionUnderflow); + } + + producer.submitted -= completed_count; + Ok(producer) +} + +/// One range inside provider-owned queue storage. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClStorageRange { + pub(crate) offset: usize, + pub(crate) size: usize, +} + +/// The scheduler record at +0x200 aliases the registered queue's 16 KiB +/// shared-support page. A second full-page support write would erase it. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClRetainedStorageLayout { + pub(crate) shared_support: G17PClStorageRange, + pub(crate) scheduler: G17PClStorageRange, +} + +pub(crate) const G17P_CL_RETAINED_STORAGE_LAYOUT: G17PClRetainedStorageLayout = + G17PClRetainedStorageLayout { + shared_support: G17PClStorageRange { + offset: 0, + size: 0x4000, + }, + scheduler: G17PClStorageRange { + offset: 0x200, + size: 0x100, + }, + }; + +/// Required provider-owned storage for the measured pre-B2 state writes. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PClB2PrimaryStateLayout { + pub(crate) object_size: usize, + pub(crate) computed_write_extent: usize, + pub(crate) region_1_write_extent: usize, + pub(crate) region_2_write_extent: usize, +} + +pub(crate) const G17P_CL_B2_PRIMARY_STATE_LAYOUT: G17PClB2PrimaryStateLayout = + G17PClB2PrimaryStateLayout { + object_size: 0x4000, + computed_write_extent: 0xe30, + region_1_write_extent: 0x20, + region_2_write_extent: 0x18, + }; + +pub(crate) const fn g17p_cl_ranges_overlap( + left: G17PClStorageRange, + right: G17PClStorageRange, +) -> bool { + let left_end = match left.offset.checked_add(left.size) { + Some(end) => end, + None => return false, + }; + let right_end = match right.offset.checked_add(right.size) { + Some(end) => end, + None => return false, + }; + left.offset < right_end && right.offset < left_end +} + +pub(crate) const fn g17p_cl_range_contains( + outer: G17PClStorageRange, + inner: G17PClStorageRange, +) -> bool { + let outer_end = match outer.offset.checked_add(outer.size) { + Some(end) => end, + None => return false, + }; + let inner_end = match inner.offset.checked_add(inner.size) { + Some(end) => end, + None => return false, + }; + outer.offset <= inner.offset && inner_end <= outer_end +} + +/// A whole-page support initializer is valid only before queue registration. +pub(crate) const fn g17p_cl_full_support_initialization_allowed(registered: bool) -> bool { + !registered +} + +/// The one compute private-memory pool, distinct from render's pool ID zero. +/// B1 indexes the same registration table with tag15 +0x46 and FList +0x08. +pub(crate) const G17P_COMPUTE_FLIST_ID: u16 = crate::g18::fw::config::COMPUTE_FLIST_ID; + +/// Initialize pool-owned state before its first firmware registration. +pub(crate) fn apply_g17p_cl_flist_static_state( + shared_support: &mut [u8], + support_state: &mut [u8], + page_list_gpu_va: u64, + run_table_gpu_va: u64, + support_state_gpu_va: u64, + pool_id: u16, +) -> Result<(), SubmissionError> { + if shared_support.len() < 0x4000 || support_state.len() < 0x4000 { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + shared_support.fill(0); + support_state.fill(0); + put_u64_le(shared_support, 0x00, 2); + put_u64_le(shared_support, 0x08, u64::from(pool_id)); + put_u64_le(shared_support, 0x10, 2); + put_u64_le(shared_support, 0x14, page_list_gpu_va); + put_u32_le(shared_support, 0x1c, 0x4_0000); + put_u64_le(shared_support, 0x20, 0x0000_1500_0000_0000); + put_u64_le(shared_support, 0x28, 0x0000_1500_0000_0000); + put_u64_le(shared_support, 0x30, run_table_gpu_va); + put_u64_le(shared_support, 0x40, 4); + put_u32_le(shared_support, 0x48, 0xa8); + put_u64_le(shared_support, 0x4c, support_state_gpu_va); + // B1 compares *FList[+0x4c] (submitted) with FList[+0x54] + // (completed). Both are pool-wide u64 counters, initially zero. + put_u64_le(shared_support, 0x54, 0); + put_u32_le(shared_support, 0x5c, 1); + put_u32_le(shared_support, 0x60, 2); + Ok(()) +} + +/// Historical captured per-queue fixture initializer. Production pool creation +/// uses `apply_g17p_cl_flist_static_state`; never call this on its shared pages. +pub(crate) fn apply_g17p_cl_queue_static_state( + shared_support: &mut [u8], + support_state: &mut [u8], + scheduler_state: &mut [u8], + page_list_gpu_va: u64, + run_table_gpu_va: u64, + support_state_gpu_va: u64, + scheduler_state_gpu_va: u64, +) -> Result<(), SubmissionError> { + if shared_support.len() < 0x4000 + || support_state.len() < 0x4000 + || scheduler_state.len() < 0x4000 + { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + + shared_support.fill(0); + support_state.fill(0); + scheduler_state.fill(0); + + put_u64_le(shared_support, 0x00, 2); + // AGXUSCPrivMemFList+0x08 is the pool ID copied into the firmware's + // internal allocator entry and ultimately written to gUPM's pool-select + // operation. All compute queues share this pool and UMA slot; QoS + // HardwareBufferIDs remain per queue. + put_u64_le(shared_support, 0x08, 1); + put_u64_le(shared_support, 0x10, 2); + // _AGFIUMAPagePoolState stores the full VA at unaligned +0x14. + // Copying the native +0x18 qword leaves compute naming render's pool. + put_u64_le(shared_support, 0x14, page_list_gpu_va); + put_u32_le(shared_support, 0x1c, 0x4_0000); + put_u64_le(shared_support, 0x20, 0x0000_1500_0000_0000); + put_u64_le(shared_support, 0x28, 0x0000_1500_0000_0000); + // Every queue names the run table that describes the shared backing. + // A private empty table here is incoherent with the shared pool and leaves + // overlapping spill allocation without the pool's initialized run map. + put_u64_le(shared_support, 0x30, run_table_gpu_va); + put_u64_le(shared_support, 0x40, 4); + put_u32_le(shared_support, 0x48, 0xa8); + put_u64_le(shared_support, 0x4c, support_state_gpu_va); + put_u32_le(shared_support, 0x54, 0); + put_u32_le(shared_support, 0x5c, 1); + put_u32_le(shared_support, 0x60, 2); + + let scheduler = G17P_CL_RETAINED_STORAGE_LAYOUT.scheduler.offset; + put_u64_le(shared_support, scheduler, scheduler_state_gpu_va + 8); + put_u32_le(shared_support, scheduler + 0x08, 1); + put_u32_le(shared_support, scheduler + 0x0c, 0); + put_u32_le(shared_support, scheduler + 0x10, 0x50); + put_u32_le(support_state, 0, 1); + put_u32_le(scheduler_state, 8, 1); + + Ok(()) +} + +/// Initialize the one channel-control page retained by physical CL_2. +pub(crate) fn apply_g17p_cl_channel_control_static_state( + channel_control: &mut [u8], +) -> Result<(), SubmissionError> { + if channel_control.len() < 0x40 { + return Err(SubmissionError::KickRetainedStorageBufferTooSmall); + } + channel_control.fill(0); + + put_u64_le(channel_control, 0x00, 0x0000_0100_0000_ffff); + put_u64_le(channel_control, 0x20, 0x0002_0000_0000_0000); + put_u64_le(channel_control, 0x30, 0x0000_0000_ff00_0000); + Ok(()) +} + +/// Compatibility helper for byte-layout tests covering both ownership +/// domains. Production initializes the queue pages and CL_2 page separately. +pub(crate) fn apply_g17p_cl_retained_static_state( + shared_support: &mut [u8], + support_state: &mut [u8], + scheduler_state: &mut [u8], + channel_control: &mut [u8], + page_list_gpu_va: u64, + run_table_gpu_va: u64, + support_state_gpu_va: u64, + scheduler_state_gpu_va: u64, +) -> Result<(), SubmissionError> { + apply_g17p_cl_queue_static_state( + shared_support, + support_state, + scheduler_state, + page_list_gpu_va, + run_table_gpu_va, + support_state_gpu_va, + scheduler_state_gpu_va, + )?; + apply_g17p_cl_channel_control_static_state(channel_control) +} + +/// Apply the exact host-owned pre-B2 values to three provider-owned pages. +/// Bytes outside the listed fields remain unchanged. `channel_control_gpu_va` +/// comes from the retained Linux mapping, not the captured macOS address. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PClB2ActivationState { + Pending, + Published, +} + +pub(crate) fn apply_g17p_cl_b2_primary_state( + activation: G17PClB2ActivationState, + computed: &mut [u8], + region_1: &mut [u8], + region_2: &mut [u8], + channel_control_gpu_va: u64, +) -> Result { + if activation == G17PClB2ActivationState::Published { + return Err(SubmissionError::KickB2PrimaryStateAlreadyPublished); + } + let layout = G17P_CL_B2_PRIMARY_STATE_LAYOUT; + if computed.len() < layout.object_size + || region_1.len() < layout.object_size + || region_2.len() < layout.object_size + { + return Err(SubmissionError::KickB2PrimaryStateBufferTooSmall); + } + + let activation = apply_g17p_cl_b2_computed_state( + activation, + computed, + channel_control_gpu_va, + )?; + apply_g17p_cl_b2_descriptor_tables(region_1, region_2)?; + Ok(activation) +} + +/// Apply the session-wide CL_2 half of B2 activation. The pointer is pinned to +/// a dedicated bootstrap owner; logical queues have separate scheduler pages. +/// The exact consumer of this firmware-private slot remains unresolved. Never +/// retarget it during ordinary queue switching. PBDesc/UMA are separate state. +pub(crate) fn apply_g17p_cl_b2_computed_state( + activation: G17PClB2ActivationState, + computed: &mut [u8], + channel_control_gpu_va: u64, +) -> Result { + if activation == G17PClB2ActivationState::Published { + return Err(SubmissionError::KickB2PrimaryStateAlreadyPublished); + } + let layout = G17P_CL_B2_PRIMARY_STATE_LAYOUT; + if computed.len() < layout.object_size { + return Err(SubmissionError::KickB2PrimaryStateBufferTooSmall); + } + + computed[0x000] = 0xff; + computed[0x008] = 0xff; + computed[0x010..0x012].copy_from_slice(&0x0f01u16.to_le_bytes()); + computed[0x018..0x01a].copy_from_slice(&0x0f01u16.to_le_bytes()); + put_u32_le(computed, 0x408, 2); + put_u32_le(computed, 0x40c, 2); + put_u32_le(computed, 0x604, 4); + put_u64_le(computed, 0x808, channel_control_gpu_va); + put_u32_le(computed, 0xc04, 0x1000); + put_u64_le(computed, 0xe00, 7); + put_u64_le(computed, 0xe08, 0x0000_0001_0e22_618a); + put_u64_le(computed, 0xe20, 4); + put_u64_le(computed, 0xe28, 4); + + Ok(G17PClB2ActivationState::Published) +} + +/// Apply the device-global descriptor-table half of B2 initialization. +/// +/// Native performs this before client halLoadPBDesc/halLoadUMADesc updates. +/// Reapplying these captured cold entries for every later compute queue would +/// overwrite whichever live render owner currently holds PBDesc index 1 and +/// UMA index 0. +pub(crate) fn apply_g17p_cl_b2_descriptor_tables( + region_1: &mut [u8], + region_2: &mut [u8], +) -> Result<(), SubmissionError> { + let layout = G17P_CL_B2_PRIMARY_STATE_LAYOUT; + if region_1.len() < layout.object_size || region_2.len() < layout.object_size { + return Err(SubmissionError::KickB2PrimaryStateBufferTooSmall); + } + + put_u64_le(region_1, 0x10, 0x0000_0080_0007_7000); + put_u32_le(region_1, 0x18, 0x17); + put_u32_le(region_1, 0x1c, 0x17); + + put_u64_le(region_2, 0x00, 0x0800_0000_e000_0000); + put_u64_le(region_2, 0x08, 0x0000_3d40_0000_3400); + put_u64_le(region_2, 0x10, 0x0000_0000_0000_1d00); + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PClRetainedPublicationStep { + InitializeSharedSupport, + WriteSchedulerRecord, + RegisterPhysicalQueue, + PublishCommandOwnedState, + CleanCommandOwnedState, + WriteB2PrimaryState, + CleanB2PrimaryState, + SystemBarrierBeforeInnerPayload, + PublishInnerQueuePayload, + SystemBarrierBeforeInnerProducer, + PublishInnerQueueProducer, + SystemBarrierAfterInnerProducer, +} + +/// One-time order that makes the scheduler alias valid before registration. +pub(crate) const G17P_CL_RETAINED_REGISTRATION_ORDER: [G17PClRetainedPublicationStep; 3] = [ + G17PClRetainedPublicationStep::InitializeSharedSupport, + G17PClRetainedPublicationStep::WriteSchedulerRecord, + G17PClRetainedPublicationStep::RegisterPhysicalQueue, +]; + +/// One-time activation order proven by retained command B2. +pub(crate) const G17P_CL_RETAINED_ACTIVATION_ORDER: [G17PClRetainedPublicationStep; 9] = [ + G17PClRetainedPublicationStep::PublishCommandOwnedState, + G17PClRetainedPublicationStep::CleanCommandOwnedState, + G17PClRetainedPublicationStep::WriteB2PrimaryState, + G17PClRetainedPublicationStep::CleanB2PrimaryState, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerPayload, + G17PClRetainedPublicationStep::PublishInnerQueuePayload, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerProducer, + G17PClRetainedPublicationStep::PublishInnerQueueProducer, + G17PClRetainedPublicationStep::SystemBarrierAfterInnerProducer, +]; + +/// Steady-state order proven by command B3 on the same physical queue. +pub(crate) const G17P_CL_RETAINED_SUBMISSION_ORDER: [G17PClRetainedPublicationStep; 6] = [ + G17PClRetainedPublicationStep::PublishCommandOwnedState, + G17PClRetainedPublicationStep::CleanCommandOwnedState, + G17PClRetainedPublicationStep::PublishInnerQueuePayload, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerProducer, + G17PClRetainedPublicationStep::PublishInnerQueueProducer, + G17PClRetainedPublicationStep::SystemBarrierAfterInnerProducer, +]; + +pub(crate) const fn g17p_cl_retained_submission_order( + activation: G17PClB2ActivationState, +) -> &'static [G17PClRetainedPublicationStep] { + match activation { + G17PClB2ActivationState::Pending => &G17P_CL_RETAINED_ACTIVATION_ORDER, + G17PClB2ActivationState::Published => &G17P_CL_RETAINED_SUBMISSION_ORDER, + } +} + +// --------------------------------------------------------------------------- +// 6. Host addKicks MMIO publication (AGX3-specific). +// +// The host does not write the firmware-side `timestamp | (qid << 40)` word +// directly. The pinned G17P/G17G/G17X implementations publish a +// target-specific 64-bit queue tag followed by the zero-extended add count +// as a second 64-bit store to two +// runtime-selected scratch offsets. The host adds nine times the runtime +// stride to both bases for every target. Publication never reads either +// location and preserves the exact word-0 then word-1 order. The separate +// host diagnostic path may read word 1; word 0 must never be read. +// G17P `addKicks` performs `str x20` then `str x19` at +// 0xfffffe0008a57b5c/0xfffffe0008a57b60; the earlier `mov w19,w1` +// zero-extends the add count. +// --------------------------------------------------------------------------- + +/// One ordered transaction for the two SKSM scratch FIFO ports. +/// +/// `word0` must be stored first and `word1` second. There is deliberately no +/// read operation in this publication interface. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17SksmOrderedWritePair { + pub(crate) word0_offset: u32, + pub(crate) word0: u64, + pub(crate) word1_offset: u32, + pub(crate) word1: u64, +} + +/// Minimal backend for an ordered pair of 64-bit FIFO stores. +pub(crate) trait G17SksmPairWriter { + type Error; + + fn write64(&self, offset: u32, value: u64) -> core::result::Result<(), Self::Error>; +} + +/// Publish one pair in the only order accepted by the pinned host sequence. +pub(crate) fn publish_g17p_sksm_write_pair( + port: &P, + pair: G17SksmOrderedWritePair, +) -> core::result::Result<(), P::Error> { + port.write64(pair.word0_offset, pair.word0)?; + port.write64(pair.word1_offset, pair.word1) +} + +/// Host producer family with an independently recovered `addKicks` tag. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum SksmAddKicksTarget { + /// A18 Pro / HAL200 (`AGXKSMKickQueueG16`). + G17P, + /// M5 / HAL300 G target. + G17G, + /// M5 Pro/Max HAL300 X-family target (G17S/G17C host implementation). + G17X, +} + +/// Exact ordered values and offsets for one host `addKicks` publication. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedSksmAddKicksMmio { + /// First store: 64-bit target tag to `tag_offset`. + pub(crate) tag_offset: u32, + pub(crate) tag: u64, + /// Second store: zero-extended number of newly published kicks. + pub(crate) count_offset: u32, + pub(crate) add_count: u32, +} + +impl PreparedSksmAddKicksMmio { + /// Convert the decoded host values to the exact two 64-bit FIFO stores. + pub(crate) const fn write_pair(self) -> G17SksmOrderedWritePair { + G17SksmOrderedWritePair { + word0_offset: self.tag_offset, + word0: self.tag, + word1_offset: self.count_offset, + word1: self.add_count as u64, + } + } +} + +const fn sksm_add_kicks_tag(target: SksmAddKicksTarget, queue_id: u8) -> u64 { + match target { + // mac17p 0x8a57b28-0x8a57b34: base bit 42, qid at bit 43. + SksmAddKicksTarget::G17P => G17P_ADD_KICKS_TAG | ((queue_id as u64) << 43), + // mac17g 0x8b91978-0x8b91984: base bit 43, qid at bit 44. + SksmAddKicksTarget::G17G => G17G_ADD_KICKS_TAG | ((queue_id as u64) << 44), + // mac17j 0x8b53144-0x8b53154: G17G form plus host-family bit 53. + SksmAddKicksTarget::G17X => G17X_ADD_KICKS_TAG | ((queue_id as u64) << 44), + } +} + +const G17P_ADD_KICKS_TAG: u64 = 0x0000_0400_0002_1018; +const G17G_ADD_KICKS_TAG: u64 = 0x0000_0800_0002_1018; +const G17X_ADD_KICKS_TAG: u64 = 0x0020_0800_0002_1018; +/// Prepare the host's two ordered MMIO stores without touching the aperture. +pub(crate) const fn prepare_sksm_add_kicks_mmio( + target: SksmAddKicksTarget, + queue_id: u8, + add_count: u32, + geometry: G17SksmScratchGeometry, +) -> Result { + if queue_id > KICK_QID_MASK as u8 { + return Err(SubmissionError::QueueIdOutOfRange); + } + + let offsets = match geometry.fifo_offsets() { + Ok(offsets) => offsets, + Err(_) => return Err(SubmissionError::KickMmioOffsetOverflow), + }; + + Ok(PreparedSksmAddKicksMmio { + tag_offset: offsets.write0, + tag: sksm_add_kicks_tag(target, queue_id), + count_offset: offsets.write1, + add_count, + }) +} + +// --------------------------------------------------------------------------- +// 7. A18 Pro / G17P SKSM hardware bridge and completion descriptors. +// +// Grounded from the pinned G17P host. `AGXKSMKickQueueG16::configureHardware` +// at 0xfffffe0008a56580-0xfffffe0008a5682c reads the runtime MMIO geometry +// from accelerator-object fields +0x10730/+0x10734/+0x10738, writes a +// 64-bit queue tag and 64-bit queue payload, then publishes the QID bit at +// MMIO +0x21068. The object fields contain base0=0x4000, base1=0x4008 and +// stride=0x10; they are not MMIO locations. The same geometry is used by +// `addKicks` (§6). Linux resource 0 (`sgx`) owns this window. Its two +// scratch locations receive ordered stores; queue enable is a separate +// read/modify/write register after the pair. The production publication +// surface intentionally exposes no scratch read operation. +// +// `AGXAcceleratorG16::populateKSMCompletionQueue` at +// 0xfffffe0008a53034-0xfffffe0008a532b8 emits four 0x20-byte descriptors. +// Each present queue has 0x40-byte entries and a fixed ordinal 0..3. The +// matched firmware consumes those queues. D93AP IRQ 1001 and RTBuddy AKF +// message type 2 converge on source 4 and drain the event rings that carry +// completion stamps. The completion path remains pending until an actual +// SKSM kick proves that complete round trip on D93AP hardware. +// --------------------------------------------------------------------------- + +/// Accelerator-object fields holding the three runtime geometry values. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PSksmRuntimeGeometryObjectLayout { + pub(crate) write0_base_field: u32, + pub(crate) write1_base_field: u32, + pub(crate) stride_field: u32, +} + +pub(crate) const G17P_SKSM_RUNTIME_GEOMETRY_OBJECT_LAYOUT: G17PSksmRuntimeGeometryObjectLayout = + G17PSksmRuntimeGeometryObjectLayout { + write0_base_field: 0x10730, + write1_base_field: 0x10734, + stride_field: 0x10738, + }; + +/// QID-valid mask base written after the two queue-configuration stores. +pub(crate) const G17P_SKSM_QUEUE_ENABLE_MASK_BASE: u32 = 0x21068; +/// QID-disable mask base written before changing a registered queue's routing. +pub(crate) const G17P_SKSM_QUEUE_DISABLE_MASK_BASE: u32 = 0x21078; +/// Queue-entry address bits preserved by the host payload encoder. +pub(crate) const G17P_SKSM_QUEUE_ENTRY_ADDRESS_MASK: u64 = 0x0000_07ff_ffff_ffe0; +/// Fixed portion of the first G17P queue-configuration store. +const G17P_SKSM_QUEUE_CONFIG_TAG: u64 = 0x0000_0400_0002_1008; +const G17P_SKSM_QUEUE_COMPLETION_SELECTOR_MAX: u32 = 4; +/// `_AGFIDataMasterType` selected by the G17P TA channel. +pub(crate) const G17P_TA_DATA_MASTER: u32 = 0; +/// `_AGFIDataMasterType` selected by the G17P 3D channel. +pub(crate) const G17P_3D_DATA_MASTER: u32 = 1; +/// `_AGFIDataMasterType` selected by `AGXCommandQueue::processComputeSetup`. +pub(crate) const G17P_COMPUTE_DATA_MASTER: u32 = 2; + +/// Runtime routing state that the compute-channel setup owns for one queue. +/// +/// The pinned host selects the queue by the CL channel QID, prepares the +/// queue's mapping resource, obtains a completion selector from the channel, +/// and stores `selector | (2 << 32)` at queue `+0x1c`. Keeping the selector +/// and queue address together prevents a caller from supplying a non-compute +/// data-master value to the production compute path. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PComputeQueueRouting { + pub(crate) queue_id: u8, + pub(crate) completion_selector: u32, + pub(crate) entry_gpu_va: u64, + pub(crate) geometry: G17SksmQueueGeometry, +} + +/// Pure description of the ordered host-side publications that make one A18 +/// Pro SKSM queue visible to the hardware bridge. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PreparedG17PSksmQueueConfig { + /// First store: 64-bit QID tag. + pub(crate) tag_offset: u32, + pub(crate) tag: u64, + /// Second store: 64-bit queue address/data-master/completion selector. + pub(crate) payload_offset: u32, + pub(crate) payload: u64, + /// Final stores: high 64 QIDs first, then low 64 QIDs. + pub(crate) enable_pair: G17SksmOrderedWritePair, + /// Stores used before changing an already-enabled queue's routing. + pub(crate) disable_pair: G17SksmOrderedWritePair, +} + +impl PreparedG17PSksmQueueConfig { + /// Convert the queue configuration to its two ordered write-only stores. + pub(crate) const fn write_pair(self) -> G17SksmOrderedWritePair { + G17SksmOrderedWritePair { + word0_offset: self.tag_offset, + word0: self.tag, + word1_offset: self.payload_offset, + word1: self.payload, + } + } +} + +/// Prepare the exact G17P host queue-configuration stores without touching +/// MMIO. Only the paired A18 Pro identity is admitted. +pub(crate) const fn prepare_g17p_sksm_queue_config( + target: G17ChannelAbiTarget, + queue_id: u8, + data_master: u32, + completion_selector: u32, + entry_gpu_va: u64, + geometry: G17SksmScratchGeometry, +) -> Result { + match target { + G17ChannelAbiTarget::A18ProG17P => {} + G17ChannelAbiTarget::M5G17G | G17ChannelAbiTarget::M5ProMaxG17X => { + return Err(SubmissionError::ChannelAbiUnconfirmed) + } + } + if queue_id > KICK_QID_MASK { + return Err(SubmissionError::QueueIdOutOfRange); + } + if data_master >= 3 { + return Err(SubmissionError::KickQueueDataMasterOutOfRange); + } + if completion_selector > G17P_SKSM_QUEUE_COMPLETION_SELECTOR_MAX { + return Err(SubmissionError::KickQueueCompletionSelectorOutOfRange); + } + if entry_gpu_va & !G17P_SKSM_QUEUE_ENTRY_ADDRESS_MASK != 0 { + return Err(SubmissionError::KickQueueEntryAddressUnencodable); + } + + let offsets = match geometry.fifo_offsets() { + Ok(offsets) => offsets, + Err(_) => return Err(SubmissionError::KickMmioOffsetOverflow), + }; + + // Host sequence: 3 * (4 - selector) + 0x3fe, shifted into bit 54. + let completion_field = (3 * (4 - completion_selector) + 0x3fe) as u64; + let payload = + (((data_master as u64) + 1) << 60) | entry_gpu_va | completion_field.wrapping_shl(54); + let low_mask = if queue_id < 64 { 1u64 << queue_id } else { 0 }; + let high_mask = if queue_id >= 64 { + 1u64 << (queue_id - 64) + } else { + 0 + }; + let enable_pair = G17SksmOrderedWritePair { + word0_offset: G17P_SKSM_QUEUE_ENABLE_MASK_BASE + 8, + word0: high_mask, + word1_offset: G17P_SKSM_QUEUE_ENABLE_MASK_BASE, + word1: low_mask, + }; + let disable_pair = G17SksmOrderedWritePair { + word0_offset: G17P_SKSM_QUEUE_DISABLE_MASK_BASE + 8, + word0: high_mask, + word1_offset: G17P_SKSM_QUEUE_DISABLE_MASK_BASE, + word1: low_mask, + }; + + Ok(PreparedG17PSksmQueueConfig { + tag_offset: offsets.write0, + tag: G17P_SKSM_QUEUE_CONFIG_TAG | ((queue_id as u64) << 43), + payload_offset: offsets.write1, + payload, + enable_pair, + disable_pair, + }) +} + +/// Prepare the queue-configuration transaction produced by the G17P compute +/// setup path. +/// +/// `AGXCommandQueue::processComputeSetup` at +/// `0xfffffe0008a077a4..0xfffffe0008a078d4` selects the queue from the CL +/// channel QID, prepares its mapping resource, obtains the channel completion +/// selector, and writes the packed selector/data-master pair. The captured +/// native CL2 queue uses QID 4, selector 2, and entry VA `0x70004d8000`. +pub(crate) const fn prepare_g17p_compute_sksm_queue_config( + routing: G17PComputeQueueRouting, + geometry: G17SksmScratchGeometry, +) -> Result { + match routing.geometry.validate() { + Ok(()) => {} + Err(error) => return Err(error), + } + if routing.entry_gpu_va & (routing.geometry.allocation_alignment() - 1) != 0 { + return Err(SubmissionError::KickQueueEntryAddressUnencodable); + } + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + routing.queue_id, + G17P_COMPUTE_DATA_MASTER, + routing.completion_selector, + routing.entry_gpu_va, + geometry, + ) +} + +/// Prepare the queue-configuration transaction for one G17P *render* queue. +/// +/// Identical to the compute helper except for the data master, which the +/// twelve captured macOS render passes pin to TA = 0 and 3D = 1 in three +/// independent places (outer ring slot `+0x10`, tag-14 `+0x11`, tag-15 +/// `+0x22`). The compute data master is rejected here so a render caller +/// cannot silently configure a CL queue. +pub(crate) const fn prepare_g17p_render_sksm_queue_config( + routing: G17PComputeQueueRouting, + data_master: u32, + geometry: G17SksmScratchGeometry, +) -> Result { + match routing.geometry.validate() { + Ok(()) => {} + Err(error) => return Err(error), + } + if data_master != G17P_TA_DATA_MASTER && data_master != G17P_3D_DATA_MASTER { + return Err(SubmissionError::KickQueueDataMasterOutOfRange); + } + if routing.entry_gpu_va & (routing.geometry.allocation_alignment() - 1) != 0 { + return Err(SubmissionError::KickQueueEntryAddressUnencodable); + } + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + routing.queue_id, + data_master, + routing.completion_selector, + routing.entry_gpu_va, + geometry, + ) +} + +pub(crate) const G17P_KSM_COMPLETION_QUEUE_COUNT: u8 = 4; +pub(crate) const G17P_KSM_COMPLETION_DESCRIPTOR_SIZE: usize = 0x20; +pub(crate) const G17P_KSM_COMPLETION_ENTRY_SIZE: u32 = 0x40; +pub(crate) const G17P_KSM_COMPLETION_CAPACITY: u32 = 0x800; +pub(crate) const G17P_KSM_COMPLETION_BACKING_SIZE: usize = + G17P_KSM_COMPLETION_CAPACITY as usize * G17P_KSM_COMPLETION_ENTRY_SIZE as usize; +pub(crate) const G17P_KSM_COMPLETION_PRESENT_ORDINALS: [u8; 2] = [0, 2]; +/// Normal direct CL completion uses ordinal 0, independently of the compute +/// SKSM queue's numeric completion selector 2. +pub(crate) const G17P_NORMAL_CL_COMPLETION_ORDINAL: u8 = 0; + +/// Exact field offsets in one host-produced G17P completion descriptor. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmCompletionDescriptorLayout { + pub(crate) gpu_va: usize, + pub(crate) cpu_va: usize, + pub(crate) capacity: usize, + pub(crate) entry_size: usize, + pub(crate) ordinal: usize, + pub(crate) processed: usize, +} + +pub(crate) const G17P_KSM_COMPLETION_DESCRIPTOR_LAYOUT: G17PKsmCompletionDescriptorLayout = + G17PKsmCompletionDescriptorLayout { + gpu_va: 0x00, + cpu_va: 0x08, + capacity: 0x10, + entry_size: 0x14, + ordinal: 0x18, + processed: 0x1c, + }; + +/// Encode one present completion queue exactly as the G17P host populates it. +pub(crate) fn encode_g17p_ksm_completion_descriptor( + target: G17ChannelAbiTarget, + ordinal: u8, + gpu_va: u64, + cpu_va: u64, + capacity: u32, +) -> Result<[u8; G17P_KSM_COMPLETION_DESCRIPTOR_SIZE], SubmissionError> { + if target != G17ChannelAbiTarget::A18ProG17P { + return Err(SubmissionError::ChannelAbiUnconfirmed); + } + if ordinal >= G17P_KSM_COMPLETION_QUEUE_COUNT { + return Err(SubmissionError::CompletionQueueOrdinalOutOfRange); + } + if capacity == 0 { + return Err(SubmissionError::CompletionQueueCapacityZero); + } + + let mut raw = [0u8; G17P_KSM_COMPLETION_DESCRIPTOR_SIZE]; + let layout = G17P_KSM_COMPLETION_DESCRIPTOR_LAYOUT; + put_u64_le(&mut raw, layout.gpu_va, gpu_va); + put_u64_le(&mut raw, layout.cpu_va, cpu_va); + put_u32_le(&mut raw, layout.capacity, capacity); + put_u32_le(&mut raw, layout.entry_size, G17P_KSM_COMPLETION_ENTRY_SIZE); + put_u32_le(&mut raw, layout.ordinal, ordinal as u32); + put_u32_le(&mut raw, layout.processed, 0); + Ok(raw) +} + +/// The host-side completion descriptor and SKSM bridge equations are grounded; +/// this does not assert that Linux knows the physical IRQ source or has +/// observed an interrupt round trip. +pub(crate) const G17P_COMPLETION_HOST_ABI_GROUNDED: bool = true; + +// --------------------------------------------------------------------------- +// 8. A18 Pro / G17P top-level channel-command dispatch. +// +// This ABI is deliberately identity-scoped, just like §2c. The pinned G17P +// host grounds the top-level names below through its command-request and +// submit paths. The matching a000 firmware dispatcher is +// `agfaProcessChannelCommand` at 0xfffffc0000010fa8 with the 19-entry table +// at 0xfffffc000001233c. The b000/b100 dispatcher starts at +// 0xfffffc0000010f14 and uses the byte-identical table at +// 0xfffffc00000122a8. +// +// All three firmware variants validate at least four bytes, load a u32 tag +// from command offset zero, and treat tags above 18 as invalid. Their jump tables +// process 0/1/2/3/4/5/7/9/12/14/15/16/18. Tag 6 reaches the dedicated +// `PMRebuild must not be received in SKSM` fatal, while 8/10/11/13/17 +// reach the invalid-command fatal. The host nevertheless names 17 `Nop`; +// that producer identity is preserved, but the admission helper marks it invalid +// because the matched consumer cannot process it. +// +// Tags 6/8/10/11 remain semantically unresolved instead of inheriting an +// AGX2 or nested SKU-stream meaning. SKU-stream opcodes are a distinct +// nested domain and intentionally have no representation in the top-level +// enum. M5-family identities remain unsupported pending their own paired +// producer/consumer pass. +// +// The optional pre-dispatch filter uses literal mask 0x1d03f for tags <= +// 16. It is exposed separately so a future live producer can preserve that +// gate instead of conflating it with complete jump-table dispatch. +// --------------------------------------------------------------------------- + +/// Offset and width of the little-endian top-level command tag. +pub(crate) const G17P_TOP_LEVEL_COMMAND_TAG_OFFSET: u32 = 0; +pub(crate) const G17P_TOP_LEVEL_COMMAND_TAG_WIDTH: u32 = 4; + +/// Host-grounded G17P top-level command types. +/// +/// This type intentionally excludes unresolved tags and every nested +/// SKU-stream opcode. `Nop` records the host ABI but is invalid for +/// [`admit_g17p_top_level_command`] because all matched firmware variants send +/// tag 17 to the invalid-command fatal path. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum G17PTopLevelCommandType { + Ta = 0, + ThreeD = 1, + FastBlit = 2, + Cl = 3, + Barrier = 4, + PmGrow = 5, + RemoteNode = 7, + SharedEventSignal = 9, + AddFinalFrgGtpcKick = 12, + KsmKickQueueAddKicks = 14, + ConfigUpdate = 15, + EntrySignal = 16, + Nop = 17, + LateEvalEventSignal = 18, +} + +/// Highest tag admitted by the G17P jump-table bounds check. +pub(crate) const G17P_CHANNEL_COMMAND_TYPE_MAX: u32 = 18; + +/// Tags named by the pinned host's top-level ABI, including invalid `Nop`. +pub(crate) const G17P_TOP_LEVEL_COMMAND_KNOWN_MASK: u32 = 0x0007_d2bf; +/// In-range top-level tags whose host/firmware semantics remain unresolved. +pub(crate) const G17P_TOP_LEVEL_COMMAND_UNRESOLVED_MASK: u32 = 0x0000_0d40; +/// Tag 13 is in range but is neither a grounded host type nor unresolved. +pub(crate) const G17P_TOP_LEVEL_COMMAND_INVALID_MASK: u32 = 0x0000_2000; + +/// Tags whose firmware jump-table entries reach a normal processing case. +pub(crate) const G17P_CHANNEL_COMMAND_PROCESS_MASK: u32 = 0x0005_d2bf; +/// Tag 6 has a dedicated PM-rebuild-in-SKSM fatal path. +pub(crate) const G17P_CHANNEL_COMMAND_PM_REBUILD_MASK: u32 = 0x0000_0040; +/// In-range tags whose jump-table entries reach the invalid-command fatal. +pub(crate) const G17P_CHANNEL_COMMAND_INVALID_MASK: u32 = 0x0002_2d00; +/// Literal mask used by the optional pre-dispatch filter for tags <= 16. +pub(crate) const G17P_CHANNEL_COMMAND_PREFILTER_MASK: u32 = 0x0001_d03f; + +/// Firmware disposition for one raw command tag. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G17PChannelCommandDisposition { + Process, + PmRebuildForbiddenFatal, + InvalidCommandFatal, +} + +/// Admit the command ABI only for the exact paired G17P identity. +const fn ensure_g17p_command_abi(target: G17ChannelAbiTarget) -> Result<(), SubmissionError> { + match target { + G17ChannelAbiTarget::A18ProG17P => Ok(()), + G17ChannelAbiTarget::M5G17G | G17ChannelAbiTarget::M5ProMaxG17X => { + Err(SubmissionError::CommandAbiUnconfirmed) + } + } +} + +/// Read the complete little-endian u32 tag at top-level command offset zero. +const fn g17p_raw_top_level_command_tag(command: &[u8]) -> Result { + let offset = G17P_TOP_LEVEL_COMMAND_TAG_OFFSET as usize; + let end = offset + G17P_TOP_LEVEL_COMMAND_TAG_WIDTH as usize; + if command.len() < end { + return Err(SubmissionError::CommandHeaderTooShort); + } + Ok(u32::from_le_bytes([ + command[offset], + command[offset + 1], + command[offset + 2], + command[offset + 3], + ])) +} + +/// Decode a host-grounded top-level G17P command name. +/// +/// This records producer identity only. Call +/// [`admit_g17p_top_level_command`] before treating a decoded tag as +/// consumable by the matched firmware. +pub(crate) const fn decode_g17p_top_level_command( + target: G17ChannelAbiTarget, + command: &[u8], +) -> Result { + match ensure_g17p_command_abi(target) { + Ok(()) => {} + Err(error) => return Err(error), + } + + let raw = match g17p_raw_top_level_command_tag(command) { + Ok(raw) => raw, + Err(error) => return Err(error), + }; + match raw { + 0 => Ok(G17PTopLevelCommandType::Ta), + 1 => Ok(G17PTopLevelCommandType::ThreeD), + 2 => Ok(G17PTopLevelCommandType::FastBlit), + 3 => Ok(G17PTopLevelCommandType::Cl), + 4 => Ok(G17PTopLevelCommandType::Barrier), + 5 => Ok(G17PTopLevelCommandType::PmGrow), + 7 => Ok(G17PTopLevelCommandType::RemoteNode), + 9 => Ok(G17PTopLevelCommandType::SharedEventSignal), + 12 => Ok(G17PTopLevelCommandType::AddFinalFrgGtpcKick), + 14 => Ok(G17PTopLevelCommandType::KsmKickQueueAddKicks), + 15 => Ok(G17PTopLevelCommandType::ConfigUpdate), + 16 => Ok(G17PTopLevelCommandType::EntrySignal), + 17 => Ok(G17PTopLevelCommandType::Nop), + 18 => Ok(G17PTopLevelCommandType::LateEvalEventSignal), + 6 | 8 | 10 | 11 => Err(SubmissionError::CommandTagUnresolved), + _ => Err(SubmissionError::CommandTagInvalid), + } +} + +/// Classify a raw tag exactly like the matched G17P firmware jump table. +pub(crate) const fn classify_g17p_channel_command( + target: G17ChannelAbiTarget, + raw: u32, +) -> Result { + match ensure_g17p_command_abi(target) { + Ok(()) => {} + Err(error) => return Err(error), + } + + if raw > G17P_CHANNEL_COMMAND_TYPE_MAX { + return Ok(G17PChannelCommandDisposition::InvalidCommandFatal); + } + + let bit = 1u32 << raw; + if bit & G17P_CHANNEL_COMMAND_PROCESS_MASK != 0 { + Ok(G17PChannelCommandDisposition::Process) + } else if bit & G17P_CHANNEL_COMMAND_PM_REBUILD_MASK != 0 { + Ok(G17PChannelCommandDisposition::PmRebuildForbiddenFatal) + } else { + Ok(G17PChannelCommandDisposition::InvalidCommandFatal) + } +} + +/// Decode a top-level tag and require a normal matched-firmware dispatch case. +/// +/// In particular, host-grounded `Nop = 17` is invalid here because the +/// firmware sends it to the invalid-command fatal path. +pub(crate) const fn admit_g17p_top_level_command( + target: G17ChannelAbiTarget, + command: &[u8], +) -> Result { + let command_type = match decode_g17p_top_level_command(target, command) { + Ok(command_type) => command_type, + Err(error) => return Err(error), + }; + match classify_g17p_channel_command(target, command_type as u32) { + Ok(G17PChannelCommandDisposition::Process) => Ok(command_type), + Ok( + G17PChannelCommandDisposition::PmRebuildForbiddenFatal + | G17PChannelCommandDisposition::InvalidCommandFatal, + ) => Err(SubmissionError::CommandTagInvalidForFirmware), + Err(error) => Err(error), + } +} + +/// Apply the firmware's optional `tag <= 16 && bit(tag) & 0x1d03f` filter. +pub(crate) const fn g17p_channel_command_prefilter_allows( + target: G17ChannelAbiTarget, + raw: u32, +) -> Result { + match ensure_g17p_command_abi(target) { + Ok(()) => {} + Err(error) => return Err(error), + } + Ok(raw <= 16 && ((1u32 << raw) & G17P_CHANNEL_COMMAND_PREFILTER_MASK) != 0) +} + +// --------------------------------------------------------------------------- +// 9. Provenance ledger + live gaps. +// --------------------------------------------------------------------------- + +/// How each submission-ABI component was grounded, so reuse-vs-fork is explicit. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum AbiProvenance { + /// Byte-identical to the shipping AGX2 producer; reused unchanged. + IdenticalToAgx2, + /// AGX3-specific, grounded first-hand from matched host/firmware artifacts. + ForkedAgx3Grounded, + /// AGX2 layout the driver produces; the G17 firmware's *own* consumer + /// offsets were not read first-hand this pass. Kept unsupported. + Agx2InheritedUnconfirmed, +} + +/// Per-component provenance for the submission transport. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SubmissionAbiLedger { + pub(crate) tx_doorbell: AbiProvenance, + pub(crate) channel_state: AbiProvenance, + pub(crate) g17p_channel_state: AbiProvenance, + pub(crate) ring_state: AbiProvenance, + pub(crate) channel_descriptor: AbiProvenance, + pub(crate) host_channel_state_reset: AbiProvenance, + pub(crate) host_uncached_channel_reset: AbiProvenance, + pub(crate) g17p_command_types: AbiProvenance, + pub(crate) kick_word: AbiProvenance, + pub(crate) kick_queue_producer: AbiProvenance, + pub(crate) host_add_kicks_mmio: AbiProvenance, + pub(crate) g17p_queue_config: AbiProvenance, + pub(crate) g17p_completion_descriptors: AbiProvenance, + pub(crate) kick_record: AbiProvenance, +} + +/// The doorbell is byte-grounded (§1); the SKSM kick word/record and host +/// queue-timestamp producer are AGX3 forks grounded first-hand (§2-§5). The +/// generic channel/ring producer records remain inherited and unsupported. +/// A separate G17P-only channel-state entry records the matched A18 Pro host +/// producer and firmware consumer (§2c); it does not admit either M5 family. +/// The queue bridge/completion descriptors and command dispatch are grounded, +/// G17P-only AGX3 forks (§7-§8): the M5 +/// identities remain unsupported; unresolved values do not inherit AGX2 or +/// nested SKU meanings, and the host's `Nop = 17` is not firmware-admitted. +pub(crate) const SUBMISSION_ABI_LEDGER: SubmissionAbiLedger = SubmissionAbiLedger { + tx_doorbell: AbiProvenance::IdenticalToAgx2, + channel_state: AbiProvenance::Agx2InheritedUnconfirmed, + g17p_channel_state: AbiProvenance::ForkedAgx3Grounded, + ring_state: AbiProvenance::Agx2InheritedUnconfirmed, + channel_descriptor: AbiProvenance::ForkedAgx3Grounded, + host_channel_state_reset: AbiProvenance::ForkedAgx3Grounded, + host_uncached_channel_reset: AbiProvenance::ForkedAgx3Grounded, + g17p_command_types: AbiProvenance::ForkedAgx3Grounded, + kick_word: AbiProvenance::ForkedAgx3Grounded, + kick_queue_producer: AbiProvenance::ForkedAgx3Grounded, + host_add_kicks_mmio: AbiProvenance::ForkedAgx3Grounded, + g17p_queue_config: AbiProvenance::ForkedAgx3Grounded, + g17p_completion_descriptors: AbiProvenance::ForkedAgx3Grounded, + kick_record: AbiProvenance::ForkedAgx3Grounded, +}; + +/// The pinned G17P/G17G/G17X hosts all use the target-specific §6 MMIO +/// publication for `KSMKickQueue::addKicks`. EP 0x21 remains the separate +/// channel/work-ring doorbell; the two producers are complementary, not an +/// unresolved selector. +pub(crate) const HOST_SKSM_ADD_KICKS_PATH_SELECTED: bool = true; + +/// Whether the AGX3 per-pipe channel *descriptor* stride is proven. The G17S +/// firmware directly walks twelve 0x20-byte entries at +/// `runtime_pointers+0x020..0x19f` (§2a). +pub(crate) const CHANNEL_DESCRIPTOR_STRIDE_CONFIRMED: bool = true; + +/// The offline-unprovable pieces that keep the transport non-executable. Each +/// needs a live target device. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SubmissionLiveGaps { + /// Linux has an ordered pair writer on resource 0, including fixed-slot + /// geometry and serialization across queue owners. + pub(crate) sksm_resource0_ordered_write_implemented: bool, + /// EP 0x21 plus the retained CL2 channel queue has executed real work. + pub(crate) channel_doorbell_round_trip_proven: bool, + /// Linux allocates and publishes every object in the measured first-partial + /// graphics contract through its runtime path. + pub(crate) partial_graphics_runtime_implemented: bool, + /// The measured first-partial channel path executed paired graphics work. + pub(crate) partial_graphics_round_trip_proven: bool, + /// A native SKSM queue entry followed by the host `addKicks` MMIO pair has + /// executed real work. The shipping kext uses this path, but the classic + /// first-partial graphics path is independently executable. + pub(crate) sksm_add_kicks_round_trip_proven: bool, + /// Apple's host builds platform interrupt event sources from runtime + /// `interrupts`/`interrupts-valid` properties. D93AP physical IRQ 1001 and + /// RTBuddy AKF type 2 both converge exactly on source 4 and drain the main + /// firmware event rings, but an on-device SKSM completion round trip has + /// not proven that this ingress is sufficient for Linux completion. + pub(crate) completion_irq_route_proven: bool, + /// Ordinal-0 notification, entry, QID/timestamp, and EP0x20 stamp + /// correlation is implemented as a pure transaction before outstanding + /// subtraction. Physical IRQ/event-ring/ACK ownership remains separate. + pub(crate) normal_cl_completion_consumer_implemented: bool, +} + +impl SubmissionLiveGaps { + pub(crate) const fn classic_graphics_proven(&self) -> bool { + self.channel_doorbell_round_trip_proven + && self.partial_graphics_runtime_implemented + && self.partial_graphics_round_trip_proven + && self.completion_irq_route_proven + } + + pub(crate) const fn native_sksm_proven(&self) -> bool { + self.sksm_resource0_ordered_write_implemented + && self.sksm_add_kicks_round_trip_proven + && self.normal_cl_completion_consumer_implemented + && self.completion_irq_route_proven + } + + /// True when either independently modeled transport can complete work. + pub(crate) const fn all_proven(&self) -> bool { + self.classic_graphics_proven() || self.native_sksm_proven() + } +} + +/// The host-side writer is implemented. The channel-doorbell execution result +/// comes from `20260827-g17p-compact-contract-queue-live12`: exact 64-float +/// add3 output, then two more submissions on the same CL2 queue at queue and +/// channel counters 6/6/6 and 9/9/9. That does not prove the separate native +/// SKSM `addKicks` FIFO used by the shipping graphics channels. +pub(crate) const SUBMISSION_LIVE_GAPS: SubmissionLiveGaps = SubmissionLiveGaps { + sksm_resource0_ordered_write_implemented: true, + channel_doorbell_round_trip_proven: true, + partial_graphics_runtime_implemented: false, + partial_graphics_round_trip_proven: true, + sksm_add_kicks_round_trip_proven: false, + completion_irq_route_proven: false, + normal_cl_completion_consumer_implemented: true, +}; + +#[cfg(test)] +mod tests { + use super::*; + + /// Witnessed against the macOS capture: a real Metal render leaves the + /// parameter-buffer ring state at {7, 7, 0, 0}, and `PBState[+0x28]` + /// holds the full tagged address `0x1000190000`. The encode and decode + /// must be exact inverses -- that is what makes the field map from the two + /// kernelcache functions trustworthy. + #[test] + fn g17p_pbdesc_round_trips_the_witnessed_ring_state() { + assert_eq!(g17p_pbdesc_segment(0x10_0019_0000), 0); + assert_eq!( + g17p_pbdesc_reconstructed_address(0x10_0019_0000), + 0x10_0019_0000 + ); + assert_eq!( + g17p_pbdesc_reconstructed_address(0x70_0247_8000), + 0x70_0247_8000 + ); + assert_eq!( + g17p_pbdesc_reconstructed_address(0x0247_8000), + 0x80_0247_8000 + ); + let state = G17PPbDescRingState { + counter_a: 7, + counter_b: 7, + flag_high: 0, + flag_low: 0, + }; + let descriptor = encode_g17p_pbdesc([0; 4], 0x10_0019_0000, 0, state); + // dword0 carries the address >> 4; dword1 the segment >> 8 up top. + assert_eq!(descriptor[0], 0x0001_9000); + assert_eq!(descriptor[1] >> 28, 0x0); + assert_eq!(decode_g17p_pbdesc(descriptor), state); + + // Bits the firmware owns survive a re-encode. + let previous = [0x0000_0005u32, 0x0abc_defu32, 0xf000_0000u32, 0x4000_0000u32]; + let again = encode_g17p_pbdesc(previous, 0x10_0019_0000, 0x0012_3456, state); + assert_eq!(again[0] & 0x6, previous[0] & 0x6); + assert_eq!(again[1] & 0x003f_ffff, 0x0012_3456); + assert_eq!(again[2] & !0x003f_ffff, previous[2] & !0x003f_ffff); + assert_eq!(decode_g17p_pbdesc(again), state); + + assert_eq!(g17p_pbdesc_offset(2), 2 * G17P_PBDESC_ENTRY_SIZE); + } + + fn get_u64_le(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) + } + + fn get_u32_le(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) + } + + fn get_u16_le(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) + } + + fn source_fixture_fingerprint(raw: &[u8]) -> u64 { + raw.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) + } + + fn rce_binding_from_packed(word: u64) -> G17PClRceBinding { + G17PClRceBinding { + address: (word & G17P_CL_PACKED_ADDRESS_MASK) << 5, + tag: ((word >> 43) & KICK_QID_MASK as u64) as u8, + } + } + + fn native_add3_compute_operands( + descriptor: u64, + queue: u64, + converted_command_timestamp: u64, + event_mask: [u64; 4], + rce_words: [u64; 2], + ) -> G17PClKickEntryOperands<'static> { + let empty = G17PClRceBinding { address: 0, tag: 0 }; + G17PClKickEntryOperands { + descriptor_flag_4c: false, + descriptor_flag_5c8: false, + converted_command_timestamp, + barriers: &[], + mcache: None, + payload: [descriptor, queue], + event_mask, + rce_kind: 0, + rce_bindings: [ + rce_binding_from_packed(rce_words[0]), + rce_binding_from_packed(rce_words[1]), + empty, + empty, + ], + auxiliary: 0x003f_ffff_ffff_ffff, + } + } + + #[test] + fn compute_command_state_tracks_each_descriptor_slot() { + let first = prepare_g17p_compute_cl_command_state(0x70_0034_0000, 0, 0x26).unwrap(); + assert_eq!(first.event_mask, [0, 0, 4, 0]); + assert_eq!( + first.rce_bindings, + [ + G17PClRceBinding { + address: 0x70_0034_0040, + tag: 0x26, + }, + G17PClRceBinding { + address: 0x70_0034_0760, + tag: 0x04, + }, + G17PClRceBinding { address: 0, tag: 0 }, + G17PClRceBinding { address: 0, tag: 0 }, + ] + ); + + let second = prepare_g17p_compute_cl_command_state(0x70_0034_4000, 1, 0x26).unwrap(); + assert_eq!(second.event_mask, [0, 2, 8, 0]); + assert_eq!(second.rce_bindings[0].address, 0x70_0034_4040); + assert_eq!(second.rce_bindings[1].address, 0x70_0034_4760); + } + + #[test] + fn render_command_state_selects_native_descriptor_register_arrays() { + let ta = prepare_g17p_render_cl_command_state(0x70_0001_0740, true).unwrap(); + assert_eq!(ta.event_mask, [0; 4]); + assert_eq!( + ta.rce_bindings, + [ + G17PClRceBinding { + address: 0x70_0001_07a0, + tag: 0x47, + }, + G17PClRceBinding { address: 0, tag: 0 }, + G17PClRceBinding { address: 0, tag: 0 }, + G17PClRceBinding { address: 0, tag: 0 }, + ] + ); + + let fragment = + prepare_g17p_render_cl_command_state(0x70_0016_5800, false).unwrap(); + assert_eq!( + fragment.rce_bindings, + [ + G17PClRceBinding { + address: 0x70_0016_58a0, + tag: 0x56, + }, + G17PClRceBinding { + address: 0x70_0016_5fc0, + tag: 0x10, + }, + G17PClRceBinding { + address: 0x70_0016_66e0, + tag: 0x17, + }, + G17PClRceBinding { + address: 0x70_0016_6e00, + tag: 0x0a, + }, + ] + ); + } + + #[test] + fn tx_doorbell_matches_the_shipping_agx2_wire_form() { + // gpu.rs: MSG_TX_DOORBELL = 0x83 << 48; run_job kicks + // MSG_TX_DOORBELL | pipe_type | (index << 2). + const MSG_TX_DOORBELL: u64 = 0x83 << 48; + assert_eq!(EP_DOORBELL, 0x21); + for (pipe, raw) in [ + (PipeType::Vertex, 0u64), + (PipeType::Fragment, 1), + (PipeType::Compute, 2), + ] { + for index in 0u8..=7 { + let msg = encode_tx_doorbell(pipe, index).unwrap(); + assert_eq!(msg, MSG_TX_DOORBELL | raw | ((index as u64) << 2)); + // Firmware type extraction: ubfx x8,x1,#48,#6 == 3. + assert_eq!(doorbell_type(msg), DOORBELL_TYPE_TX); + assert_eq!(decode_tx_doorbell(msg), Ok((pipe, index))); + } + } + assert_eq!( + encode_tx_doorbell(PipeType::Vertex, 8), + Err(SubmissionError::DoorbellIndexOutOfRange) + ); + } + + #[test] + fn native_initial_render_uses_the_paired_priority_zero_doorbell() { + // Pipe identity at priority zero: 3D is the low Fragment pipe 1 and + // TA is the low Vertex pipe 0. Compute is the remaining pipe 2. + assert_eq!( + encode_tx_doorbell(PipeType::Fragment, 0), + Ok(0x0083_0000_0000_0001) + ); + assert_eq!( + encode_tx_doorbell(PipeType::Vertex, 0), + Ok(0x0083_0000_0000_0000) + ); + assert_eq!( + encode_tx_doorbell(PipeType::Compute, 0), + Ok(0x0083_0000_0000_0002) + ); + + assert_eq!(G17P_NATIVE_RENDER_DOORBELL_PRIORITY, 0); + assert_eq!(G17P_NATIVE_3D_DOORBELL, 0x0083_0000_0000_0001); + assert_eq!(G17P_NATIVE_TA_DOORBELL, 0x0083_0000_0000_0000); + assert_eq!( + encode_tx_doorbell(PipeType::Fragment, 0), + Ok(G17P_NATIVE_3D_DOORBELL) + ); + assert_eq!( + encode_tx_doorbell(PipeType::Vertex, 0), + Ok(G17P_NATIVE_TA_DOORBELL) + ); + } + + #[test] + fn doorbell_types_come_from_the_firmware_dispatch() { + // 0x83/0x84/0x85 & 0x3f -> 3/4/5, the firmware's cmp targets. + assert_eq!(DOORBELL_TYPE_TX, (0x83u8 & 0x3f)); + assert_eq!(DOORBELL_TYPE_FWCTL, (0x84u8 & 0x3f)); + assert_eq!(DOORBELL_TYPE_HALT, (0x85u8 & 0x3f)); + // A FwCtl (0x84<<48) doorbell is not a TX doorbell. + assert_eq!( + decode_tx_doorbell(0x84u64 << 48), + Err(SubmissionError::NotATxDoorbell) + ); + } + + #[test] + fn firmware_marks_pipe_three_invalid() { + // The firmware `cmp w8,#0x3 ; b.eq done` path: pipe 3 is not a pipe. + let msg = (0x83u64 << 48) | 0x3; + assert_eq!(doorbell_type(msg), DOORBELL_TYPE_TX); + assert_eq!(decode_tx_doorbell(msg), Err(SubmissionError::ReservedPipe)); + } + + #[test] + fn wakeup_doorbell_payloads_decode_like_the_firmware() { + // DOORBELL_KICKFW=0x10 -> pipe 0, index 4; DOORBELL_DEVCTRL=0x11 -> + // pipe 1, index 4 (msg & 3, (msg>>2)&7). + let kickfw = (0x83u64 << 48) | DOORBELL_KICKFW as u64; + assert_eq!(decode_tx_doorbell(kickfw), Ok((PipeType::Vertex, 4))); + let devctrl = (0x83u64 << 48) | DOORBELL_DEVCTRL as u64; + assert_eq!(decode_tx_doorbell(devctrl), Ok((PipeType::Fragment, 4))); + } + + #[test] + fn g17p_running_client_power_gate_precedes_every_submit_mutation() { + let contract = G17P_RUNNING_CLIENT_POWER_CONTRACT; + assert_eq!(contract.role_index, 1); + assert_eq!(contract.service_id, 0x20); + assert_eq!(contract.target, 1); + assert!(!contract.management_request_used); + assert!(contract.managed); + assert!(!contract.standalone); + assert_eq!( + G17P_COMPUTE_POWER_LEASE_ORDER, + [ + G17PComputePowerLeaseStep::AcquireBeforeReadiness, + G17PComputePowerLeaseStep::HoldThroughClaim, + G17PComputePowerLeaseStep::HoldThroughCompletion, + G17PComputePowerLeaseStep::ReleaseAfterCompletion, + ] + ); + + let mut vm_bind_mutations = 0; + let mut readiness_mutations = 0; + let mut sksm_mutations = 0; + let mut mailbox_mutations = 0; + let gated: Result<(), SubmissionError> = (|| { + require_g17p_running_client_power_transition(contract, true, false)?; + vm_bind_mutations += 1; + readiness_mutations += 1; + sksm_mutations += 1; + mailbox_mutations += 1; + Ok(()) + })(); + assert_eq!( + gated, + Err(SubmissionError::RunningClientPowerTransitionUnavailable) + ); + assert_eq!(vm_bind_mutations, 0); + assert_eq!(readiness_mutations, 0); + assert_eq!(sksm_mutations, 0); + assert_eq!(mailbox_mutations, 0); + assert_eq!( + require_g17p_running_client_power_transition(contract, true, true), + Ok(()) + ); + } + + #[test] + fn g17p_native_boot_is_ep21_quiet_until_first_real_user_cl() { + assert!(G17P_NATIVE_PRE_USER_EP21_MESSAGES.is_empty()); + assert_eq!( + G17P_NATIVE_FIRST_USER_EP21_MESSAGES, + [ + encode_g17p_cl_activation(2).unwrap(), + encode_g17p_direct_cl_kick(), + ] + ); + assert_eq!( + G17P_NATIVE_FIRST_USER_EP21_MESSAGES.map(doorbell_type), + [3, 3] + ); + assert!(!G17P_NATIVE_FIRST_USER_EP21_MESSAGES + .iter() + .any(|message| doorbell_type(*message) == 4)); + } + + #[test] + fn g17p_direct_cl_notifications_match_first_and_repeat_fixtures() { + const ACTIVATION: &[u8; 8] = + include_bytes!("fixtures/g17p_first_submit/ep21-first-activation.bin"); + const DIRECT_KICK: &[u8; 8] = + include_bytes!("fixtures/g17p_first_submit/ep21-direct-kick.bin"); + const FIRST_SEQUENCE: &[u8; 16] = + include_bytes!("fixtures/g17p_first_submit/ep21-first-direct-sequence.bin"); + + let first_state = G17PDirectClNotificationState { + cached_channel_identity: 0, + submitted_channel_identity: 0xffff_fc20_c07a_1dc0, + firmware_ready: true, + direct_feature: true, + submission_suppressed: false, + command_submission_enabled: true, + }; + let first = prepare_g17p_direct_cl_notifications(first_state).unwrap(); + assert_eq!(first.activation.unwrap().to_le_bytes(), *ACTIVATION); + assert_eq!(first.direct_kick.unwrap().to_le_bytes(), *DIRECT_KICK); + let mut sequence = [0u8; 16]; + sequence[..8].copy_from_slice(&first.activation.unwrap().to_le_bytes()); + sequence[8..].copy_from_slice(&first.direct_kick.unwrap().to_le_bytes()); + assert_eq!(sequence, *FIRST_SEQUENCE); + + let repeat = prepare_g17p_direct_cl_notifications(G17PDirectClNotificationState { + cached_channel_identity: first_state.submitted_channel_identity, + ..first_state + }) + .unwrap(); + assert_eq!(repeat.activation, None); + assert_eq!(repeat.direct_kick.unwrap().to_le_bytes(), *DIRECT_KICK); + + let not_ready = prepare_g17p_direct_cl_notifications(G17PDirectClNotificationState { + firmware_ready: false, + ..first_state + }) + .unwrap(); + assert_eq!(not_ready.activation, None); + assert_eq!(not_ready.direct_kick.unwrap().to_le_bytes(), *DIRECT_KICK); + + for disabled in [ + G17PDirectClNotificationState { + direct_feature: false, + ..first_state + }, + G17PDirectClNotificationState { + submission_suppressed: true, + ..first_state + }, + G17PDirectClNotificationState { + command_submission_enabled: false, + ..first_state + }, + ] { + let prepared = prepare_g17p_direct_cl_notifications(disabled).unwrap(); + assert_eq!(prepared.activation.unwrap().to_le_bytes(), *ACTIVATION); + assert_eq!(prepared.direct_kick, None); + } + + assert_eq!( + T8140_G17P_CL_BACKEND, + G17PClBackendConfig { + backend: G17ClSubmissionBackend::G17PHal200Hybrid, + direct_feature: true, + submission_suppressed: false, + command_submission_enabled: true, + } + ); + assert!(g17p_first_bind_tag15_required(true)); + assert!(g17p_first_bind_tag16_required( + T8140_G17P_CL_BACKEND, + true + )); + assert!(!g17p_first_bind_tag16_required( + G17PClBackendConfig { + backend: G17ClSubmissionBackend::Classic, + ..T8140_G17P_CL_BACKEND + }, + true + )); + assert!(!g17p_first_bind_tag16_required( + G17PClBackendConfig { + direct_feature: false, + ..T8140_G17P_CL_BACKEND + }, + true + )); + assert!(G17P_FIRST_BIND_COMMAND_PLAN.hardware_release_ready()); + assert!(G17P_FIRST_BIND_DIRECT_HARDWARE_RELEASE_READY); + + let first_lifecycle = prepare_g17p_direct_cl_lifecycle( + G17PDirectClLifecycleState { + queue_config_dirty: true, + cached_channel_identity: 0, + next_correlation_token: 1, + }, + first_state.submitted_channel_identity, + true, + T8140_G17P_CL_BACKEND, + ) + .unwrap(); + assert!(first_lifecycle.publish_config_update); + assert!(first_lifecycle.publish_entry_signal); + assert_eq!(first_lifecycle.correlation_token, 1); + assert_eq!(first_lifecycle.notifications, first); + assert_eq!( + first_lifecycle.next, + G17PDirectClLifecycleState { + queue_config_dirty: false, + cached_channel_identity: first_state.submitted_channel_identity, + next_correlation_token: 2, + } + ); + + let repeat_lifecycle = prepare_g17p_direct_cl_lifecycle( + first_lifecycle.next, + first_state.submitted_channel_identity, + true, + T8140_G17P_CL_BACKEND, + ) + .unwrap(); + assert!(!repeat_lifecycle.publish_config_update); + assert!(!repeat_lifecycle.publish_entry_signal); + assert_eq!(repeat_lifecycle.correlation_token, 2); + assert_eq!(repeat_lifecycle.notifications.activation, None); + assert_eq!(repeat_lifecycle.notifications.direct_kick, first.direct_kick); + assert_eq!(repeat_lifecycle.next.next_correlation_token, 3); + assert_eq!( + prepare_g17p_direct_cl_lifecycle( + G17PDirectClLifecycleState { + next_correlation_token: 0, + ..first_lifecycle.next + }, + first_state.submitted_channel_identity, + true, + T8140_G17P_CL_BACKEND, + ), + Err(SubmissionError::ComputeEventCounterOverflow) + ); + + let mut command_ring_mutations = 0; + let mut sksm_storage_mutations = 0; + let mut sksm_mmio_mutations = 0; + let mut outer_ring_mutations = 0; + let mut mailbox_mutations = 0; + let blocked_plan = G17PFirstBindCommandPlan { + optional_correlation_ring: G17PCorrelationRingImplementationState::Pending, + ..G17P_FIRST_BIND_COMMAND_PLAN + }; + let gated: Result<(), SubmissionError> = (|| { + require_g17p_first_bind_direct_hardware_release(blocked_plan)?; + command_ring_mutations += 2; + sksm_storage_mutations += 1; + sksm_mmio_mutations += 1; + outer_ring_mutations += 1; + mailbox_mutations += 2; + Ok(()) + })(); + assert_eq!( + gated, + Err(SubmissionError::FirstBindCommandPublicationIncomplete) + ); + assert_eq!(command_ring_mutations, 0); + assert_eq!(sksm_storage_mutations, 0); + assert_eq!(sksm_mmio_mutations, 0); + assert_eq!(outer_ring_mutations, 0); + assert_eq!(mailbox_mutations, 0); + + let resolved = G17P_FIRST_BIND_COMMAND_PLAN; + assert_eq!(resolved.fallback_logical_flush_id, 0); + assert!(!resolved.flush_id_serialized_in_tag15); + assert!(!resolved.flush_id_completion_release_required); + assert!(resolved.hardware_release_ready()); + assert_eq!( + require_g17p_first_bind_direct_hardware_release(resolved), + Ok(()) + ); + for unresolved_implementation in [ + G17PFlushIdImplementationState::Pending, + G17PFlushIdImplementationState::ProvenAbsentFallbackUnverified, + G17PFlushIdImplementationState::ProvenPresentUnimplemented, + ] { + assert!(!G17PFirstBindCommandPlan { + dedicated_flush_id: unresolved_implementation, + ..resolved + } + .hardware_release_ready()); + } + assert!(G17PFirstBindCommandPlan { + dedicated_flush_id: G17PFlushIdImplementationState::ProvenPresentImplemented, + ..resolved + } + .hardware_release_ready()); + assert!(!G17PFirstBindCommandPlan { + optional_correlation_ring: G17PCorrelationRingImplementationState::Pending, + ..resolved + } + .hardware_release_ready()); + assert!(!G17PFirstBindCommandPlan { + native_command_pointer_ring: G17PContractPredicateState::Pending, + ..resolved + } + .hardware_release_ready()); + assert!(G17PFirstBindCommandPlan { + optional_correlation_ring: G17PCorrelationRingImplementationState::ProvenInactive, + ..resolved + } + .hardware_release_ready()); + assert!(G17PFirstBindCommandPlan { + optional_correlation_ring: + G17PCorrelationRingImplementationState::ProvenRequiredImplemented, + ..resolved + } + .hardware_release_ready()); + assert!(!g17p_first_bind_tag16_required( + G17PClBackendConfig { + backend: G17ClSubmissionBackend::Hal300, + ..T8140_G17P_CL_BACKEND + }, + true + )); + } + + #[test] + fn pending_pipe_bit_matches_1_shl_pipe() { + assert_eq!(pending_pipe_bit(PipeType::Vertex), 1); + assert_eq!(pending_pipe_bit(PipeType::Fragment), 2); + assert_eq!(pending_pipe_bit(PipeType::Compute), 4); + } + + #[test] + fn inherited_channel_and_ring_state_offsets_are_labeled_agx2() { + // channel.rs raw::ChannelState: read_ptr@0x00, write_ptr@0x20. + assert_eq!(AGX2_CHANNEL_STATE_READ_PTR_OFFSET, 0x00); + assert_eq!(AGX2_CHANNEL_STATE_WRITE_PTR_OFFSET, 0x20); + assert_eq!(AGX2_CHANNEL_STATE_SIZE, 0x40); + // fw/workqueue.rs raw::RingState: doneptr@0x00, rptr@0x30, wptr@0x40, + // size@0x50 (0x10-strided). + assert_eq!(AGX2_RING_STATE_GPU_DONEPTR_OFFSET, 0x00); + assert_eq!(AGX2_RING_STATE_GPU_RPTR_OFFSET, 0x30); + assert_eq!(AGX2_RING_STATE_CPU_WPTR_OFFSET, 0x40); + assert_eq!(AGX2_RING_STATE_RB_SIZE_OFFSET, 0x50); + assert_eq!(AGX2_RING_ENTRY_BYTES, 8); + assert_eq!(HOST_MAX_OUTSTANDING_JOB_SLOTS, 127); + } + + #[test] + fn g17p_native_channel_layout_is_identity_scoped() { + assert_eq!( + g17_channel_abi(G17ChannelAbiTarget::A18ProG17P), + Ok(G17P_CHANNEL_ABI) + ); + assert_eq!( + g17_channel_abi(G17ChannelAbiTarget::M5G17G), + Err(SubmissionError::ChannelAbiUnconfirmed) + ); + assert_eq!( + g17_channel_abi(G17ChannelAbiTarget::M5ProMaxG17X), + Err(SubmissionError::ChannelAbiUnconfirmed) + ); + + assert_eq!( + G17P_CHANNEL_ABI.cached, + G17PCachedChannelStateLayout { + uncached_state_gpu_va: 0x00, + command_pointer_ring_gpu_va: 0x08, + coherency_cursor: 0x1c, + record_size: 0x80, + } + ); + assert_eq!( + G17P_CHANNEL_ABI.uncached, + G17PUncachedChannelStateLayout { + consumer: 0x00, + coherency_shadow: 0x30, + producer: 0x40, + recovery_target: 0x50, + ring_entry_count: 0x60, + cursor_width: 4, + write_extent: 0x64, + } + ); + assert_eq!(G17P_CHANNEL_ABI.command_pointer_entry_bytes, 8); + assert_eq!(G17P_CHANNEL_ABI.accelerator_target_width, 2); + assert!(G17P_CHANNEL_PRODUCER_REQUIRES_DMB_ISH); + } + + #[test] + fn g17p_classic_work_slots_and_sksm_pointer_entries_are_distinct() { + assert_eq!(G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE, 0x18); + assert_eq!(G17P_CHANNEL_ABI.command_pointer_entry_bytes, 0x08); + + let sksm = prepare_g17p_channel_write( + G17ChannelAbiTarget::A18ProG17P, + 1, + 3, + 4, + ) + .unwrap(); + assert_eq!(sksm.entry_byte_offset, 0x08); + + let classic = prepare_g17p_partial_opening_outer_slot( + G17PPartialOpeningOuterChannelState { + consumers: [0, 0], + producer: 1, + queue_gpu_va: 0xffff_fc20_c000_00c0, + queue_write_index: 3, + }, + G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX, + 1, + 1, + ) + .unwrap(); + assert_eq!(classic.slot_byte_offset, 0x18); + assert_eq!(get_u64_le(&classic.slot, 0x08), 0xffff_fc20_c000_00c0); + } + + #[test] + fn g17p_channel_write_plan_matches_host_ordering_and_wrap() { + assert_eq!( + prepare_g17p_channel_write(G17ChannelAbiTarget::A18ProG17P, 2, 0, 4), + Ok(G17PChannelWritePlan { + entry_byte_offset: 0x10, + next_producer: 3, + }) + ); + assert_eq!( + prepare_g17p_channel_write(G17ChannelAbiTarget::A18ProG17P, 3, 2, 4), + Ok(G17PChannelWritePlan { + entry_byte_offset: 0x18, + next_producer: 0, + }) + ); + assert_eq!( + prepare_g17p_channel_write(G17ChannelAbiTarget::A18ProG17P, 1, 2, 4), + Err(SubmissionError::RingFull) + ); + assert_eq!( + prepare_g17p_channel_write(G17ChannelAbiTarget::A18ProG17P, 0, 0, 0), + Err(SubmissionError::RingSizeZero) + ); + assert_eq!( + prepare_g17p_channel_write(G17ChannelAbiTarget::M5ProMaxG17X, 0, 2, 4), + Err(SubmissionError::ChannelAbiUnconfirmed) + ); + } + + #[test] + fn g17p_partial_opening_pages_and_context_match_the_successful_graph() { + assert_eq!(G17P_PARTIAL_OPENING_PAGE_SIZE, 0x4000); + assert_eq!(G17P_PARTIAL_OPENING_SCHEDULER_GPU_VA, 0xffff_fc20_015e_0000); + assert_eq!( + G17P_PARTIAL_OPENING_COMPUTED_GPU_VA, + 0xffff_fc20_015d_8000 + ); + assert_eq!( + G17P_PARTIAL_OPENING_PRIMARY_INDEX_FIRMWARE_GPU_VA, + 0xffff_fc20_c084_8000 + ); + assert_eq!( + G17P_PARTIAL_OPENING_PRIMARY_INDEX_GPU_VA, + 0x0000_0010_0019_0000 + ); + assert_eq!(G17P_PARTIAL_OPENING_PRIMARY_INDEX_SIZE, 0x10000); + + let scheduler = build_g17p_partial_opening_scheduler_page(); + assert_eq!(get_u64_le(&scheduler, 0), 0x0000_0020_0001_9000); + assert!(scheduler[8..].iter().all(|value| *value == 0)); + let mut scheduler_slice = [0xa5; G17P_PARTIAL_OPENING_PAGE_SIZE]; + assert_eq!( + apply_g17p_partial_opening_scheduler_page(&mut scheduler_slice), + Ok(()) + ); + assert_eq!(scheduler_slice, scheduler); + + let primary = build_g17p_partial_opening_primary_index_page(); + let expected = [ + 0x11, 0x12, 0x13, 0x14, 0x16, 0x17, 0x18, 0x19, 0x1b, 0x1c, 0x1d, 0x1e, 0x20, 0x21, + 0x22, 0x23, 0x25, 0x26, 0x27, 0x28, 0x2a, 0x2b, 0x2c, 0x2d, 0x3c, 0x3d, 0x3e, 0x3f, + 0x41, 0x42, 0x43, 0x44, + ]; + for (index, value) in expected.into_iter().enumerate() { + assert_eq!(get_u32_le(&primary, index * 4), value); + } + assert!(primary[expected.len() * 4..] + .iter() + .all(|value| *value == 0)); + let mut primary_slice = [0xa5; G17P_PARTIAL_OPENING_PAGE_SIZE]; + assert_eq!( + apply_g17p_partial_opening_primary_index_page(&mut primary_slice), + Ok(()) + ); + assert_eq!(primary_slice, primary); + assert_eq!( + apply_g17p_partial_opening_scheduler_page(&mut []), + Err(SubmissionError::PartialOpeningPageSize) + ); + assert_eq!( + apply_g17p_partial_opening_primary_index_page(&mut [0; 1]), + Err(SubmissionError::PartialOpeningPageSize) + ); + + assert_eq!( + G17P_PARTIAL_OPENING_CONTEXT, + G17PPartialOpeningContextContract { + context0_root_slot: 0, + context0_context_id: 0, + render_root_slot: 1, + render_context_id: 1, + context0_empty_high_root_slot: 0, + render_empty_high_root_slot: 1, + transport_pair: 0, + descriptor_pair: 0, + shared_queue_pair_namespace: 0, + optional_identity_context_id: 1, + descriptor_context_id: 1, + scheduler_class: 2, + optional_grids: [0, 1], + queue_identity: 0x15, + optional_submission_ordinal: 0, + event_counter: 0x102, + work_doorbell_channel: 0, + context0_low_root_present_at_first_work: true, + render_low_root_present_at_first_work: true, + source_firmware_high_root_table_resident: false, + } + ); + } + + #[test] + fn g17p_cold_opening_support_graph_matches_the_source_bytes() { + let addresses = G17PColdOpeningGraphAddresses { + primary_index: 0x70_0100_0000, + secondary_index: 0x70_0100_4000, + pool_a_slots: 0x70_0100_8000, + pool_b_slots: 0x70_0100_c000, + shared_slots: 0x70_0101_0000, + flag: 0x70_0101_4000, + }; + let mut page = [0xa5; G17P_PARTIAL_OPENING_PAGE_SIZE]; + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::SecondaryIndex, &mut page).unwrap(); + for (index, expected) in [0x11, 0x16, 0x1b, 0x20, 0x25, 0x2a, 0x3c, 0x41] + .into_iter() + .enumerate() + { + assert_eq!(get_u64_le(&page, index * 8), expected); + } + assert_eq!(source_fixture_fingerprint(&page), 0x5538_4e84_8d5a_840b); + assert!(page[0x40..].iter().all(|value| *value == 0)); + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::PoolASlots, &mut page).unwrap(); + assert_eq!(get_u32_le(&page, 4), 2); + assert_eq!(source_fixture_fingerprint(&page), 0xcfc1_218a_7364_2c07); + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::PoolBSlots, &mut page).unwrap(); + assert_eq!(source_fixture_fingerprint(&page), 0x9c1b_da7f_8c87_2325); + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::SharedSlots, &mut page).unwrap(); + assert_eq!(get_u32_le(&page, 0), 8); + assert_eq!(get_u32_le(&page, 4), 8); + assert_eq!(get_u32_le(&page, 0x60), 1); + assert_eq!(source_fixture_fingerprint(&page), 0x53c2_7f11_bcfb_4124); + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::Flag, &mut page).unwrap(); + assert_eq!(get_u32_le(&page, 0), 1); + assert_eq!(source_fixture_fingerprint(&page), 0xba85_253c_a046_2324); + + apply_g17p_cold_opening_leaf_page(G17PColdOpeningLeaf::PrimaryIndex, &mut page).unwrap(); + assert_eq!(source_fixture_fingerprint(&page), 0x85e8_d166_b202_3e2d); + + let mut pool_a = [0u8; G17P_COLD_OPENING_RECORD_POOL_A_SIZE]; + apply_g17p_cold_opening_record_pool_a(addresses, &mut pool_a).unwrap(); + assert_eq!(get_u64_le(&pool_a, 0), addresses.pool_a_slots); + assert_eq!(get_u32_le(&pool_a, 0x10), 0x50); + assert_eq!( + get_u64_le(&pool_a, 34 * 0x100), + addresses.pool_a_slots + 34 * 4 + ); + assert_eq!(source_fixture_fingerprint(&pool_a), 0x42cb_4f83_a145_54be); + + let mut pool_b = [0u8; G17P_COLD_OPENING_RECORD_POOL_B_SIZE]; + apply_g17p_cold_opening_record_pool_b(addresses, &mut pool_b).unwrap(); + assert_eq!(get_u32_le(&pool_b, 0), 0x80004); + assert_eq!(get_u32_le(&pool_b, 0x28), 0x178020); + assert_eq!(get_u32_le(&pool_b, 35 * 0x80 + 0x28), 0x178000); + assert_eq!(get_u32_le(&pool_b, 36 * 0x80 + 0x28), 0x178020); + assert_eq!(get_u64_le(&pool_b, 0x40), addresses.shared_slots + 0x40); + assert_eq!(get_u32_le(&pool_b, 0x4c), 1); + assert_eq!(source_fixture_fingerprint(&pool_b), 0xb98b_d0a4_1c25_9a6e); + + let mut shared = [0u8; G17P_COLD_OPENING_SHARED_OBJECT_SIZE]; + apply_g17p_cold_opening_shared_object(addresses, &mut shared).unwrap(); + assert_eq!(get_u32_le(&shared, 0x0c), 0); + assert_eq!( + get_u32_le(&shared, 0x0c), + u32::from(G17P_PARTIAL_OPENING_CONTEXT.shared_queue_pair_namespace) + ); + assert_eq!(get_u64_le(&shared, 0x20), addresses.primary_index); + assert_eq!(get_u32_le(&shared, 0x34), 0x20); + assert_eq!(get_u32_le(&shared, 0x3c), 8); + assert_eq!(get_u64_le(&shared, 0x44), addresses.secondary_index); + assert_eq!(get_u64_le(&shared, 0x4c), addresses.shared_slots); + assert_eq!(get_u32_le(&shared, 0x54), 0x1f); + assert_eq!(get_u64_le(&shared, 0x64), addresses.flag); + assert_eq!(source_fixture_fingerprint(&shared), 0x5d97_133c_cbf5_a77c); + + let graph_range = 0x70_0100_0000..0x70_0101_8000; + for address in [ + get_u64_le(&pool_a, 0), + get_u64_le(&pool_a, 34 * 0x100), + get_u64_le(&pool_b, 0x08), + get_u64_le(&pool_b, 78 * 0x80 + 0x08), + get_u64_le(&pool_b, 0x40), + get_u64_le(&shared, 0x20), + get_u64_le(&shared, 0x44), + get_u64_le(&shared, 0x4c), + get_u64_le(&shared, 0x64), + ] { + assert!(graph_range.contains(&address)); + } + + let mut control = [0u8; G17P_COLD_OPENING_SHARED_CONTROL_SIZE]; + apply_g17p_cold_opening_shared_control(0x70_0020_8000, 0x70_0102_0000, &mut control) + .unwrap(); + assert_eq!(get_u32_le(&control, 0), 1); + assert_eq!(get_u32_le(&control, 0x10), 2); + assert_eq!(get_u64_le(&control, 0x20), 0x0000_1900_0000_0000); + assert_eq!(get_u64_le(&control, 0x30), 0x70_0020_8000); + assert_eq!(get_u32_le(&control, 0x48), 0xe0); + assert_eq!(get_u64_le(&control, 0x4c), 0x70_0102_0000); + assert_eq!(get_u32_le(&control, 0x60), 3); + assert_eq!(source_fixture_fingerprint(&control), 0x9457_5ea8_9f02_1db4); + assert!(G17P_COLD_OPENING_OPERAND_TABLE_GPU_VA >= 0x70_0000_0000); + assert!(G17P_COLD_OPENING_OPERAND_TABLE_GPU_VA < 0x71_0000_0000); + assert!((0x70_0102_0000..0x70_0102_4000).contains(&get_u64_le(&control, 0x4c))); + let mut inner = [0u8; G17P_COLD_OPENING_SHARED_CONTROL_INNER_SIZE]; + apply_g17p_cold_opening_shared_control_inner(&mut inner).unwrap(); + assert_eq!(get_u32_le(&inner, 0), 2); + assert_eq!(source_fixture_fingerprint(&inner), 0x8d1a_ce90_4a39_8d17); + + let mut channel = [0u8; G17P_COLD_OPENING_CHANNEL_CONTROL_SIZE]; + apply_g17p_cold_opening_channel_control(&mut channel).unwrap(); + assert_eq!(get_u64_le(&channel, 0), 0x0000_0100_0000_ffff); + assert_eq!(get_u64_le(&channel, 0x20), 0x0002_0000_0000_0000); + assert_eq!(get_u64_le(&channel, 0x30), 0x0000_0000_ff00_0000); + assert_eq!(source_fixture_fingerprint(&channel), 0x448e_89f5_b330_26cb); + } + + #[test] + fn g17p_native_cold_pool_a_starts_at_initialized_record_one_only() { + let addresses = G17PColdOpeningGraphAddresses { + primary_index: 0x70_0100_0000, + secondary_index: 0x70_0100_4000, + pool_a_slots: 0x70_0100_8000, + pool_b_slots: 0x70_0100_c000, + shared_slots: 0x70_0101_0000, + flag: 0x70_0101_4000, + }; + assert_eq!(g17p_render_pool_a_record_index(0, false), 0); + assert_eq!(g17p_render_pool_a_record_index(0, true), 1); + assert_eq!(g17p_render_pool_a_record_index(1, false), 2); + assert_eq!(g17p_render_pool_a_record_index(1, true), 2); + + let mut slots = [0xa5; G17P_PARTIAL_OPENING_PAGE_SIZE]; + apply_g17p_cold_opening_pool_a_start_slot(1, &mut slots).unwrap(); + assert_eq!(get_u32_le(&slots, 4), 1); + assert_eq!(get_u32_le(&slots, 8), 0); + assert!(slots[..4].iter().all(|byte| *byte == 0)); + assert!(slots[12..].iter().all(|byte| *byte == 0)); + + let mut records = [0xa5; G17P_COLD_OPENING_RECORD_POOL_A_SIZE]; + apply_g17p_cold_opening_record_pool_a_at(addresses, 1, &mut records).unwrap(); + assert_eq!(get_u64_le(&records, 0), addresses.pool_a_slots); + assert_eq!(get_u32_le(&records, 0x10), 0); + assert_eq!(get_u64_le(&records, 0x100), addresses.pool_a_slots + 4); + assert_eq!(get_u32_le(&records, 0x110), 0x50); + assert_eq!(get_u32_le(&records, 0x108), 0); + assert_eq!( + get_u64_le( + &records, + (G17P_COLD_OPENING_RECORD_POOL_A_COUNT - 1) + * G17P_COLD_OPENING_RECORD_POOL_A_STRIDE, + ), + addresses.pool_a_slots + + ((G17P_COLD_OPENING_RECORD_POOL_A_COUNT - 1) * 4) as u64 + ); + } + + #[test] + fn g17p_cold_opening_work_records_keep_identity_context_and_relocatable_pointers() { + let optional_addresses = G17PColdOpeningOptionalAddresses { + context_scratch: 0x70_0200_0000, + firmware_scratch: 0x70_0200_4000, + usc_priv_mem_flist: 0x70_0200_8000, + usc_freelist_hardware_buffer_id: 0, + channel_control: 0x70_0200_c000, + tiling_shared_object: 0x70_0201_0000, + parameter_buffer_token: 0, + owner_pid: 0x15, + ta_hardware_buffer_id: 0, + }; + let supplied = 0x70_0200_0000..0x70_0202_0000; + for stage in [G17PColdOpeningStage::Tiling, G17PColdOpeningStage::Fragment] { + let mut optional = [0u8; G17P_COLD_OPENING_OPTIONAL_SIZE]; + apply_g17p_cold_opening_optional( + stage, + optional_addresses, + 0, + 0, + 0, + 1, + 1, + &mut optional, + ) + .unwrap(); + assert_eq!(get_u32_le(&optional, 0), 0x0f); + for offset in [0x08, 0x10, 0x36, 0x4a] { + assert!(supplied.contains(&get_u64_le(&optional, offset))); + } + assert_eq!(get_u16_le(&optional, 0x1e), 2); + assert_eq!(get_u16_le(&optional, 0x32), 1); + assert_eq!(get_u16_le(&optional, 0x46), 0); + assert_eq!(get_u16_le(&optional, 0x56), 0); + assert_eq!(get_u32_le(&optional, 0x5a), 0x15); + assert_eq!(get_u16_le(&optional, 0x5e), 2); + assert_eq!(get_u16_le(&optional, 0x3e), 0); + assert_eq!( + source_fixture_fingerprint(&optional), + match stage { + G17PColdOpeningStage::Tiling => 0x3511_6ec9_a9f3_7149, + G17PColdOpeningStage::Fragment => 0x2a62_5e4c_5170_22ef, + } + ); + + let mut event = [0u8; G17P_COLD_OPENING_EVENT_SIZE]; + apply_g17p_cold_opening_event(stage, 0, &mut event).unwrap(); + assert_eq!(get_u32_le(&event, 0x08), 0x102); + assert_eq!( + source_fixture_fingerprint(&event), + match stage { + G17PColdOpeningStage::Tiling => 0x9eb4_fa84_39f9_667b, + G17PColdOpeningStage::Fragment => 0x86a6_d8e2_4f16_eea1, + } + ); + + let descriptor = if stage == G17PColdOpeningStage::Tiling { + G17P_COLD_OPENING_TA_DESCRIPTOR_BASE + } else { + G17P_COLD_OPENING_3D_DESCRIPTOR_BASE + }; + let queue = if stage == G17PColdOpeningStage::Tiling { + 0x70_0201_c000 + } else { + 0x70_0201_e000 + }; + let mut context = [0u8; G17P_COLD_OPENING_QUEUE_CONTEXT_SIZE]; + apply_g17p_cold_opening_queue_context(stage, descriptor, queue, &mut context).unwrap(); + assert_eq!(get_u64_le(&context, 0x210), descriptor); + assert_eq!(get_u64_le(&context, 0x218), queue); + assert_eq!( + get_u64_le(&context, 0x200), + match stage { + G17PColdOpeningStage::Tiling => 0x1000_0000_0000_0004, + G17PColdOpeningStage::Fragment => 0x1000_0400_0000_0004, + } + ); + if stage == G17PColdOpeningStage::Fragment { + assert_eq!(get_u64_le(&context, 0x208), 0); + assert_eq!(get_u64_le(&context, 0x358), 0x0000_8003_8000_4c3e); + assert_eq!(get_u64_le(&context, 0x360), 0x0000_b803_8000_4c77); + assert_eq!(get_u64_le(&context, 0x368), 0x0000_5003_8000_4cb0); + } else { + assert_eq!(get_u64_le(&context, 0x350), 0x0002_3803_8000_0003); + } + assert!(supplied.contains(&queue)); + assert_eq!( + source_fixture_fingerprint(&context), + match stage { + G17PColdOpeningStage::Tiling => 0xd30d_fd3c_7336_4025, + G17PColdOpeningStage::Fragment => 0xdfda_d80d_e0cc_292b, + } + ); + } + + let queue_addresses = G17PColdOpeningQueueAddresses { + pointers: 0x70_0200_0000, + item_ring: 0x70_0200_4000, + job_list: 0x70_0200_8000, + channel_control: 0x70_0200_c000, + owner_pid: 0x15, + }; + let mut queue = [0u8; G17P_COLD_OPENING_QUEUE_RECORD_SIZE]; + apply_g17p_cold_opening_queue_record(queue_addresses, &mut queue).unwrap(); + for offset in [0x00, 0x08, 0x10, 0x70] { + assert!(supplied.contains(&get_u64_le(&queue, offset))); + } + assert_eq!(get_u32_le(&queue, 0x48), 0); + assert_eq!(get_u32_le(&queue, 0x28), 2); + assert_eq!(get_u32_le(&queue, 0x3c), 2); + assert_eq!(get_u32_le(&queue, 0x40), u32::MAX); + assert_eq!(get_u32_le(&queue, 0x44), 0x15); + assert_eq!(source_fixture_fingerprint(&queue), 0x0704_bf11_9ee1_8084); + + let mut pointers = [0u8; G17P_COLD_OPENING_POINTER_BLOCK_SIZE]; + apply_g17p_cold_opening_pointer_block(&mut pointers).unwrap(); + assert_eq!(get_u32_le(&pointers, 0x50), u32::MAX); + assert_eq!(get_u64_le(&pointers, 0x60), 0x500); + assert_eq!(source_fixture_fingerprint(&pointers), 0x41b7_257d_6bc8_1cb2); + + let mut descriptor = [0u8; G17P_PARTIAL_OPENING_PAGE_SIZE]; + apply_g17p_cold_opening_descriptor_overrides(G17PColdOpeningStage::Tiling, &mut descriptor) + .unwrap(); + assert_eq!(get_u32_le(&descriptor, 0x0c), 1); + assert_eq!(descriptor[0x0789], 0x08); + assert_eq!(&descriptor[0x093e..0x0940], &[0xd0, 0x91]); + assert_eq!( + source_fixture_fingerprint(&descriptor), + 0x0a67_4d00_ea32_e630 + ); + + descriptor.fill(0); + apply_g17p_cold_opening_descriptor_overrides( + G17PColdOpeningStage::Fragment, + &mut descriptor, + ) + .unwrap(); + assert_eq!( + source_fixture_fingerprint(&descriptor), + 0xaac5_6b92_e70a_d459 + ); + } + + #[test] + fn g17p_cold_opening_context_locators_follow_descriptor_relocation() { + for (stage, base, locator_offsets) in [ + ( + G17PColdOpeningStage::Tiling, + G17P_COLD_OPENING_TA_DESCRIPTOR_BASE, + &[0x350usize][..], + ), + ( + G17PColdOpeningStage::Fragment, + G17P_COLD_OPENING_3D_DESCRIPTOR_BASE, + &[0x350usize, 0x358, 0x360, 0x368][..], + ), + ] { + let mut native = [0u8; G17P_COLD_OPENING_QUEUE_CONTEXT_SIZE]; + let mut relocated = [0u8; G17P_COLD_OPENING_QUEUE_CONTEXT_SIZE]; + apply_g17p_cold_opening_queue_context(stage, base, 0x70_0201_c000, &mut native) + .unwrap(); + apply_g17p_cold_opening_queue_context( + stage, + base + 0x8000, + 0x70_0201_c000, + &mut relocated, + ) + .unwrap(); + + for offset in locator_offsets { + assert_eq!( + get_u64_le(&relocated, *offset), + get_u64_le(&native, *offset) + 0x400 + ); + } + assert_eq!(get_u64_le(&relocated, 0x210), base + 0x8000); + + assert_eq!( + apply_g17p_cold_opening_queue_context( + stage, + base - 0x20, + 0x70_0201_c000, + &mut relocated, + ), + Err(SubmissionError::PartialOpeningDescriptorLocatorUnencodable) + ); + assert_eq!( + apply_g17p_cold_opening_queue_context( + stage, + base + 1, + 0x70_0201_c000, + &mut relocated, + ), + Err(SubmissionError::PartialOpeningDescriptorLocatorUnencodable) + ); + } + } + + #[test] + fn g17p_cold_opening_operand_graph_is_exact() { + let mut table = [0u8; G17P_PARTIAL_OPENING_PAGE_SIZE]; + apply_g17p_cold_opening_operand_table(&mut table).unwrap(); + assert_eq!(source_fixture_fingerprint(&table), 0x9c1b_da7f_8c87_2325); + assert!(table.iter().all(|value| *value == 0)); + + apply_g17p_populated_operand_table(&mut table).unwrap(); + assert_eq!( + get_u64_le(&table, 0), + (1u64 << 60) | G17P_COLD_OPENING_OPERAND_BUFFER_BASE + ); + assert_eq!( + get_u64_le(&table, 27 * 0x40), + (1u64 << 60) + | (G17P_COLD_OPENING_OPERAND_BUFFER_BASE + + 27 * G17P_COLD_OPENING_OPERAND_BUFFER_STRIDE) + ); + assert!(table[28 * 0x40..].iter().all(|value| *value == 0)); + assert_eq!(source_fixture_fingerprint(&table), 0x61a7_b063_3bea_e785); + + let mut directory = [0u8; G17P_COLD_OPENING_OPERAND_DIRECTORY_SIZE]; + apply_g17p_cold_opening_operand_directory(&mut directory).unwrap(); + assert_eq!( + source_fixture_fingerprint(&directory), + 0xeb05_052e_a5b6_2325 + ); + assert!(directory.iter().all(|value| *value == 0)); + + apply_g17p_populated_operand_directory(&mut directory).unwrap(); + assert_eq!(get_u64_le(&directory, 0), 0x70_0022_0000); + assert_eq!(get_u64_le(&directory, 255 * 8), 0x70_0031_f000); + assert_eq!(get_u64_le(&directory, 256 * 8), 0x70_0032_8000); + assert_eq!(get_u64_le(&directory, (7168 - 1) * 8), 0x70_01ef_7000); + assert!(directory[7168 * 8..].iter().all(|value| *value == 0)); + assert_eq!( + source_fixture_fingerprint(&directory), + 0xd5c6_623f_4366_03a5 + ); + } + + #[test] + fn g17p_partial_opening_status_is_exact_and_checked() { + const KERNEL_VA_BASE: u64 = 0xffff_fc20_0000_0000; + assert_eq!( + prepare_g17p_partial_opening_pre_0x84_status(KERNEL_VA_BASE), + Ok([ + G17PPartialOpeningStatusWrite { + offset: 0x4018, + value: 0x0005_0601_0000_0000, + }, + G17PPartialOpeningStatusWrite { + offset: 0x4020, + value: 61, + }, + G17PPartialOpeningStatusWrite { + offset: 0x40b0, + value: 5, + }, + G17PPartialOpeningStatusWrite { + offset: 0x48e0, + value: KERNEL_VA_BASE + 0x001c_0000, + }, + G17PPartialOpeningStatusWrite { + offset: 0x48e8, + value: KERNEL_VA_BASE + 0x001c_0040, + }, + ]) + ); + assert_eq!( + prepare_g17p_partial_opening_pre_0x84_status(u64::MAX), + Err(SubmissionError::PartialOpeningAddressOverflow) + ); + } + + #[test] + fn g17p_partial_opening_publishes_3d_then_ta_with_one_pair_kick() { + assert_eq!( + G17P_PARTIAL_OPENING_PUBLICATION_ORDER, + [ + G17PPartialOpeningPublicationStep::PrimarySchedulerPage, + G17PPartialOpeningPublicationStep::PrimaryIndexPage, + G17PPartialOpeningPublicationStep::PreControlStatus, + G17PPartialOpeningPublicationStep::ControlDone0x84, + G17PPartialOpeningPublicationStep::FragmentState, + G17PPartialOpeningPublicationStep::EmptyHighRoots, + G17PPartialOpeningPublicationStep::FragmentProducer, + G17PPartialOpeningPublicationStep::LateTilingState, + G17PPartialOpeningPublicationStep::SystemBarrierBeforeTilingProducer, + G17PPartialOpeningPublicationStep::TilingProducer, + G17PPartialOpeningPublicationStep::SystemBarrierBeforePairedDoorbell, + G17PPartialOpeningPublicationStep::PairedWorkDoorbell0x83, + ] + ); + let prepared = prepare_g17p_partial_opening_work_pair( + G17PPartialOpeningOuterChannelState { + consumers: [0, 0], + producer: 0, + queue_gpu_va: 0xffff_fc20_c000_00c0, + queue_write_index: 3, + }, + G17PPartialOpeningOuterChannelState { + consumers: [0, 0], + producer: 0, + queue_gpu_va: 0xffff_fc20_c000_0000, + queue_write_index: 3, + }, + 1, + 0, + true, + ) + .unwrap(); + assert_eq!( + prepared.fragment, + G17PPartialOpeningOuterSlotPlan { + channel_table_index: 7, + slot_index: 0, + slot_byte_offset: 0, + slot: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0xc0, 0x00, 0x00, 0xc0, 0x20, 0xfc, 0xff, 0xff, 1, 0, + 0, 0, 3, 0, 1, 1, + ], + next_producer: 1, + }, + 1, + 0 + ); + assert_eq!( + prepared.tiling, + G17PPartialOpeningOuterSlotPlan { + channel_table_index: 6, + slot_index: 0, + slot_byte_offset: 0, + slot: [ + 0, 0, 0, 0, 0, 0, 0, 0, 0x00, 0x00, 0x00, 0xc0, 0x20, 0xfc, 0xff, 0xff, 0, 0, + 0, 0, 3, 0, 0, 1, + ], + next_producer: 1, + } + ); + assert_eq!(G17P_PARTIAL_OPENING_OUTER_SLOT_SIZE, 0x18); + assert_eq!(G17P_PARTIAL_OPENING_OUTER_SLOT_COUNT, 0x100); + assert_eq!(G17P_PARTIAL_OPENING_TILING_CHANNEL_TABLE_INDEX, 6); + assert_eq!(G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX, 7); + assert_eq!(G17P_PARTIAL_OPENING_WORK_DOORBELL_CHANNEL, 0); + assert_eq!(prepared.paired_doorbell, 0x0083_0000_0000_0000); + assert_eq!( + prepared.paired_doorbell, + encode_tx_doorbell(PipeType::Vertex, 0).unwrap() + ); + + let compute = prepare_g17p_compute_channel_work( + G17PPartialOpeningOuterChannelState { + consumers: [0, 0], + producer: 0, + queue_gpu_va: 0xffff_fc20_c100_0000, + queue_write_index: 3, + }, + 4, + false, + ) + .unwrap(); + assert_eq!(compute.outer.channel_table_index, 8); + assert_eq!(compute.outer.slot_index, 0); + assert_eq!(compute.outer.next_producer, 1); + assert_eq!( + u64::from_le_bytes(compute.outer.slot[8..16].try_into().unwrap()), + 0xffff_fc20_c100_0000 + ); + assert_eq!( + u32::from_le_bytes(compute.outer.slot[0x10..0x14].try_into().unwrap()), + 2 + ); + assert_eq!( + u32::from_le_bytes(compute.outer.slot[0x14..0x18].try_into().unwrap()), + 0x0104_0003 + ); + assert_eq!(compute.doorbell, 0x0083_0000_0000_000a); + + let compute_repeat = prepare_g17p_compute_channel_work( + G17PPartialOpeningOuterChannelState { + consumers: [1, 1], + producer: 1, + queue_gpu_va: 0xffff_fc20_c100_0000, + queue_write_index: 7, + }, + 4, + true, + ) + .unwrap(); + assert_eq!( + u32::from_le_bytes( + compute_repeat.outer.slot[0x14..0x18] + .try_into() + .unwrap() + ), + 0x0004_0007 + ); + + let full = G17PPartialOpeningOuterChannelState { + consumers: [0, 7], + producer: 6, + queue_gpu_va: 0, + queue_write_index: 3, + }; + assert_eq!( + prepare_g17p_partial_opening_work_pair(full, full, 1, 0, true), + Err(SubmissionError::RingFull) + ); + assert_eq!( + prepare_g17p_partial_opening_work_pair( + G17PPartialOpeningOuterChannelState { + producer: 0x100, + ..full + }, + full, + 1, + 0, + true, + ), + Err(SubmissionError::PartialOpeningChannelCounterOutOfRange) + ); + assert_eq!( + prepare_g17p_partial_opening_work_pair( + G17PPartialOpeningOuterChannelState { + consumers: [0, 0], + producer: 0, + queue_write_index: 0x1_0000, + ..full + }, + full, + 1, + 0, + true, + ), + Err(SubmissionError::PartialOpeningQueueHeadOutOfRange) + ); + } + + #[test] + fn g17_channel_descriptor_geometry_matches_the_firmware_walk() { + assert_eq!(G17_CHANNEL_DESCRIPTOR_TABLE_OFFSET, 0x20); + assert_eq!(G17_CHANNEL_DESCRIPTORS_PER_PRIORITY, 3); + assert_eq!(G17_CHANNEL_PRIORITY_COUNT, 4); + assert_eq!(G17_CHANNEL_DESCRIPTOR_COUNT, 12); + assert_eq!(G17_CHANNEL_DESCRIPTOR_STRIDE, 0x20); + assert_eq!(G17_CHANNEL_ENTRY_STRIDE, 0x18); + assert_eq!( + G17_CHANNEL_DESCRIPTOR_LAYOUT, + G17ChannelDescriptorLayout { + dispatch_cursor: 0x00, + coherency_cursor: 0x08, + producer_cursor: 0x10, + entries: 0x18, + } + ); + + for priority in 0..G17_CHANNEL_PRIORITY_COUNT { + for (pipe, pipe_index) in [ + (PipeType::Vertex, 0u32), + (PipeType::Fragment, 1), + (PipeType::Compute, 2), + ] { + let linear = priority as u32 * 3 + pipe_index; + assert_eq!( + g17_channel_descriptor_offset(priority, pipe), + Ok(0x20 + linear * 0x20) + ); + } + } + + // Descriptor eleven occupies 0x180..0x19f. Priority four must not + // alias the independent cursor pair beginning at +0x1a0. + assert_eq!( + g17_channel_descriptor_offset(3, PipeType::Compute), + Ok(0x180) + ); + assert_eq!( + g17_channel_descriptor_offset(4, PipeType::Vertex), + Err(SubmissionError::ChannelPriorityOutOfRange) + ); + } + + #[test] + fn g17_host_channel_state_reset_matches_all_three_pinned_hosts() { + let values = G17HostChannelStateResetValues { + gpu_va_00: 0x0011_2233_4455_6677, + gpu_va_08: 0x8899_aabb_ccdd_eeff, + gpu_va_10: 0x1020_3040_5060_7080, + word_44: 0x7654_3210, + gpu_va_70: 0xfedc_ba98_7654_3210, + }; + let raw = encode_g17_host_channel_state_reset(values); + let layout = G17_HOST_CHANNEL_STATE_RESET_LAYOUT; + + assert_eq!(raw.len(), 0x80); + assert_eq!( + &raw[layout.gpu_va_00..layout.gpu_va_00 + 8], + &values.gpu_va_00.to_le_bytes() + ); + assert_eq!( + &raw[layout.gpu_va_08..layout.gpu_va_08 + 8], + &values.gpu_va_08.to_le_bytes() + ); + assert_eq!( + &raw[layout.gpu_va_10..layout.gpu_va_10 + 8], + &values.gpu_va_10.to_le_bytes() + ); + assert_eq!(&raw[layout.zero_18..layout.zero_18 + 4], &[0; 4]); + assert_eq!(&raw[layout.zero_1c..layout.zero_1c + 4], &[0; 4]); + assert_eq!(&raw[layout.zero_20..layout.zero_20 + 4], &[0; 4]); + assert_eq!( + &raw[layout.sentinel_24..layout.sentinel_24 + 4], + &u32::MAX.to_le_bytes() + ); + assert_eq!( + &raw[layout.fixed_4_28..layout.fixed_4_28 + 4], + &4u32.to_le_bytes() + ); + assert_eq!( + &raw[layout.sentinel_40..layout.sentinel_40 + 4], + &u32::MAX.to_le_bytes() + ); + assert_eq!( + &raw[layout.word_44..layout.word_44 + 4], + &values.word_44.to_le_bytes() + ); + assert_eq!(&raw[layout.zero_58..layout.zero_58 + 4], &[0; 4]); + assert_eq!( + &raw[layout.gpu_va_70..layout.gpu_va_70 + 8], + &values.gpu_va_70.to_le_bytes() + ); + + // resetChannelState clears the entire record before its scalar stores. + // Every byte outside the nonzero runtime/fixed ranges stays zero. + for (index, byte) in raw.iter().enumerate() { + let nonzero_field = (0x00..0x18).contains(&index) + || (0x24..0x2c).contains(&index) + || (0x40..0x48).contains(&index) + || (0x70..0x78).contains(&index); + if !nonzero_field { + assert_eq!(*byte, 0, "unexpected reset byte at {index:#x}"); + } + } + assert!(G17_HOST_CHANNEL_RESET_TARGETS_CONFIRMED); + } + + #[test] + fn g17_uncached_channel_reset_preserves_unwritten_bytes() { + let mut raw = [0xa5; G17_UNCACHED_CHANNEL_RESET_WRITE_EXTENT]; + let ring_entry_count = 0x800; + apply_g17_uncached_channel_reset(&mut raw, ring_entry_count); + let layout = G17_UNCACHED_CHANNEL_RESET_LAYOUT; + + for offset in [ + layout.zero_00, + layout.zero_10, + layout.zero_20, + layout.zero_30, + layout.zero_40, + ] { + assert_eq!(&raw[offset..offset + 4], &[0; 4]); + } + assert_eq!( + &raw[layout.sentinel_50..layout.sentinel_50 + 4], + &u32::MAX.to_le_bytes() + ); + assert_eq!( + &raw[layout.ring_entry_count_60..layout.ring_entry_count_60 + 4], + &ring_entry_count.to_le_bytes() + ); + + for (index, byte) in raw.iter().enumerate() { + let written = [0x00usize, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60] + .iter() + .any(|offset| (*offset..*offset + 4).contains(&index)); + if !written { + assert_eq!(*byte, 0xa5, "unproven byte changed at {index:#x}"); + } + } + } + + /// Native tag-14 AddKicks, PROVEN on all 25 captured publishes: + /// `+0x00 = 0x0e`, `+0x04 u16 = qid`, `+0x06 u16 = 1` kick, `+0x08 u8 = 2` + /// queue priority, `+0x09 u64 = entry stamp`, `+0x11 u8 = data master`, + /// everything else zero. + #[test] + fn g17p_retained_event_carries_native_addkicks_bytes() { + for (stage, qid, data_master) in [ + (G17PColdOpeningStage::Tiling, 0u16, 0u8), + (G17PColdOpeningStage::Fragment, 1u16, 1u8), + ] { + let mut event = [0xa5u8; G17P_COLD_OPENING_EVENT_SIZE]; + apply_g17p_retained_event_with_stamp(stage, 0x3d9, qid, &mut event).unwrap(); + assert_eq!(get_u32_le(&event, 0x00), 0x0e); + assert_eq!(get_u16_le(&event, 0x04), qid); + assert_eq!(get_u16_le(&event, 0x06), 1); + assert_eq!(event[0x08], G17P_NATIVE_RENDER_DOORBELL_PRIORITY); + assert_eq!( + u32::from_le_bytes([event[0x09], event[0x0a], event[0x0b], 0]), + 0x3d9 + ); + assert_eq!(event[0x11], data_master); + for (index, byte) in event.iter().enumerate() { + if !(0x00..0x0c).contains(&index) && index != 0x11 { + assert_eq!(*byte, 0, "unproven byte set at {index:#x}"); + } + } + } + // The native 40-bit producer wraps to zero. Values above its width + // must be refused before any record is modified. + let mut event = [0u8; G17P_COLD_OPENING_EVENT_SIZE]; + assert!( + apply_g17p_retained_event_with_stamp(G17PColdOpeningStage::Tiling, 1u64 << 40, 0, &mut event) + .is_err() + ); + } + + /// The render pair configures TA on data master 0 and 3D on data master 1; + /// the compute data master must be refused on this path. + #[test] + fn g17p_render_queue_config_admits_only_render_data_masters() { + let geometry = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY; + let scratch = g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch; + for (queue_id, data_master, entry) in [ + ( + G17P_PARTIAL_OPENING_TILING_CHANNEL_TABLE_INDEX, + G17P_TA_DATA_MASTER, + 0x70_0043_8000u64, + ), + ( + G17P_PARTIAL_OPENING_FRAGMENT_CHANNEL_TABLE_INDEX, + G17P_3D_DATA_MASTER, + 0x70_0046_0000u64, + ), + ] { + let routing = G17PComputeQueueRouting { + queue_id, + completion_selector: 2, + entry_gpu_va: entry, + geometry, + }; + let config = + prepare_g17p_render_sksm_queue_config(routing, data_master, scratch).unwrap(); + assert_eq!(config.payload & G17P_SKSM_QUEUE_ENTRY_ADDRESS_MASK, entry); + assert_eq!(config.payload >> 60, data_master as u64 + 1); + assert_eq!(config.enable_pair.word1, 1u64 << queue_id); + assert_eq!(config.enable_pair.word0, 0); + assert!(prepare_g17p_render_sksm_queue_config( + routing, + G17P_COMPUTE_DATA_MASTER, + scratch + ) + .is_err()); + } + } + + #[test] + fn g17p_secondary_render_pair_matches_native_configure_words() { + let geometry = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY; + let scratch = g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch; + for (queue_id, data_master, entry, tag, payload, mask) in [ + ( + 2, + G17P_TA_DATA_MASTER, + 0x70_01f4_8000, + 0x0000_1400_0002_1008, + 0x1280_0070_01f4_8000, + 0x4, + ), + ( + 3, + G17P_3D_DATA_MASTER, + 0x70_01f7_0000, + 0x0000_1c00_0002_1008, + 0x2280_0070_01f7_0000, + 0x8, + ), + ] { + let config = prepare_g17p_render_sksm_queue_config( + G17PComputeQueueRouting { + queue_id, + completion_selector: 0, + entry_gpu_va: entry, + geometry, + }, + data_master, + scratch, + ) + .unwrap(); + assert_eq!(config.tag, tag); + assert_eq!(config.payload, payload); + assert_eq!(config.enable_pair.word0, 0); + assert_eq!(config.enable_pair.word1, mask); + } + } + + #[test] + fn g17p_render_config_update_carries_context_hardware_buffer_id_and_pid() { + let addresses = G17PColdOpeningOptionalAddresses { + context_scratch: 0x70_0200_0000, + firmware_scratch: 0x70_0200_4000, + usc_priv_mem_flist: 0x70_0200_8000, + usc_freelist_hardware_buffer_id: 5, + channel_control: 0x70_0200_c000, + tiling_shared_object: 0x70_0201_0000, + parameter_buffer_token: 0, + owner_pid: 0x625, + ta_hardware_buffer_id: 7, + }; + for context_id in [5, 62] { + for stage in [ + G17PColdOpeningStage::Tiling, + G17PColdOpeningStage::Fragment, + ] { + let mut optional = [0u8; G17P_COLD_OPENING_OPTIONAL_SIZE]; + apply_g17p_retained_optional( + stage, + addresses, + 0, + false, + 0, + 1, + 5, + 1, + context_id, + &mut optional, + ) + .unwrap(); + assert_eq!(get_u16_le(&optional, 0x32), context_id); + assert_eq!(get_u32_le(&optional, 0x46), 5); + assert_eq!(get_u16_le(&optional, 0x56), 5); + assert_eq!(get_u32_le(&optional, 0x5a), 0x625); + assert_eq!( + get_u32_le(&optional, 0x7e), + if stage == G17PColdOpeningStage::Tiling { + 7 + } else { + u32::MAX + } + ); + } + } + } + + #[test] + fn g17p_retained_canonical_records_advance_logically_without_moving() { + let addresses = G17PColdOpeningOptionalAddresses { + context_scratch: 0x70_0200_0000, + firmware_scratch: 0x70_0200_4000, + usc_priv_mem_flist: 0x70_0200_8000, + usc_freelist_hardware_buffer_id: 0, + channel_control: 0x70_0200_c000, + tiling_shared_object: 0x70_0201_0000, + parameter_buffer_token: 0, + owner_pid: 0x15, + ta_hardware_buffer_id: 0, + }; + for stage in [G17PColdOpeningStage::Tiling, G17PColdOpeningStage::Fragment] { + let mut optional = [0u8; G17P_COLD_OPENING_OPTIONAL_SIZE]; + apply_g17p_retained_optional( + stage, + addresses, + 1, + false, + 0, + 0, + 0, + 1, + 1, + &mut optional, + ) + .unwrap(); + assert_eq!(get_u16_le(&optional, 0x1a), 0); + assert_eq!(get_u16_le(&optional, 0x2a), 1); + assert_eq!(get_u16_le(&optional, 0x2e), 0x100); + assert_eq!(get_u16_le(&optional, 0x3e), 1); + assert_eq!(get_u16_le(&optional, 0x52), 0); + assert_eq!(get_u16_le(&optional, 0x62), 0); + assert_eq!( + get_u16_le(&optional, 0x76), + if stage == G17PColdOpeningStage::Tiling { + 0 + } else { + u16::MAX + } + ); + + let mut event = [0u8; G17P_COLD_OPENING_EVENT_SIZE]; + apply_g17p_retained_event(stage, 1, 0, &mut event).unwrap(); + assert_eq!(get_u32_le(&event, 0x08), 0x202); + } + + let pair = prepare_g17p_partial_opening_work_pair( + G17PPartialOpeningOuterChannelState { + consumers: [1, 1], + producer: 1, + queue_gpu_va: 0x70_0300_00c0, + queue_write_index: 3, + }, + G17PPartialOpeningOuterChannelState { + consumers: [1, 1], + producer: 1, + queue_gpu_va: 0x70_0300_0000, + queue_write_index: 3, + }, + 1, + 0, + false, + ) + .unwrap(); + assert_eq!(pair.fragment.slot_index, 1); + assert_eq!(pair.tiling.slot_index, 1); + assert_eq!(pair.fragment.next_producer, 2); + assert_eq!(pair.tiling.next_producer, 2); + assert_eq!(get_u32_le(&pair.fragment.slot, 0x14), 0x0001_0003); + assert_eq!(get_u32_le(&pair.tiling.slot, 0x14), 0x0000_0003); + } + + #[test] + fn g17p_context_update_is_fresh_graph_state_not_global_ordinal_state() { + let addresses = G17PColdOpeningOptionalAddresses { + context_scratch: 0x70_0200_0000, + firmware_scratch: 0x70_0200_4000, + usc_priv_mem_flist: 0x70_0200_8000, + usc_freelist_hardware_buffer_id: 0, + channel_control: 0x70_0200_c000, + tiling_shared_object: 0x70_0201_0000, + parameter_buffer_token: 0, + owner_pid: 0x15, + ta_hardware_buffer_id: 0, + }; + for stage in [G17PColdOpeningStage::Tiling, G17PColdOpeningStage::Fragment] { + for (ordinal, context_update) in [(0, true), (195, true), (196, false)] { + let mut optional = [0u8; G17P_COLD_OPENING_OPTIONAL_SIZE]; + apply_g17p_retained_optional_with_flist_generation_and_context_update( + stage, addresses, ordinal, 2022, false, 2, 3, 2, 0, 6, + context_update, &mut optional, + ) + .unwrap(); + assert_eq!(get_u16_le(&optional, 0x1a), 0); + assert_eq!(get_u16_le(&optional, 0x52), if ordinal == 0 { 1 } else { 0 }); + assert_eq!(get_u16_le(&optional, 0x62), u16::from(context_update)); + assert_eq!(get_u16_le(&optional, 0x32), 6); + } + } + } + + #[test] + fn ring_index_wraps_at_size() { + assert_eq!(ring_next(0, 4), Ok(1)); + assert_eq!(ring_next(3, 4), Ok(0)); + assert_eq!(ring_next(0x4ff, 0x500), Ok(0)); + assert_eq!(ring_next(0, 0), Err(SubmissionError::RingSizeZero)); + assert_eq!(ring_next(4, 4), Err(SubmissionError::RingIndexOutOfRange)); + } + + #[test] + fn kick_word_fields_match_the_firmware_extraction() { + // and x0,x5,#0xffffffffff ; ubfx x1,x5,#40,#7 ; ubfx x6,x5,#46,#1. + assert_eq!(KICK_TS_BITS, 40); + assert_eq!(KICK_TS_MASK, 0xff_ffff_ffff); + assert_eq!(KICK_QID_SHIFT, 40); + assert_eq!(KICK_QID_MASK, 0x7f); + assert_eq!(KICK_SELECTOR_BIT, 46); + + // Compose the exact word consumed from the firmware scratch FIFO; + // QID bit 6 is the mask selector. + let ts = 0x89_abcd_ef42u64; + let word = encode_kick_word(ts, 0x7f).unwrap(); + assert_eq!( + decode_kick_word(word), + Ok(SksmKickWord { + timestamp: ts, + queue_id: 0x7f, + mask_selector: 1, + }), + 1, + 0 + ); + // A word with bit 47 set is outside the modeled form. + assert_eq!( + decode_kick_word(1 << 47), + Err(SubmissionError::KickWordUnmodeledBits) + ); + assert_eq!( + encode_kick_word(KICK_TS_MASK + 1, 0), + Err(SubmissionError::KickTimestampOutOfRange) + ); + assert_eq!( + encode_kick_word(0, 0x80), + Err(SubmissionError::QueueIdOutOfRange) + ); + } + + #[test] + fn g17p_3d_completion_scratch_matches_native_full_command() { + assert_eq!( + encode_g17p_3d_completion_scratch( + 0xf76, + 1, + ), + Ok(0x0000_0100_0000_0f76), + ); + assert_eq!( + encode_g17p_3d_completion_scratch( + KICK_TS_MASK + 1, + 1, + ), + Err(SubmissionError::KickTimestampOutOfRange), + ); + } + + #[test] + fn g17p_serialized_render_scratch_payload_advances_and_wraps_independently() { + for (ordinal, timestamp, expected) in [ + (0, 1, 0x0000_0100_0000_0001), + (1, 2, 0x0000_0200_0000_0002), + (2, 3, 0x0000_0300_0000_0003), + (254, 255, 0x0000_ff00_0000_00ff), + (255, 256, 0x0000_0000_0000_0100), + (256, 257, 0x0000_0100_0000_0101), + // A channel payload is not inferred from the physical queue's + // slot or timestamp: distinct valid values must remain distinct. + (1, 0x1234_5678_9a, 0x0000_0212_3456_789a), + ] { + assert_eq!( + encode_g17p_3d_completion_scratch( + timestamp, g17p_serialized_render_scratch_payload(ordinal), + ), + Ok(expected), + ); + } + assert_eq!(g17p_serialized_render_scratch_payload(u64::from(u32::MAX)), 0); + } + + #[test] + fn kick_queue_preparation_couples_slot_queue_and_next_timestamp() { + let timestamp = encode_kick_timestamp(KickTimestamp { + stamp: 0x1234_5678, + stamp_index: 7, + }); + let prepared = prepare_kick_submission(timestamp, 0x2a, 16, 0x180).unwrap(); + + assert_eq!(prepared.entry_index, 7); + assert_eq!(prepared.entry_offset, 7 * 0x180); + assert_eq!(prepared.timestamp, 0x12_3456_7807); + assert_eq!(prepared.queue_id, 0x2a); + assert_eq!( + decode_kick_timestamp(prepared.next_timestamp), + Ok(KickTimestamp { + stamp: 0x1234_5678, + stamp_index: 8, + }) + ); + } + + #[test] + fn kick_queue_timestamp_wrap_matches_the_host_divide_remainder_sequence() { + // Slot 15 wraps modulo 16, so the host adds 0x100 to the timestamp: + // the low byte becomes zero and the 32-bit stamp increments. + let timestamp = encode_kick_timestamp(KickTimestamp { + stamp: 0x1234_5678, + stamp_index: 15, + }); + let prepared = prepare_kick_submission(timestamp, 0x7f, 16, 0x180).unwrap(); + assert_eq!(prepared.entry_index, 15); + assert_eq!(prepared.entry_offset, 15 * 0x180); + assert_eq!( + decode_kick_timestamp(prepared.next_timestamp), + Ok(KickTimestamp { + stamp: 0x1234_5679, + stamp_index: 0, + }) + ); + + // A 256-slot queue is representable despite num_stamps being larger + // than u8; slot 255 wraps and the 32-bit stamp wraps exactly like the + // host's masked add. + let timestamp = encode_kick_timestamp(KickTimestamp { + stamp: u32::MAX, + stamp_index: 0xff, + }); + let prepared = prepare_kick_submission(timestamp, 0x40, 256, 0x180).unwrap(); + assert_eq!(prepared.entry_index, 0xff); + assert_eq!( + decode_kick_timestamp(prepared.next_timestamp), + Ok(KickTimestamp { + stamp: 0, + stamp_index: 0, + }) + ); + } + + #[test] + fn g17p_hal200_queue_geometry_is_exact_and_other_backends_stay_unconfirmed() { + let geometry = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY; + assert_eq!(geometry.validate(), Ok(())); + assert_eq!(geometry.allocation_alignment(), 0x100); + assert_eq!(geometry.entry_stride(), 0x200); + assert_eq!(geometry.stamp_count(), 0x100); + assert_eq!(geometry.timestamp_seed(), 1); + assert_eq!(geometry.backing_size(), 0x20000); + assert_eq!( + geometry.entry_stride() as usize * geometry.stamp_count() as usize, + geometry.backing_size() + ); + assert_eq!( + g17_sksm_queue_geometry( + G17ChannelAbiTarget::A18ProG17P, + G17ClSubmissionBackend::G17PHal200Hybrid, + ), + Some(geometry) + ); + for backend in [ + G17ClSubmissionBackend::Classic, + G17ClSubmissionBackend::Hal300, + ] { + assert_eq!( + g17_sksm_queue_geometry(G17ChannelAbiTarget::A18ProG17P, backend), + None + ); + } + for target in [ + G17ChannelAbiTarget::M5G17G, + G17ChannelAbiTarget::M5ProMaxG17X, + ] { + assert_eq!( + g17_sksm_queue_geometry(target, G17ClSubmissionBackend::G17PHal200Hybrid), + None + ); + } + + let producer = geometry + .initial_producer_state( + 4, + 2, + G17PClQosConfig { + queue_byte_32: 1, + class_40: 8, + word_48: 0xffff, + }, + ) + .unwrap(); + assert_eq!(producer.geometry, geometry); + assert_eq!(producer.submitted, 0); + assert_eq!(producer.current_timestamp, geometry.timestamp_seed()); + assert_eq!(producer.last_add_kicks_timestamp, geometry.timestamp_seed()); + assert_eq!(producer.base_timestamp, geometry.timestamp_seed()); + assert!(!producer.previous_valid); + } + + #[test] + fn g17p_hal200_queue_geometry_drives_last_slot_and_wrap() { + let geometry = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY; + let timestamp = encode_kick_timestamp(KickTimestamp { + stamp: 0x1234_5678, + stamp_index: (geometry.stamp_count() - 1) as u8, + }); + let prepared = prepare_kick_submission( + timestamp, + 4, + geometry.stamp_count(), + geometry.entry_stride(), + ) + .unwrap(); + + assert_eq!(prepared.entry_index, 0xff); + assert_eq!( + prepared.entry_offset as usize + geometry.entry_stride() as usize, + geometry.backing_size() + ); + assert_eq!( + decode_kick_timestamp(prepared.next_timestamp), + Ok(KickTimestamp { + stamp: 0x1234_5679, + stamp_index: 0, + }) + ); + } + + #[test] + fn kick_queue_preparation_matches_all_runtime_stamp_geometries() { + for num_stamps in 1u16..=KICK_NUM_STAMPS_MAX { + for index in 0u16..num_stamps { + let current = KickTimestamp { + stamp: 0x89ab_cdef, + stamp_index: index as u8, + }; + let timestamp = encode_kick_timestamp(current); + let prepared = prepare_kick_submission(timestamp, 3, num_stamps, 0x180).unwrap(); + let expected_index = (index + 1) % num_stamps; + let expected_stamp = if expected_index == 0 { + current.stamp.wrapping_add(1) + } else { + current.stamp + }; + + assert_eq!(prepared.entry_index, index as u8); + assert_eq!(prepared.entry_offset, index as u32 * 0x180); + assert_eq!(prepared.timestamp, timestamp); + assert_eq!( + decode_kick_timestamp(prepared.next_timestamp), + Ok(KickTimestamp { + stamp: expected_stamp, + stamp_index: expected_index as u8, + }) + ); + } + } + } + + #[test] + fn kick_queue_preparation_reports_unrepresentable_state_before_publication() { + assert_eq!( + prepare_kick_submission(1 << 40, 0, 1, 0x180), + Err(SubmissionError::KickTimestampOutOfRange) + ); + assert_eq!( + prepare_kick_submission(0, 0, 0, 0x180), + Err(SubmissionError::KickStampCountOutOfRange) + ); + assert_eq!( + prepare_kick_submission(0, 0, 257, 0x180), + Err(SubmissionError::KickStampCountOutOfRange) + ); + assert_eq!( + prepare_kick_submission(4, 0, 4, 0x180), + Err(SubmissionError::KickStampIndexOutOfRange) + ); + assert_eq!( + prepare_kick_submission(0, 0, 1, 0), + Err(SubmissionError::KickEntryStrideZero) + ); + assert_eq!( + prepare_kick_submission(2, 0, 3, u32::MAX), + Err(SubmissionError::KickEntryOffsetOverflow) + ); + assert_eq!( + prepare_kick_submission(0, 0x80, 1, 0x180), + Err(SubmissionError::QueueIdOutOfRange) + ); + } + + #[test] + fn g17p_cl_kick_layout_and_publication_order_are_exact() { + assert_eq!(G17P_CL_KICK_ENTRY_SIZE, 0x180); + assert_eq!(G17P_CL_DEPENDENCIES_MAX, 32); + assert_eq!(G17P_CL_EXPLICIT_DEPENDENCIES_MAX, 31); + assert_eq!( + G17P_CL_KICK_QUEUE_STATE_LAYOUT, + G17PClKickQueueStateLayout { + queue_id: 0x0c, + num_stamps: 0x10, + entry_stride: 0x14, + completion_selector: 0x1c, + submitted: 0x44, + current_timestamp: 0x48, + last_add_kicks_timestamp: 0x50, + entry_cpu_base: 0x78, + entry_gpu_base: 0x80, + base_timestamp: 0x90, + previous_valid: 0x98, + previous_timestamp: 0xa0, + } + ); + assert_eq!( + G17P_CL_KICK_ENTRY_LAYOUT, + G17PClKickEntryLayout { + header: 0x00, + mcache: 0x08, + payload: [0x10, 0x18], + dependency_control: 0x20, + dependencies: 0x28, + event_mask: [0x128, 0x130, 0x138, 0x140], + rce_kind: 0x148, + rce_bindings: [0x150, 0x158, 0x160, 0x168], + zero_170: 0x170, + auxiliary: 0x178, + } + ); + assert_eq!( + G17P_CL_KICK_PUBLICATION_ORDER, + [ + G17PClKickPublicationStep::ZeroEntryStride, + G17PClKickPublicationStep::EntryBody, + G17PClKickPublicationStep::InnerShareableBarrier, + G17PClKickPublicationStep::NextTimestamp, + G17PClKickPublicationStep::SubmittedAtomicAdd, + G17PClKickPublicationStep::PreviousValid, + G17PClKickPublicationStep::PreviousTimestamp, + ] + ); + assert_eq!( + G17P_FIRST_SUBMIT_PUBLICATION_ORDER, + [ + G17PFirstSubmitPublicationStep::AcquireGfx1RunningPower, + G17PFirstSubmitPublicationStep::PublishB2, + G17PFirstSubmitPublicationStep::PublishConfigUpdatePointer, + G17PFirstSubmitPublicationStep::ConfigureQid4Hardware, + G17PFirstSubmitPublicationStep::ZeroSksmStride, + G17PFirstSubmitPublicationStep::WriteSksmEntry, + G17PFirstSubmitPublicationStep::EntryBarrier, + G17PFirstSubmitPublicationStep::PublishSksmProducer, + G17PFirstSubmitPublicationStep::AdvanceDescriptorGeneration, + G17PFirstSubmitPublicationStep::RecordHostCorrelationToken, + G17PFirstSubmitPublicationStep::PublishEntrySignalPointer, + G17PFirstSubmitPublicationStep::AddKicks, + G17PFirstSubmitPublicationStep::PublishCl2, + G17PFirstSubmitPublicationStep::SendActivation, + G17PFirstSubmitPublicationStep::DirectKickBarrier, + G17PFirstSubmitPublicationStep::SendDirectKick, + ] + ); + } + + #[test] + fn g17p_conditional_transport_events_match_the_independent_fixtures() { + const PRIMARY: &[u8; G17P_COMPUTE_PRIMARY_EVENT_SIZE] = + include_bytes!("fixtures/g17p_first_submit/channel-event-primary.bin"); + const OPTIONAL: &[u8; G17P_COMPUTE_OPTIONAL_EVENT_SIZE] = + include_bytes!("fixtures/g17p_first_submit/channel-event-optional.bin"); + + let primary = prepare_g17p_compute_primary_event(0, 0, 4, 2, 1).unwrap(); + assert_eq!(primary.descriptor_generation, 1); + assert_eq!(primary.bytes, *PRIMARY); + assert_eq!(advance_g17p_compute_descriptor_generation(0, 0), Ok(1)); + assert_eq!(advance_g17p_compute_descriptor_generation(2, 3), Ok(6)); + + let mut optional = [0u8; G17P_COMPUTE_OPTIONAL_EVENT_SIZE]; + encode_g17p_compute_optional_event( + G17PComputeOptionalEvent { + queue_id: 4, + old_timestamp: 1, + }, + &mut optional, + ) + .unwrap(); + assert_eq!(optional, *OPTIONAL); + + assert_eq!( + prepare_g17p_compute_primary_event(u32::MAX, 0, 4, 2, 1), + Err(SubmissionError::ComputeEventCounterOverflow) + ); + assert_eq!( + prepare_g17p_compute_primary_event(u16::MAX as u32, 0, 4, 2, 1), + Err(SubmissionError::ComputeEventCounterOverflow) + ); + + // The tag-14 transport builder reproduces the same captured record. + // mac17p 0xfffffe00089fa39c writes type 14, QID 4, one kick, the queue + // QoS priority 2, the pre-increment entry stamp 1, and data master 2. + let add_kicks = prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: 4, + add_count: 1, + queue_priority: 2, + entry_stamp: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.timestamp_seed(), + }) + .unwrap(); + assert_eq!(add_kicks.bytes, *PRIMARY); + assert_eq!(add_kicks.bytes.len(), G17P_KSM_ADD_KICKS_COMMAND_SIZE); + assert!(G17P_KSM_ADD_KICKS_COMMAND_SIZE <= G17P_KSM_ADD_KICKS_COMMAND_FETCH_SIZE); + assert_eq!(add_kicks.bytes[0x08], 2); + assert_eq!( + u32::from_le_bytes(add_kicks.bytes[0x11..0x15].try_into().unwrap()), + G17P_COMPUTE_DATA_MASTER + ); + + assert_eq!( + prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: 4, + add_count: 0, + queue_priority: 2, + entry_stamp: 1, + }), + Err(SubmissionError::KsmAddKicksCountZero) + ); + assert_eq!( + prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: KICK_QID_MASK + 1, + add_count: 1, + queue_priority: 2, + entry_stamp: 1, + }), + Err(SubmissionError::QueueIdOutOfRange) + ); + assert_eq!( + prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: 4, + add_count: 1, + queue_priority: 4, + entry_stamp: 1, + }), + Err(SubmissionError::KickQueueCompletionSelectorOutOfRange) + ); + assert_eq!( + prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: 4, + add_count: 1, + queue_priority: 2, + entry_stamp: 1 << KICK_TS_BITS, + }), + Err(SubmissionError::KickTimestampOutOfRange) + ); + assert_eq!( + prepare_g17p_ksm_add_kicks_command(G17PKsmAddKicksCommand { + queue_id: 4, + add_count: u16::MAX as u32 + 1, + queue_priority: 2, + entry_stamp: 1, + }), + Err(SubmissionError::KsmAddKicksCountOutOfRange) + ); + } + + #[test] + fn g17p_cl_hardware_positive_retained_items_are_exact() { + const QUEUE: u64 = 0xffff_fc20_c000_0300; + const DESCRIPTOR: u64 = 0xffff_fc20_c035_8000; + const DESCRIPTOR_STRIDE: u64 = 0x1040; + let item0_queue = G17PClKickProducerState { + queue_id: 4, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 2, + qos: G17PClQosConfig { + queue_byte_32: 1, + class_40: 8, + word_48: 0xffff, + }, + submitted: 0, + current_timestamp: 1, + last_add_kicks_timestamp: 1, + base_timestamp: 1, + previous_valid: false, + previous_timestamp: 0, + }; + let item0 = prepare_g17p_cl_kick_entry( + item0_queue, + native_add3_compute_operands( + DESCRIPTOR, + QUEUE, + 1, + [0, 0, 4, 0], + [0x0001_1003_8001_a002, 0x0000_2003_8001_a03b], + ), + ) + .unwrap(); + let mut expected0 = [0u8; G17P_CL_KICK_ENTRY_SIZE]; + for (offset, value) in [ + (0x00, 0x1000_1000_0000_0004), + (0x10, DESCRIPTOR), + (0x18, QUEUE), + (0x20, 0xffff_0801_0000_0001), + (0x28, 0x0000_0400_0000_0000), + (0x138, 4), + (0x150, 0x0001_1003_8001_a002), + (0x158, 0x0000_2003_8001_a03b), + (0x178, 0x003f_ffff_ffff_ffff), + ] { + put_u64_le(&mut expected0, offset, value); + } + assert_eq!(item0.entry, expected0); + assert_eq!(item0.zero_length, 0x200); + assert_eq!(item0.entry_index, 1); + assert_eq!(item0.entry_offset, 0x200); + assert_eq!(item0.current_timestamp, 1); + assert_eq!(item0.next_timestamp, 2); + assert_eq!(item0.submitted_before, 0); + assert_eq!(item0.submitted_after, 1); + assert_eq!(item0.previous_timestamp_after, 1); + assert_eq!(item0.producer_after.current_timestamp, 2); + assert_eq!(item0.producer_after.submitted, 1); + assert!(item0.producer_after.previous_valid); + assert_eq!(item0.producer_after.previous_timestamp, 1); + assert_eq!(item0.producer_after.qos, item0_queue.qos); + + let item1 = prepare_g17p_cl_kick_entry( + item0.producer_after, + native_add3_compute_operands( + DESCRIPTOR + DESCRIPTOR_STRIDE, + QUEUE, + 0, + [0, 2, 8, 0], + [0x0001_1003_8001_a084, 0x0000_2003_8001_a0bd], + ), + ) + .unwrap(); + let mut expected1 = [0u8; G17P_CL_KICK_ENTRY_SIZE]; + for (offset, value) in [ + (0x00, 0x1000_1000_0000_0008), + (0x10, DESCRIPTOR + DESCRIPTOR_STRIDE), + (0x18, QUEUE), + (0x20, 0xffff_0801_0000_0001), + (0x28, 0x0000_0400_0000_0001), + (0x130, 2), + (0x138, 8), + (0x150, 0x0001_1003_8001_a084), + (0x158, 0x0000_2003_8001_a0bd), + (0x178, 0x003f_ffff_ffff_ffff), + ] { + put_u64_le(&mut expected1, offset, value); + } + assert_eq!(item1.entry, expected1); + assert_eq!(item1.zero_length, 0x200); + assert_eq!(item1.entry_index, 2); + assert_eq!(item1.entry_offset, 0x400); + assert_eq!(item1.current_timestamp, 2); + assert_eq!(item1.next_timestamp, 3); + assert_eq!(item1.submitted_before, 1); + assert_eq!(item1.submitted_after, 2); + assert_eq!(item1.previous_timestamp_after, 2); + assert_eq!(item1.producer_after.current_timestamp, 3); + assert_eq!(item1.producer_after.submitted, 2); + assert_eq!(item1.producer_after.previous_timestamp, 2); + assert_eq!(item1.producer_after.qos, item0_queue.qos); + + // Command two keeps physical queue 0's QoS word while advancing its + // command-owned event mask from 4 to 8. + assert_eq!(get_u64_le(&item1.entry, 0x00), 0x1000_1000_0000_0008); + assert_eq!(get_u64_le(&item1.entry, 0x28), 0x0000_0400_0000_0001); + assert_eq!( + get_u64_le(&item1.entry, 0x20), + get_u64_le(&item0.entry, 0x20) + ); + assert_eq!(get_u64_le(&item0.entry, 0x138), 4); + assert_eq!(get_u64_le(&item1.entry, 0x138), 8); + assert_eq!( + get_u64_le(&item1.entry, 0x178), + get_u64_le(&item0.entry, 0x178) + ); + } + + #[test] + fn g17p_first_submit_matches_the_independent_full_stride_fixture() { + const BEFORE: &[u8; 0x200] = + include_bytes!("fixtures/g17p_first_submit/sksm-entry-before.bin"); + const AFTER: &[u8; 0x200] = + include_bytes!("fixtures/g17p_first_submit/sksm-entry-after.bin"); + const RELOCATION_MASK: &[u8; 0x200] = + include_bytes!("fixtures/g17p_first_submit/sksm-entry-relocation-mask.bin"); + const DESCRIPTOR_HIGH: u64 = 0xffff_fc20_c035_8000; + const DESCRIPTOR_LOW: u64 = 0x70_0034_0000; + const QUEUE: u64 = 0xffff_fc20_c000_0300; + + assert!(BEFORE.iter().all(|byte| *byte == 0)); + assert_eq!(G17P_FIRST_BIND_COMMAND_PLAN.fallback_logical_flush_id, 0); + assert!(!G17P_FIRST_BIND_COMMAND_PLAN.flush_id_serialized_in_tag15); + assert!(!G17P_FIRST_BIND_COMMAND_PLAN.flush_id_completion_release_required); + assert_eq!( + RELOCATION_MASK + .iter() + .map(|byte| byte.count_ones()) + .sum::(), + 204 + ); + + let command = prepare_g17p_compute_cl_command_state(DESCRIPTOR_LOW, 0, 0x26).unwrap(); + let empty = G17PClRceBinding { address: 0, tag: 0 }; + assert_eq!( + command.rce_bindings, + [ + G17PClRceBinding { + address: DESCRIPTOR_LOW + 0x40, + tag: 0x26, + }, + G17PClRceBinding { + address: DESCRIPTOR_LOW + 0x760, + tag: 0x04, + }, + empty, + empty, + ] + ); + let before = G17PClKickProducerState { + queue_id: 4, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 2, + qos: G17PClQosConfig { + queue_byte_32: 1, + class_40: 8, + word_48: 0xffff, + }, + submitted: 0, + current_timestamp: 1, + last_add_kicks_timestamp: 1, + base_timestamp: 1, + previous_valid: false, + previous_timestamp: 0, + }; + let prepared = prepare_g17p_cl_kick_entry( + before, + G17PClKickEntryOperands { + descriptor_flag_4c: false, + descriptor_flag_5c8: false, + converted_command_timestamp: 1, + barriers: &[], + mcache: None, + payload: [DESCRIPTOR_HIGH, QUEUE], + event_mask: command.event_mask, + rce_kind: 0, + rce_bindings: command.rce_bindings, + auxiliary: 0x003f_ffff_ffff_ffff, + }, + ) + .unwrap(); + let mut actual = [0u8; 0x200]; + actual[..G17P_CL_KICK_ENTRY_SIZE].copy_from_slice(&prepared.entry); + + for index in 0..actual.len() { + assert_eq!( + actual[index] & !RELOCATION_MASK[index], + AFTER[index] & !RELOCATION_MASK[index], + "normalized fixture mismatch at offset {:#x}", + index + ); + } + // The fixture uses the same captured addresses as this test, so the + // relocation-bearing fields must also agree without normalization. + assert_eq!(actual, *AFTER); + assert_eq!(prepared.zero_length, 0x200); + assert_eq!(prepared.entry_index, 1); + assert_eq!(prepared.entry_offset, 0x200); + assert_eq!(prepared.current_timestamp, 1); + assert_eq!(prepared.next_timestamp, 2); + assert_eq!(prepared.submitted_before, 0); + assert_eq!(prepared.submitted_after, 1); + assert_eq!( + prepared.producer_after, + G17PClKickProducerState { + submitted: 1, + current_timestamp: 2, + previous_valid: true, + previous_timestamp: 1, + ..before + } + ); + + const SECOND_AFTER: &[u8; 0x200] = + include_bytes!("fixtures/g17p_first_submit/sksm-entry-second-after.bin"); + let second_command = + prepare_g17p_compute_cl_command_state(DESCRIPTOR_LOW + 0x1040, 1, 0x26).unwrap(); + let second = prepare_g17p_cl_kick_entry( + prepared.producer_after, + G17PClKickEntryOperands { + descriptor_flag_4c: false, + descriptor_flag_5c8: false, + converted_command_timestamp: 0, + barriers: &[], + mcache: None, + payload: [DESCRIPTOR_HIGH + 0x1040, QUEUE], + event_mask: second_command.event_mask, + rce_kind: 0, + rce_bindings: second_command.rce_bindings, + auxiliary: 0x003f_ffff_ffff_ffff, + }, + ) + .unwrap(); + let mut second_actual = [0u8; 0x200]; + second_actual[..G17P_CL_KICK_ENTRY_SIZE].copy_from_slice(&second.entry); + assert_eq!(second_actual, *SECOND_AFTER); + assert_eq!(second.entry_index, 2); + assert_eq!(second.entry_offset, 0x400); + assert_eq!(second.current_timestamp, 2); + assert_eq!(second.next_timestamp, 3); + assert_eq!(second.submitted_before, 1); + assert_eq!(second.submitted_after, 2); + assert_eq!(second.producer_after.current_timestamp, 3); + assert_eq!(second.producer_after.previous_timestamp, 2); + assert_eq!(second.producer_after.last_add_kicks_timestamp, 1); + } + + #[test] + fn g17p_cl_descriptor_flags_map_directly_to_header_bits() { + let empty = G17PClRceBinding { address: 0, tag: 0 }; + let producer = G17PClKickProducerState { + queue_id: 4, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 2, + qos: G17PClQosConfig { + queue_byte_32: 1, + class_40: 8, + word_48: 0xffff, + }, + submitted: 0, + current_timestamp: 1, + last_add_kicks_timestamp: 1, + base_timestamp: 1, + previous_valid: false, + previous_timestamp: 0, + }; + let flag_mask = (1u64 << 1) | (1u64 << 61); + + for (flag_4c, flag_5c8, expected) in [ + (false, false, 0), + (true, false, 1u64 << 1), + (false, true, 1u64 << 61), + (true, true, flag_mask), + ] { + let prepared = prepare_g17p_cl_kick_entry( + producer, + G17PClKickEntryOperands { + descriptor_flag_4c: flag_4c, + descriptor_flag_5c8: flag_5c8, + converted_command_timestamp: 1, + barriers: &[], + mcache: None, + payload: [0; 2], + event_mask: [0; 4], + rce_kind: 0, + rce_bindings: [empty; 4], + auxiliary: 0, + }, + ) + .unwrap(); + + assert_eq!(get_u64_le(&prepared.entry, 0) & flag_mask, expected); + } + } + + #[test] + fn g17p_cl_completion_updates_only_the_submitted_count() { + let producer = G17PClKickProducerState { + queue_id: 4, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 2, + qos: G17PClQosConfig { + queue_byte_32: 1, + class_40: 8, + word_48: 0xffff, + }, + submitted: 2, + current_timestamp: 3, + last_add_kicks_timestamp: 2, + base_timestamp: 1, + previous_valid: true, + previous_timestamp: 2, + }; + + assert_eq!(complete_g17p_cl_kicks(producer, 0), Ok(producer)); + let after_one = complete_g17p_cl_kicks(producer, 1).unwrap(); + assert_eq!( + after_one, + G17PClKickProducerState { + submitted: 1, + ..producer + } + ); + let empty = complete_g17p_cl_kicks(producer, 2).unwrap(); + assert_eq!(empty.submitted, 0); + assert_eq!(complete_g17p_cl_kicks(empty, 0), Ok(empty)); + assert_eq!( + complete_g17p_cl_kicks(empty, 1), + Err(SubmissionError::KickCompletionUnderflow) + ); + assert_eq!( + complete_g17p_cl_kicks(producer, 3), + Err(SubmissionError::KickCompletionUnderflow) + ); + } + + #[test] + fn g17p_cl_retained_storage_overlap_and_order_are_exact() { + let layout = G17P_CL_RETAINED_STORAGE_LAYOUT; + assert_eq!( + layout, + G17PClRetainedStorageLayout { + shared_support: G17PClStorageRange { + offset: 0, + size: 0x4000, + }, + scheduler: G17PClStorageRange { + offset: 0x200, + size: 0x100, + }, + } + ); + assert_eq!( + layout.scheduler.offset - layout.shared_support.offset, + 0x200 + ); + assert!(g17p_cl_range_contains( + layout.shared_support, + layout.scheduler + )); + assert!(g17p_cl_ranges_overlap( + layout.shared_support, + layout.scheduler + )); + assert!(g17p_cl_full_support_initialization_allowed(false)); + assert!(!g17p_cl_full_support_initialization_allowed(true)); + + assert_eq!( + G17P_CL_RETAINED_REGISTRATION_ORDER, + [ + G17PClRetainedPublicationStep::InitializeSharedSupport, + G17PClRetainedPublicationStep::WriteSchedulerRecord, + G17PClRetainedPublicationStep::RegisterPhysicalQueue, + ] + ); + assert_eq!( + G17P_CL_RETAINED_ACTIVATION_ORDER, + [ + G17PClRetainedPublicationStep::PublishCommandOwnedState, + G17PClRetainedPublicationStep::CleanCommandOwnedState, + G17PClRetainedPublicationStep::WriteB2PrimaryState, + G17PClRetainedPublicationStep::CleanB2PrimaryState, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerPayload, + G17PClRetainedPublicationStep::PublishInnerQueuePayload, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerProducer, + G17PClRetainedPublicationStep::PublishInnerQueueProducer, + G17PClRetainedPublicationStep::SystemBarrierAfterInnerProducer, + ] + ); + assert_eq!( + G17P_CL_RETAINED_SUBMISSION_ORDER, + [ + G17PClRetainedPublicationStep::PublishCommandOwnedState, + G17PClRetainedPublicationStep::CleanCommandOwnedState, + G17PClRetainedPublicationStep::PublishInnerQueuePayload, + G17PClRetainedPublicationStep::SystemBarrierBeforeInnerProducer, + G17PClRetainedPublicationStep::PublishInnerQueueProducer, + G17PClRetainedPublicationStep::SystemBarrierAfterInnerProducer, + ] + ); + assert_eq!( + g17p_cl_retained_submission_order(G17PClB2ActivationState::Pending), + G17P_CL_RETAINED_ACTIVATION_ORDER + ); + assert_eq!( + g17p_cl_retained_submission_order(G17PClB2ActivationState::Published), + G17P_CL_RETAINED_SUBMISSION_ORDER + ); + } + + #[test] + fn g17p_cl_b2_primary_state_builder_is_byte_exact() { + const FILL: u8 = 0xa5; + const CHANNEL_CONTROL: u64 = 0xffff_fc20_1234_8000; + let layout = G17P_CL_B2_PRIMARY_STATE_LAYOUT; + assert_eq!( + layout, + G17PClB2PrimaryStateLayout { + object_size: 0x4000, + computed_write_extent: 0xe30, + region_1_write_extent: 0x20, + region_2_write_extent: 0x18, + } + ); + + let mut computed = [FILL; 0x4000]; + let mut region_1 = [FILL; 0x4000]; + let mut region_2 = [FILL; 0x4000]; + let activation = apply_g17p_cl_b2_primary_state( + G17PClB2ActivationState::Pending, + &mut computed, + &mut region_1, + &mut region_2, + CHANNEL_CONTROL, + ) + .unwrap(); + assert_eq!(activation, G17PClB2ActivationState::Published); + + let mut expected_computed = [FILL; 0x4000]; + expected_computed[0x000] = 0xff; + expected_computed[0x008] = 0xff; + expected_computed[0x010..0x012].copy_from_slice(&0x0f01u16.to_le_bytes()); + expected_computed[0x018..0x01a].copy_from_slice(&0x0f01u16.to_le_bytes()); + for (offset, value) in [(0x408, 2), (0x40c, 2), (0x604, 4), (0xc04, 0x1000)] { + put_u32_le(&mut expected_computed, offset, value); + } + for (offset, value) in [ + (0x808, CHANNEL_CONTROL), + (0xe00, 7), + (0xe08, 0x0000_0001_0e22_618a), + (0xe20, 4), + (0xe28, 4), + ] { + put_u64_le(&mut expected_computed, offset, value); + } + + let mut expected_region_1 = [FILL; 0x4000]; + put_u64_le(&mut expected_region_1, 0x10, 0x0000_0080_0007_7000); + put_u32_le(&mut expected_region_1, 0x18, 0x17); + put_u32_le(&mut expected_region_1, 0x1c, 0x17); + + let mut expected_region_2 = [FILL; 0x4000]; + put_u64_le(&mut expected_region_2, 0x00, 0x0800_0000_e000_0000); + put_u64_le(&mut expected_region_2, 0x08, 0x0000_3d40_0000_3400); + put_u64_le(&mut expected_region_2, 0x10, 0x0000_0000_0000_1d00); + + assert_eq!(computed, expected_computed); + assert_eq!(region_1, expected_region_1); + assert_eq!(region_2, expected_region_2); + assert_eq!( + apply_g17p_cl_b2_primary_state( + activation, + &mut computed, + &mut region_1, + &mut region_2, + CHANNEL_CONTROL, + ), + Err(SubmissionError::KickB2PrimaryStateAlreadyPublished) + ); + assert_eq!(computed, expected_computed); + assert_eq!(region_1, expected_region_1); + assert_eq!(region_2, expected_region_2); + + let mut short = [0u8; 1]; + assert_eq!( + apply_g17p_cl_b2_primary_state( + G17PClB2ActivationState::Pending, + &mut short, + &mut region_1, + &mut region_2, + CHANNEL_CONTROL, + ), + Err(SubmissionError::KickB2PrimaryStateBufferTooSmall) + ); + } + + #[test] + fn g17p_cl_entry_encodes_every_optional_section() { + let barriers = [G17PClBarrierDependency { + native_timestamp: None, + event_word: 0x100, + shared_stamp: 0, + queue_id: 3, + }]; + let rce_bindings = [ + G17PClRceBinding { + address: 0x2000, + tag: 1, + }, + G17PClRceBinding { + address: 0x4000, + tag: 2, + }, + G17PClRceBinding { + address: 0x6000, + tag: 3, + }, + G17PClRceBinding { + address: 0x8000, + tag: 4, + }, + ]; + let queue = G17PClKickProducerState { + queue_id: 4, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 2, + qos: G17PClQosConfig { + queue_byte_32: 0x12, + class_40: 0x1b, + word_48: 0x3456, + }, + submitted: 2, + current_timestamp: 2, + last_add_kicks_timestamp: 1, + base_timestamp: 1, + previous_valid: true, + previous_timestamp: 7, + }; + let prepared = prepare_g17p_cl_kick_entry( + queue, + G17PClKickEntryOperands { + descriptor_flag_4c: true, + descriptor_flag_5c8: true, + converted_command_timestamp: 0, + barriers: &barriers, + mcache: Some(G17PClMcacheAperture { + address: 0x12340, + index: 5, + count: 3, + }), + payload: [0x1111_2222_3333_4444, 0x5555_6666_7777_8888], + event_mask: [0x10, 0x20, 0x30, 0x40], + rce_kind: 2, + rce_bindings, + auxiliary: 0xa5a5_5a5a_1234_5678, + }, + ) + .unwrap(); + let header = + (2u64 << 2) | (4u64 << 42) | (1u64 << 58) | (2u64 << 59) | (1u64 << 1) | (1u64 << 61); + assert_eq!(get_u64_le(&prepared.entry, 0x00), header); + assert_eq!( + get_u64_le(&prepared.entry, 0x08), + 5 | ((0x12340u64 >> 5) << 6) | (2u64 << 54) + ); + assert_eq!(get_u64_le(&prepared.entry, 0x10), 0x1111_2222_3333_4444); + assert_eq!(get_u64_le(&prepared.entry, 0x18), 0x5555_6666_7777_8888); + assert_eq!(get_u64_le(&prepared.entry, 0x20), 0x3456_1b12_0000_0003); + assert_eq!(get_u64_le(&prepared.entry, 0x28), (3u64 << 40) | 1); + assert_eq!(get_u64_le(&prepared.entry, 0x30), (4u64 << 40) | 7); + assert_eq!( + [ + get_u64_le(&prepared.entry, 0x128), + get_u64_le(&prepared.entry, 0x130), + get_u64_le(&prepared.entry, 0x138), + get_u64_le(&prepared.entry, 0x140), + ], + [0x10, 0x20, 0x30, 0x40] + ); + assert_eq!(prepared.entry[0x148], 2); + for (index, binding) in rce_bindings.into_iter().enumerate() { + assert_eq!( + get_u64_le(&prepared.entry, 0x150 + index * 8), + (binding.address >> 5) | ((binding.tag as u64) << 43) + ); + } + assert_eq!(get_u64_le(&prepared.entry, 0x170), 0); + assert_eq!(get_u64_le(&prepared.entry, 0x178), 0xa5a5_5a5a_1234_5678); + assert_eq!(prepared.submitted_after, 3); + assert_eq!(prepared.previous_timestamp_after, 2); + } + + /// G17P's native graphics wrappers select stage-specific launch classes, + /// while direct entry-ring captures show the same 0xffff word for both + /// ordinary TA and 3D kicks. Keep those exact +0x20 qwords pinned + /// independently of the CL profile. + #[test] + fn g17p_native_render_dependency_control_words_are_exact() { + fn operands(barriers: &[G17PClBarrierDependency]) -> G17PClKickEntryOperands<'_> { + let empty = G17PClRceBinding { address: 0, tag: 0 }; + G17PClKickEntryOperands { + descriptor_flag_4c: false, + descriptor_flag_5c8: false, + converted_command_timestamp: 1, + barriers, + mcache: None, + payload: [0; 2], + event_mask: [0; 4], + rce_kind: 0, + rce_bindings: [empty; 4], + auxiliary: 0, + } + } + + let ta = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY + .initial_producer_state( + 0, + 2, + G17PClQosConfig { + queue_byte_32: 2, + class_40: 0x0c, + word_48: 0xffff, + }, + ) + .unwrap(); + let ta = prepare_g17p_cl_kick_entry(ta, operands(&[])).unwrap(); + assert_eq!(get_u64_le(&ta.entry, 0x20), 0xffff_0c02_0000_0001); + + let fragment_barrier = [G17PClBarrierDependency { + native_timestamp: None, + event_word: 0x100, + shared_stamp: 0, + queue_id: 0, + }]; + let fragment = T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY + .initial_producer_state( + 1, + 2, + G17PClQosConfig { + queue_byte_32: 2, + class_40: 0x18, + word_48: 0xffff, + }, + ) + .unwrap(); + let fragment = + prepare_g17p_cl_kick_entry(fragment, operands(&fragment_barrier)).unwrap(); + assert_eq!( + get_u64_le(&fragment.entry, 0x20), + 0xffff_1802_0000_0003 + ); + } + + #[test] + fn g17p_cl_entry_reports_invalid_geometry_and_fields() { + let empty = G17PClRceBinding { address: 0, tag: 0 }; + let queue = G17PClKickProducerState { + queue_id: 0, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + completion_selector: 0, + qos: G17PClQosConfig { + queue_byte_32: 0, + class_40: 0, + word_48: 0, + }, + submitted: 0, + current_timestamp: 1, + last_add_kicks_timestamp: 0, + base_timestamp: 1, + previous_valid: false, + previous_timestamp: 0, + }; + let operands = G17PClKickEntryOperands { + descriptor_flag_4c: true, + descriptor_flag_5c8: true, + converted_command_timestamp: 1, + barriers: &[], + mcache: None, + payload: [0; 2], + event_mask: [0; 4], + rce_kind: 0, + rce_bindings: [empty; 4], + auxiliary: 0, + }; + + assert_eq!( + prepare_g17p_cl_kick_entry( + G17PClKickProducerState { + geometry: G17SksmQueueGeometry { + entry_stride: 0, + ..T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY + }, + ..queue + }, + operands, + ), + Err(SubmissionError::KickQueueGeometryInvalid) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + G17PClKickProducerState { + geometry: G17SksmQueueGeometry { + entry_stride: 0x17f, + ..T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY + }, + ..queue + }, + operands, + ), + Err(SubmissionError::KickQueueGeometryInvalid) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + G17PClKickProducerState { + geometry: G17SksmQueueGeometry { + stamp_count: 0, + ..T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY + }, + ..queue + }, + operands, + ), + Err(SubmissionError::KickQueueGeometryInvalid) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + G17PClKickProducerState { + submitted: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY.stamp_count() as u32, + ..queue + }, + operands, + ), + Err(SubmissionError::KickQueueSubmittedFull) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + G17PClKickProducerState { + qos: G17PClQosConfig { + class_40: 0x20, + ..queue.qos + }, + ..queue + }, + operands, + ), + Err(SubmissionError::KickQosClassOutOfRange) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + queue, + G17PClKickEntryOperands { + mcache: Some(G17PClMcacheAperture { + address: 0x20, + index: 0x40, + count: 1, + }), + ..operands + }, + ), + Err(SubmissionError::KickMcacheFieldOutOfRange) + ); + assert_eq!( + prepare_g17p_cl_kick_entry( + queue, + G17PClKickEntryOperands { + rce_bindings: [G17PClRceBinding { address: 1, tag: 0 }, empty, empty, empty,], + ..operands + }, + ), + Err(SubmissionError::KickRceAddressUnencodable) + ); + + let too_many = [G17PClBarrierDependency { + native_timestamp: None, + event_word: 0, + shared_stamp: 0, + queue_id: 0, + }; G17P_CL_DEPENDENCIES_MAX]; + assert_eq!( + prepare_g17p_cl_kick_entry( + queue, + G17PClKickEntryOperands { + barriers: &too_many, + ..operands + }, + ), + Err(SubmissionError::KickDependencyCountOutOfRange) + ); + } + + #[test] + fn host_add_kicks_tags_match_all_three_pinned_producers() { + let geometry = G17SksmScratchGeometry { + write0_base: 0x100, + write1_base: 0x104, + stride: 3, + }; + + for queue_id in 0u8..=0x7f { + let p = prepare_sksm_add_kicks_mmio(SksmAddKicksTarget::G17P, queue_id, 5, geometry) + .unwrap(); + assert_eq!(p.tag, 0x0000_0400_0002_1018 | ((queue_id as u64) << 43)); + assert_eq!(p.tag_offset, 0x11b); + assert_eq!(p.count_offset, 0x11f); + assert_eq!(p.add_count, 5); + assert_eq!( + p.write_pair(), + G17SksmOrderedWritePair { + word0_offset: 0x11b, + word0: 0x0000_0400_0002_1018 | ((queue_id as u64) << 43), + word1_offset: 0x11f, + word1: 5, + } + ); + + let g = prepare_sksm_add_kicks_mmio( + SksmAddKicksTarget::G17G, + queue_id, + 0x89ab_cdef, + geometry, + ) + .unwrap(); + assert_eq!(g.tag, 0x0000_0800_0002_1018 | ((queue_id as u64) << 44)); + + let x = + prepare_sksm_add_kicks_mmio(SksmAddKicksTarget::G17X, queue_id, u32::MAX, geometry) + .unwrap(); + assert_eq!(x.tag, 0x0020_0800_0002_1018 | ((queue_id as u64) << 44)); + assert_eq!(x.add_count, u32::MAX); + } + } + + #[test] + fn pair_writer_publishes_two_64_bit_words_in_order() { + use std::cell::RefCell; + + struct Recorder(RefCell>); + + impl G17SksmPairWriter for Recorder { + type Error = (); + + fn write64(&self, offset: u32, value: u64) -> core::result::Result<(), Self::Error> { + self.0.borrow_mut().push((offset, value)); + Ok(()) + } + } + + let pair = prepare_sksm_add_kicks_mmio( + SksmAddKicksTarget::G17P, + 0x2a, + 0x89ab_cdef, + g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch, + ) + .unwrap() + .write_pair(); + let recorder = Recorder(RefCell::new(Vec::new())); + + assert_eq!(publish_g17p_sksm_write_pair(&recorder, pair), Ok(())); + assert_eq!( + recorder.0.into_inner(), + vec![ + (0x4090, 0x0001_5400_0002_1018), + (0x4098, 0x0000_0000_89ab_cdef), + ] + ); + } + + #[test] + fn pair_writer_does_not_publish_word1_after_word0_error() { + use std::cell::Cell; + + struct FirstWriteError(Cell); + + impl G17SksmPairWriter for FirstWriteError { + type Error = u32; + + fn write64(&self, _offset: u32, _value: u64) -> core::result::Result<(), Self::Error> { + let calls = self.0.get() + 1; + self.0.set(calls); + Err(calls) + } + } + + let writer = FirstWriteError(Cell::new(0)); + let pair = G17SksmOrderedWritePair { + word0_offset: 0x4090, + word0: 1, + word1_offset: 0x4098, + word1: 2, + }; + assert_eq!(publish_g17p_sksm_write_pair(&writer, pair), Err(1)); + assert_eq!(writer.0.get(), 1); + } + + #[test] + fn host_add_kicks_mmio_offsets_report_overflow() { + let valid = G17SksmScratchGeometry { + write0_base: 0, + write1_base: 0, + stride: u32::MAX / 9, + }; + let publication = + prepare_sksm_add_kicks_mmio(SksmAddKicksTarget::G17G, 0, 1, valid).unwrap(); + assert_eq!(publication.tag_offset, (u32::MAX / 9) * 9); + + assert_eq!( + prepare_sksm_add_kicks_mmio( + SksmAddKicksTarget::G17P, + 0x80, + 1, + G17SksmScratchGeometry { + write0_base: 0, + write1_base: 4, + stride: 0, + }, + ), + Err(SubmissionError::QueueIdOutOfRange) + ); + for geometry in [ + G17SksmScratchGeometry { + write0_base: 0, + write1_base: 0, + stride: u32::MAX / 9 + 1, + }, + G17SksmScratchGeometry { + write0_base: u32::MAX, + write1_base: 0, + stride: 1, + }, + G17SksmScratchGeometry { + write0_base: 0, + write1_base: u32::MAX, + stride: 1, + }, + ] { + assert_eq!( + prepare_sksm_add_kicks_mmio(SksmAddKicksTarget::G17X, 0, 1, geometry), + Err(SubmissionError::KickMmioOffsetOverflow) + ); + } + } + + #[test] + fn g17p_sksm_queue_configuration_matches_the_host_store_sequence() { + assert_eq!( + G17P_SKSM_RUNTIME_GEOMETRY_OBJECT_LAYOUT, + G17PSksmRuntimeGeometryObjectLayout { + write0_base_field: 0x10730, + write1_base_field: 0x10734, + stride_field: 0x10738, + } + ); + assert_eq!(G17P_SKSM_QUEUE_ENABLE_MASK_BASE, 0x21068); + assert_eq!(G17P_SKSM_QUEUE_ENTRY_ADDRESS_MASK, 0x0000_07ff_ffff_ffe0); + + let config = prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0x40, + 1, + 0, + 0x12340, + g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch, + ) + .unwrap(); + + // configureHardware: base + 9*stride, tag, payload, then QID mask. + assert_eq!(config.tag_offset, 0x4090); + assert_eq!(config.tag, 0x0002_0400_0002_1008); + assert_eq!(config.payload_offset, 0x4098); + // This hard-coded value catches both the 32-bit selector arithmetic + // and the host's ADD of `(data_master + 1) << 60` before the OR. + assert_eq!(config.payload, 0x2280_0000_0001_2340); + assert_eq!( + config.write_pair(), + G17SksmOrderedWritePair { + word0_offset: 0x4090, + word0: 0x0002_0400_0002_1008, + word1_offset: 0x4098, + word1: 0x2280_0000_0001_2340, + } + ); + assert_eq!( + config.enable_pair, + G17SksmOrderedWritePair { + word0_offset: 0x21070, + word0: 1, + word1_offset: 0x21068, + word1: 0, + } + ); + assert_eq!( + config.disable_pair, + G17SksmOrderedWritePair { + word0_offset: 0x21080, + word0: 1, + word1_offset: 0x21078, + word1: 0, + } + ); + } + + #[test] + fn g17p_compute_queue_matches_the_captured_native_cl2_tuple() { + assert_eq!(G17P_TA_DATA_MASTER, 0); + assert_eq!(G17P_3D_DATA_MASTER, 1); + assert_eq!(G17P_COMPUTE_DATA_MASTER, 2); + + let config = prepare_g17p_compute_sksm_queue_config( + G17PComputeQueueRouting { + queue_id: 4, + completion_selector: 2, + entry_gpu_va: 0x0000_0070_004d_8000, + geometry: T8140_G17P_HAL200_SKSM_QUEUE_GEOMETRY, + }, + g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch, + ) + .unwrap(); + + assert_eq!(config.tag_offset, 0x4090); + assert_eq!(config.tag, 0x0000_2400_0002_1008); + assert_eq!(config.payload_offset, 0x4098); + assert_eq!(config.payload, 0x3100_0070_004d_8000); + assert_eq!( + config.enable_pair, + G17SksmOrderedWritePair { + word0_offset: 0x21070, + word0: 0, + word1_offset: 0x21068, + word1: 1 << 4, + } + ); + assert_eq!( + config.disable_pair, + G17SksmOrderedWritePair { + word0_offset: 0x21080, + word0: 0, + word1_offset: 0x21078, + word1: 1 << 4, + } + ); + } + + #[test] + fn g17p_sksm_queue_configuration_is_identity_and_range_scoped() { + let geometry = g17_manager::T8140_G17P_PLATFORM_CONFIG.sksm_scratch; + + for target in [ + G17ChannelAbiTarget::M5G17G, + G17ChannelAbiTarget::M5ProMaxG17X, + ] { + assert_eq!( + prepare_g17p_sksm_queue_config(target, 0, 0, 0, 0x20, geometry), + Err(SubmissionError::ChannelAbiUnconfirmed) + ); + } + assert_eq!( + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0x80, + 0, + 0, + 0x20, + geometry, + ), + Err(SubmissionError::QueueIdOutOfRange) + ); + assert_eq!( + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0, + 3, + 0, + 0x20, + geometry, + ), + Err(SubmissionError::KickQueueDataMasterOutOfRange) + ); + assert_eq!( + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0, + 0, + 5, + 0x20, + geometry, + ), + Err(SubmissionError::KickQueueCompletionSelectorOutOfRange) + ); + for address in [0x21, 0x0000_0800_0000_0000] { + assert_eq!( + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0, + 0, + 0, + address, + geometry, + ), + Err(SubmissionError::KickQueueEntryAddressUnencodable) + ); + } + assert_eq!( + prepare_g17p_sksm_queue_config( + G17ChannelAbiTarget::A18ProG17P, + 0, + 0, + 0, + 0x20, + G17SksmScratchGeometry { + write0_base: u32::MAX, + write1_base: 0, + stride: 1, + }, + ), + Err(SubmissionError::KickMmioOffsetOverflow) + ); + } + + #[test] + fn g17p_completion_descriptors_match_all_four_host_slots() { + assert_eq!(G17P_KSM_COMPLETION_QUEUE_COUNT, 4); + assert_eq!(G17P_KSM_COMPLETION_DESCRIPTOR_SIZE, 0x20); + assert_eq!(G17P_KSM_COMPLETION_ENTRY_SIZE, 0x40); + assert_eq!(G17P_KSM_COMPLETION_CAPACITY, 0x800); + assert_eq!(G17P_KSM_COMPLETION_BACKING_SIZE, 0x20000); + assert_eq!(G17P_KSM_COMPLETION_PRESENT_ORDINALS, [0, 2]); + assert_eq!(G17P_NORMAL_CL_COMPLETION_ORDINAL, 0); + assert_eq!( + G17P_KSM_COMPLETION_DESCRIPTOR_LAYOUT, + G17PKsmCompletionDescriptorLayout { + gpu_va: 0x00, + cpu_va: 0x08, + capacity: 0x10, + entry_size: 0x14, + ordinal: 0x18, + processed: 0x1c, + } + ); + + for ordinal in 0..G17P_KSM_COMPLETION_QUEUE_COUNT { + let gpu_va = 0x1234_0000 + ordinal as u64 * 0x1000; + let cpu_va = 0xabcd_0000 + ordinal as u64 * 0x2000; + let capacity = 0x80 + ordinal as u32; + let raw = encode_g17p_ksm_completion_descriptor( + G17ChannelAbiTarget::A18ProG17P, + ordinal, + gpu_va, + cpu_va, + capacity, + ) + .unwrap(); + + assert_eq!(&raw[0x00..0x08], &gpu_va.to_le_bytes()); + assert_eq!(&raw[0x08..0x10], &cpu_va.to_le_bytes()); + assert_eq!(&raw[0x10..0x14], &capacity.to_le_bytes()); + assert_eq!(&raw[0x14..0x18], &0x40u32.to_le_bytes()); + assert_eq!(&raw[0x18..0x1c], &(ordinal as u32).to_le_bytes()); + assert_eq!(&raw[0x1c..0x20], &[0; 4]); + } + assert!(G17P_COMPLETION_HOST_ABI_GROUNDED); + } + + #[test] + fn g17p_completion_descriptors_report_invalid_inputs() { + assert_eq!( + encode_g17p_ksm_completion_descriptor( + G17ChannelAbiTarget::M5G17G, + 0, + 0x1000, + 0x2000, + 1, + ), + Err(SubmissionError::ChannelAbiUnconfirmed) + ); + assert_eq!( + encode_g17p_ksm_completion_descriptor( + G17ChannelAbiTarget::A18ProG17P, + 4, + 0x1000, + 0x2000, + 1, + ), + Err(SubmissionError::CompletionQueueOrdinalOutOfRange) + ); + assert_eq!( + encode_g17p_ksm_completion_descriptor( + G17ChannelAbiTarget::A18ProG17P, + 0, + 0x1000, + 0x2000, + 0, + ), + Err(SubmissionError::CompletionQueueCapacityZero) + ); + } + + #[test] + fn kick_record_table_uses_the_firmware_geometry() { + // add x2,x22,#0x10 ; add x2,x2,QID,lsl#5 -> base 0x10, stride 0x20. + assert_eq!(KICK_RECORD_TABLE_BASE, 0x10); + assert_eq!(KICK_RECORD_STRIDE, 0x20); + assert_eq!(KICK_RECORD_COUNT, 128); + assert_eq!(kick_record_offset(0), Ok(0x10)); + assert_eq!(kick_record_offset(1), Ok(0x30)); + assert_eq!(kick_record_offset(0x7f), Ok(0x0ff0)); + assert_eq!( + kick_record_offset(0x80), + Err(SubmissionError::QueueIdOutOfRange) + ); + // The whole 128-entry table fits in 0x10 + 128*0x20 = 0x1010 bytes. + assert_eq!( + KICK_RECORD_TABLE_BASE + KICK_RECORD_COUNT * KICK_RECORD_STRIDE, + 0x1010 + ); + } + + #[test] + fn kick_record_internal_layout_is_exact() { + // Written across 0x10cb0-0x10d2c. + assert_eq!( + KICK_RECORD_LAYOUT, + KickRecordLayout { + ts_first: 0x00, + ts_last: 0x08, + slot_mask: 0x10, + valid_flag: 0x18, + slot_index: 0x1c, + } + ); + // Every field lies inside one 0x20-byte record. + for off in [ + KICK_RECORD_LAYOUT.ts_first, + KICK_RECORD_LAYOUT.ts_last, + KICK_RECORD_LAYOUT.slot_mask, + KICK_RECORD_LAYOUT.valid_flag, + KICK_RECORD_LAYOUT.slot_index, + ] { + assert!(off < KICK_RECORD_STRIDE); + } + } + + #[test] + fn g17p_top_level_command_names_match_the_host_abi() { + let known = [ + (G17PTopLevelCommandType::Ta, 0), + (G17PTopLevelCommandType::ThreeD, 1), + (G17PTopLevelCommandType::FastBlit, 2), + (G17PTopLevelCommandType::Cl, 3), + (G17PTopLevelCommandType::Barrier, 4), + (G17PTopLevelCommandType::PmGrow, 5), + (G17PTopLevelCommandType::RemoteNode, 7), + (G17PTopLevelCommandType::SharedEventSignal, 9), + (G17PTopLevelCommandType::AddFinalFrgGtpcKick, 12), + (G17PTopLevelCommandType::KsmKickQueueAddKicks, 14), + (G17PTopLevelCommandType::ConfigUpdate, 15), + (G17PTopLevelCommandType::EntrySignal, 16), + (G17PTopLevelCommandType::Nop, 17), + (G17PTopLevelCommandType::LateEvalEventSignal, 18), + ]; + for (command_type, raw) in known { + assert_eq!(command_type as u32, raw); + } + + assert_eq!(G17P_TOP_LEVEL_COMMAND_TAG_OFFSET, 0); + assert_eq!(G17P_TOP_LEVEL_COMMAND_TAG_WIDTH, 4); + assert_eq!(G17P_CHANNEL_COMMAND_TYPE_MAX, 18); + assert_eq!(G17P_TOP_LEVEL_COMMAND_KNOWN_MASK, 0x0007_d2bf); + assert_eq!(G17P_TOP_LEVEL_COMMAND_UNRESOLVED_MASK, 0x0000_0d40); + assert_eq!(G17P_TOP_LEVEL_COMMAND_INVALID_MASK, 0x0000_2000); + assert_eq!( + G17P_TOP_LEVEL_COMMAND_KNOWN_MASK + | G17P_TOP_LEVEL_COMMAND_UNRESOLVED_MASK + | G17P_TOP_LEVEL_COMMAND_INVALID_MASK, + (1 << (G17P_CHANNEL_COMMAND_TYPE_MAX + 1)) - 1 + ); + } + + #[test] + fn g17p_top_level_command_decoder_reads_exact_little_endian_bytes() { + let known = [ + G17PTopLevelCommandType::Ta, + G17PTopLevelCommandType::ThreeD, + G17PTopLevelCommandType::FastBlit, + G17PTopLevelCommandType::Cl, + G17PTopLevelCommandType::Barrier, + G17PTopLevelCommandType::PmGrow, + G17PTopLevelCommandType::RemoteNode, + G17PTopLevelCommandType::SharedEventSignal, + G17PTopLevelCommandType::AddFinalFrgGtpcKick, + G17PTopLevelCommandType::KsmKickQueueAddKicks, + G17PTopLevelCommandType::ConfigUpdate, + G17PTopLevelCommandType::EntrySignal, + G17PTopLevelCommandType::Nop, + G17PTopLevelCommandType::LateEvalEventSignal, + ]; + for command_type in known { + let mut command = [0xa5; 9]; + command[..4].copy_from_slice(&(command_type as u32).to_le_bytes()); + assert_eq!( + decode_g17p_top_level_command(G17ChannelAbiTarget::A18ProG17P, &command), + Ok(command_type) + ); + } + + for length in 0..4 { + assert_eq!( + decode_g17p_top_level_command( + G17ChannelAbiTarget::A18ProG17P, + &[0, 0, 0, 0][..length] + ), + Err(SubmissionError::CommandHeaderTooShort) + ); + } + + // The dispatcher reads all four little-endian bytes at offset zero; + // it neither truncates to byte zero nor scans for a nested opcode. + assert_eq!( + decode_g17p_top_level_command( + G17ChannelAbiTarget::A18ProG17P, + &[0, 0, 0, 1, 18, 0, 0, 0] + ), + Err(SubmissionError::CommandTagInvalid) + ); + } + + #[test] + fn g17p_top_level_command_decode_reports_bad_tag_and_identity() { + for raw in [6u32, 8, 10, 11] { + assert_eq!( + decode_g17p_top_level_command(G17ChannelAbiTarget::A18ProG17P, &raw.to_le_bytes()), + Err(SubmissionError::CommandTagUnresolved) + ); + } + for raw in [13u32, 19, u32::MAX] { + assert_eq!( + decode_g17p_top_level_command(G17ChannelAbiTarget::A18ProG17P, &raw.to_le_bytes()), + Err(SubmissionError::CommandTagInvalid) + ); + } + for target in [ + G17ChannelAbiTarget::M5G17G, + G17ChannelAbiTarget::M5ProMaxG17X, + ] { + assert_eq!( + decode_g17p_top_level_command(target, &0u32.to_le_bytes()), + Err(SubmissionError::CommandAbiUnconfirmed) + ); + assert_eq!( + classify_g17p_channel_command(target, 0), + Err(SubmissionError::CommandAbiUnconfirmed) + ); + } + } + + #[test] + fn g17p_command_dispatch_matches_the_firmware_jump_table() { + assert_eq!(G17P_CHANNEL_COMMAND_PROCESS_MASK, 0x0005_d2bf); + assert_eq!(G17P_CHANNEL_COMMAND_PM_REBUILD_MASK, 0x0000_0040); + assert_eq!(G17P_CHANNEL_COMMAND_INVALID_MASK, 0x0002_2d00); + assert_eq!( + G17P_CHANNEL_COMMAND_PROCESS_MASK + | G17P_CHANNEL_COMMAND_PM_REBUILD_MASK + | G17P_CHANNEL_COMMAND_INVALID_MASK, + (1 << (G17P_CHANNEL_COMMAND_TYPE_MAX + 1)) - 1 + ); + + for raw in 0..=G17P_CHANNEL_COMMAND_TYPE_MAX { + let expected = match raw { + 0 | 1 | 2 | 3 | 4 | 5 | 7 | 9 | 12 | 14 | 15 | 16 | 18 => { + G17PChannelCommandDisposition::Process + } + 6 => G17PChannelCommandDisposition::PmRebuildForbiddenFatal, + 8 | 10 | 11 | 13 | 17 => G17PChannelCommandDisposition::InvalidCommandFatal, + _ => unreachable!(), + }; + assert_eq!( + classify_g17p_channel_command(G17ChannelAbiTarget::A18ProG17P, raw), + Ok(expected) + ); + } + for raw in [19, u32::MAX] { + assert_eq!( + classify_g17p_channel_command(G17ChannelAbiTarget::A18ProG17P, raw), + Ok(G17PChannelCommandDisposition::InvalidCommandFatal) + ); + } + + // Host ABI identity and matched-firmware admission intentionally split. + let nop = (G17PTopLevelCommandType::Nop as u32).to_le_bytes(); + assert_eq!( + decode_g17p_top_level_command(G17ChannelAbiTarget::A18ProG17P, &nop), + Ok(G17PTopLevelCommandType::Nop) + ); + assert_eq!( + admit_g17p_top_level_command(G17ChannelAbiTarget::A18ProG17P, &nop), + Err(SubmissionError::CommandTagInvalidForFirmware) + ); + + for command_type in [ + G17PTopLevelCommandType::Ta, + G17PTopLevelCommandType::ThreeD, + G17PTopLevelCommandType::FastBlit, + G17PTopLevelCommandType::Cl, + G17PTopLevelCommandType::Barrier, + G17PTopLevelCommandType::PmGrow, + G17PTopLevelCommandType::RemoteNode, + G17PTopLevelCommandType::SharedEventSignal, + G17PTopLevelCommandType::AddFinalFrgGtpcKick, + G17PTopLevelCommandType::KsmKickQueueAddKicks, + G17PTopLevelCommandType::ConfigUpdate, + G17PTopLevelCommandType::EntrySignal, + G17PTopLevelCommandType::LateEvalEventSignal, + ] { + assert_eq!( + admit_g17p_top_level_command( + G17ChannelAbiTarget::A18ProG17P, + &(command_type as u32).to_le_bytes() + ), + Ok(command_type) + ); + } + } + + #[test] + fn g17p_command_prefilter_matches_literal_mask() { + assert_eq!(G17P_CHANNEL_COMMAND_PREFILTER_MASK, 0x0001_d03f); + for raw in 0..=20 { + let expected = matches!(raw, 0 | 1 | 2 | 3 | 4 | 5 | 12 | 14 | 15 | 16); + assert_eq!( + g17p_channel_command_prefilter_allows(G17ChannelAbiTarget::A18ProG17P, raw), + Ok(expected) + ); + } + assert_eq!( + g17p_channel_command_prefilter_allows(G17ChannelAbiTarget::M5G17G, 0), + Err(SubmissionError::CommandAbiUnconfirmed) + ); + } + + #[test] + fn provenance_ledger_is_explicit_about_reuse_and_fork() { + assert_eq!( + SUBMISSION_ABI_LEDGER.tx_doorbell, + AbiProvenance::IdenticalToAgx2 + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.kick_word, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.kick_queue_producer, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.host_add_kicks_mmio, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.g17p_queue_config, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.g17p_completion_descriptors, + AbiProvenance::ForkedAgx3Grounded + ); + assert!(HOST_SKSM_ADD_KICKS_PATH_SELECTED); + assert_eq!( + SUBMISSION_ABI_LEDGER.kick_record, + AbiProvenance::ForkedAgx3Grounded + ); + // The channel/ring state records stay inherited and unsupported. The + // widened native descriptor geometry is grounded first-hand. + assert_eq!( + SUBMISSION_ABI_LEDGER.channel_state, + AbiProvenance::Agx2InheritedUnconfirmed + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.g17p_channel_state, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.ring_state, + AbiProvenance::Agx2InheritedUnconfirmed + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.channel_descriptor, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.host_channel_state_reset, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.host_uncached_channel_reset, + AbiProvenance::ForkedAgx3Grounded + ); + assert_eq!( + SUBMISSION_ABI_LEDGER.g17p_command_types, + AbiProvenance::ForkedAgx3Grounded + ); + assert!(CHANNEL_DESCRIPTOR_STRIDE_CONFIRMED); + } + + #[test] + fn transport_tracks_classic_graphics_and_native_sksm_independently() { + assert!(GROUNDED_SUBMISSION_PRODUCER_AVAILABLE); + assert!(!EXECUTABLE_SUBMISSION_TRANSPORT_AVAILABLE); + assert!(!SUBMISSION_LIVE_GAPS.all_proven()); + assert!(!SUBMISSION_LIVE_GAPS.classic_graphics_proven()); + assert!(!SUBMISSION_LIVE_GAPS.native_sksm_proven()); + assert!(SUBMISSION_LIVE_GAPS.sksm_resource0_ordered_write_implemented); + assert!(SUBMISSION_LIVE_GAPS.channel_doorbell_round_trip_proven); + assert!(!SUBMISSION_LIVE_GAPS.partial_graphics_runtime_implemented); + assert!(SUBMISSION_LIVE_GAPS.partial_graphics_round_trip_proven); + assert!(!SUBMISSION_LIVE_GAPS.sksm_add_kicks_round_trip_proven); + assert!(!SUBMISSION_LIVE_GAPS.completion_irq_route_proven); + assert!(SUBMISSION_LIVE_GAPS.normal_cl_completion_consumer_implemented); + + let classic = SubmissionLiveGaps { + sksm_resource0_ordered_write_implemented: true, + channel_doorbell_round_trip_proven: true, + partial_graphics_runtime_implemented: true, + partial_graphics_round_trip_proven: true, + sksm_add_kicks_round_trip_proven: false, + completion_irq_route_proven: true, + normal_cl_completion_consumer_implemented: true, + }; + assert!(classic.classic_graphics_proven()); + assert!(!classic.native_sksm_proven()); + assert!(classic.all_proven()); + assert!(!classic.sksm_add_kicks_round_trip_proven); + + let native = SubmissionLiveGaps { + sksm_resource0_ordered_write_implemented: true, + channel_doorbell_round_trip_proven: false, + partial_graphics_runtime_implemented: false, + partial_graphics_round_trip_proven: false, + sksm_add_kicks_round_trip_proven: true, + completion_irq_route_proven: true, + normal_cl_completion_consumer_implemented: true, + }; + assert!(!native.classic_graphics_proven()); + assert!(native.native_sksm_proven()); + assert!(native.all_proven()); + assert!(!SubmissionLiveGaps { + normal_cl_completion_consumer_implemented: false, + ..native + } + .native_sksm_proven()); + } +} + +// --------------------------------------------------------------------------- +// Parameter-buffer descriptor handshake +// +// Metal performs this around every render submit and this driver has never +// performed it at all. Decoded from the 25G83 host kernelcache, which carries +// both halves symbolised: +// +// AGXAcceleratorG16::halLoadPBDesc @ 0xfffffe0008a52890 before submit +// AGXAcceleratorG16::halStorePBDesc @ 0xfffffe0008a527f0 on completion +// +// They are exact inverses, which is what makes the field map trustworthy: the +// store reads back precisely the four quantities the load wrote. Confirmed on +// hardware -- a real Metal render leaves the ring state {7, 7, 0, 0}. +// +// The ring lives at `[[accelerator + 0x11358] + 0x60]` with 16-byte entries +// indexed by the `uint` third argument, and `halStorePBDesc` runs on +// `kernel_task`, i.e. from the completion path rather than the submitting +// thread. +// --------------------------------------------------------------------------- + +/// One parameter-buffer descriptor ring entry. +pub(crate) const G17P_PBDESC_ENTRY_SIZE: usize = 16; + +/// The four quantities the handshake exchanges: two 22-bit counters and two +/// flags. This is `AGXSPBDescRingState`. +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct G17PPbDescRingState { + /// Descriptor dword2, low 22 bits. + pub(crate) counter_a: u32, + /// Descriptor dword3, low 22 bits. + pub(crate) counter_b: u32, + /// Descriptor dword3 bit 31. + pub(crate) flag_high: u32, + /// Descriptor dword0 bit 0. + pub(crate) flag_low: u32, +} + +/// Encode the high segment of a parameter-buffer address. +/// +/// `PBState[+0x28]` contains the full tagged address: native passes +/// `0x1000190000`. Hardware adds an implicit `0x1000000000` bias when it +/// rebuilds the address from this segment and dword0's low 36 bits. Bit 42 +/// selects between the two kext encoding forms. +pub(crate) const fn g17p_pbdesc_segment(pb_address: u64) -> u64 { + let base: u64 = if pb_address & 0x400_0000_0000 == 0 { + 0x70_0000_0000 + } else { + 0 + }; + let high = (pb_address >> 3) & 0x80_0000_0000; + ((base.wrapping_add(pb_address)) & 0x70_0000_0000) | high +} + +/// Address reconstructed by the first G17P PB page-list read. This models the +/// hardware bias witnessed by the exact `0x8002478000` fault when the host +/// incorrectly encoded stripped address `0x2478000`. +pub(crate) const fn g17p_pbdesc_reconstructed_address(pb_address: u64) -> u64 { + (g17p_pbdesc_segment(pb_address).wrapping_add(0x10_0000_0000)) + | (pb_address & 0x0f_ffff_ffff) +} + +/// `halLoadPBDesc`: fold the ring state and the parameter-buffer address into +/// the descriptor, preserving every bit the firmware owns. +pub(crate) const fn encode_g17p_pbdesc( + previous: [u32; 4], + pb_address: u64, + pb_state_word_34: u32, + state: G17PPbDescRingState, +) -> [u32; 4] { + let segment = g17p_pbdesc_segment(pb_address); + + // dword0: address >> 4, low 3 bits kept, then bit 0 = flag_low. + let mut dword0 = ((pb_address >> 4) as u32 & !0x7) | (previous[0] & 0x7); + dword0 = (dword0 & !1) | if state.flag_low != 0 { 1 } else { 0 }; + + // dword1: segment >> 8 in the top nibble, low 28 kept, then low 22 from + // PBState[+0x34]. + let mut dword1 = ((segment >> 8) as u32 & !0x0fff_ffff) | (previous[1] & 0x0fff_ffff); + dword1 = (dword1 & !0x003f_ffff) | (pb_state_word_34 & 0x003f_ffff); + + // dword2/dword3: the two 22-bit counters, plus bit 31. + let dword2 = (previous[2] & !0x003f_ffff) | (state.counter_a & 0x003f_ffff); + let mut dword3 = (previous[3] & !0x003f_ffff) | (state.counter_b & 0x003f_ffff); + dword3 = (dword3 & !0x8000_0000) | ((state.flag_high & 1) << 31); + + [dword0, dword1, dword2, dword3] +} + +/// `halStorePBDesc`: read the ring state back out. Exact inverse of the encode. +pub(crate) const fn decode_g17p_pbdesc(descriptor: [u32; 4]) -> G17PPbDescRingState { + G17PPbDescRingState { + counter_a: descriptor[2] & 0x003f_ffff, + counter_b: descriptor[3] & 0x003f_ffff, + flag_high: descriptor[3] >> 31, + flag_low: descriptor[0] & 1, + } +} + +/// Byte offset of one descriptor within the ring. +pub(crate) const fn g17p_pbdesc_offset(index: u32) -> usize { + index as usize * G17P_PBDESC_ENTRY_SIZE +} + +// --------------------------------------------------------------------------- +// UMA page-pool descriptor handshake +// +// `AGXAcceleratorG16::halLoadUMADesc` publishes one 0x20-byte descriptor at +// `[[accelerator + 0x11360] + 0x60] + HardwareBufferID * 0x20` before the +// command descriptors become visible. `halStoreUMADesc` is the inverse on +// completion. The fourth qword belongs to the firmware and is not touched by +// either load or store. +// --------------------------------------------------------------------------- + +pub(crate) const G17P_UMA_DESC_ENTRY_SIZE: usize = 0x20; +pub(crate) const G17P_UMA_DESC_ENTRY_COUNT: u32 = 0x100; + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct G17PUmaPagePoolState { + pub(crate) page_list: u64, + pub(crate) unit_pages: u32, + pub(crate) counter_a: u32, + pub(crate) counter_b: u32, + pub(crate) empty: u32, + pub(crate) total_pages: u32, +} + +/// Exact `halLoadUMADesc` packing. The source fields live at unaligned +/// offsets `+0x14/+0x1c/+0x20/+0x24/+0x28/+0x2c` in +/// `_AGFIUMAPagePoolState`. +pub(crate) const fn encode_g17p_umadesc( + previous: [u64; 4], + state: G17PUmaPagePoolState, +) -> [u64; 4] { + [ + ((state.unit_pages as u64) << 41) | (state.page_list >> 7), + ((state.counter_b as u64) << 33) + | ((state.counter_a as u64) << 5) + | ((state.empty as u64) << 61), + state.total_pages as u64, + previous[3], + ] +} + +pub(crate) const fn g17p_umadesc_offset(hardware_buffer_id: u32) -> Option { + if hardware_buffer_id >= G17P_UMA_DESC_ENTRY_COUNT { + return None; + } + Some(hardware_buffer_id as usize * G17P_UMA_DESC_ENTRY_SIZE) +} + +/// Recover the mutable pool state before its hardware slot changes owners. +/// The address and allocation unit belong to the retained pool; word three +/// is firmware-owned and is not part of the load/store handshake. +pub(crate) const fn decode_g17p_umadesc( + descriptor: [u64; 4], + mut state: G17PUmaPagePoolState, +) -> G17PUmaPagePoolState { + state.counter_a = ((descriptor[1] >> 5) & 0x0fff_ffff) as u32; + state.counter_b = ((descriptor[1] >> 33) & 0x0fff_ffff) as u32; + state.empty = (descriptor[1] >> 61) as u32; + state.total_pages = descriptor[2] as u32; + state +} + +#[cfg(test)] +mod uma_descriptor_tests { + use super::*; + + #[test] + fn first_render_uma_descriptor_matches_the_native_packer() { + let encoded = encode_g17p_umadesc( + [0, 0, 0, 0xfeed_face_dead_beef], + G17PUmaPagePoolState { + page_list: 0x70_0000_0000, + unit_pages: 0x4_0000, + counter_a: 0, + counter_b: 0x1600, + empty: 0, + total_pages: 0x1600, + }, + ); + assert_eq!( + encoded, + [ + 0x0800_0000_e000_0000, + 0x0000_2c00_0000_0000, + 0x0000_0000_0000_1600, + 0xfeed_face_dead_beef, + ] + ); + assert_eq!(g17p_umadesc_offset(0), Some(0)); + assert_eq!(g17p_umadesc_offset(0xff), Some(0x1fe0)); + assert_eq!(g17p_umadesc_offset(0x100), None); + } +} + +/// Opcode of the USC private-memory free-list device-control record. +/// +/// MISIDENTIFIED ONCE, corrected 2026-08-31. This was briefly taken for the +/// parameter-buffer descriptor-ring publication because it is the only +/// device-control record in a whole process lifetime that carries low VAs, and +/// it appears once per queue right after `allocParameterManagement`. That is +/// adjacency, not provenance. The record is built by +/// `AGXUSCPrivMemFList::updateSubmitInfo(AGXUMADataRec*, int)`, which belongs +/// to the USC/UMA page-pool family (`halLoadUMADesc`/`halStoreUMADesc`), not +/// the parameter-buffer family (`halLoadPBDesc`/`halStorePBDesc`). +/// +/// The real owner of the PBDesc ring is `AGXAccelerator::allocPBDescriptor`, +/// which is the ONLY site in the image that stores to `accelerator + 0x11358` +/// -- the pointer `halLoadPBDesc` dereferences. It allocates the object; it +/// publishes no device-control record, so the ring reaches firmware by some +/// other route we have not yet found. +pub(crate) const G17P_DEVCTL_USC_FREELIST: u32 = 0x20; + +/// Span of the USC free-list ring, as published by the opcode-0x20 record. +/// +/// Stable across four independent captures, so it is a fixed address rather +/// than an allocation. It is NOT the parameter-buffer descriptor ring -- see +/// `G17P_DEVCTL_USC_FREELIST`. Kept because if the USC free list is a third +/// thing this driver does not maintain, this is where it lives. +pub(crate) const G17P_USC_FREELIST_LOW_VA: u64 = 0x70_0020_8000; +pub(crate) const G17P_USC_FREELIST_LOW_END: u64 = 0x70_0020_8440; +pub(crate) const G17P_USC_FREELIST_GROW_DESCRIPTOR_BYTES: u32 = 0x28; + +/// The three fields that move between boots. 256x256 and 1024x1024 gave identical values for `word_04`, and +/// `word_0c` moved by 2 between them, so none of these encode geometry -- they +/// read as per-submission or per-boot sequence counters. +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct G17PUscFreelistSequence { + /// +0x04. Observed 1, 1, 6. + pub(crate) word_04: u32, + /// +0x0c. Observed 0x8df, 0x8e1, 0x65e. + pub(crate) word_0c: u32, + /// +0x30. Observed 2, 1, 3. + pub(crate) word_30: u32, +} + +/// Build the opcode-0x20 (USC free-list) device-control record. +/// +/// Only the first 0x40 bytes of an `AGFIAcceleratorDeviceControlEntry` ever +/// reach firmware: `AGXFirmware::submitDeviceControl` copies exactly four +/// 16-byte SIMD pairs (`ldp q0,q1` / `ldp q2,q3` then `stp`) into a ring whose +/// stride is `1 << 6` and whose bound is `cmp w8, #0x100`. Our +/// `CONTROL_RECORD_SIZE` (0x40) and `CONTROL_RING_ENTRIES` (0x100) already +/// match, so the fields the capture shows at +0x40/+0x44/+0x4c are kext-side +/// bookkeeping that never crosses the interface. +pub(crate) fn encode_g17p_usc_freelist_control( + firmware_half_va: u64, + ring_low_va: u64, + ring_low_end: u64, + sequence: G17PUscFreelistSequence, + out: &mut [u8], +) -> Result<(), SubmissionError> { + if out.len() != 0x40 { + return Err(SubmissionError::PartialOpeningGraphObjectSize); + } + if ring_low_end <= ring_low_va { + return Err(SubmissionError::PartialOpeningAddressOverflow); + } + out.fill(0); + put_u32_le(out, 0x00, G17P_DEVCTL_USC_FREELIST); + put_u32_le(out, 0x04, sequence.word_04); + put_u32_le(out, 0x08, 0x3f); // constant across all three captures + put_u32_le(out, 0x0c, sequence.word_0c); + put_u32_le(out, 0x10, 0); + put_u64_le(out, 0x14, firmware_half_va); + put_u64_le(out, 0x1c, ring_low_va); + put_u64_le(out, 0x24, ring_low_end); + put_u32_le(out, 0x2c, G17P_USC_FREELIST_GROW_DESCRIPTOR_BYTES); + put_u32_le(out, 0x30, sequence.word_30); + put_u32_le(out, 0x34, 1); // constant + Ok(()) +} + +#[cfg(test)] +mod usc_freelist_tests { + use super::*; + + fn u32_at(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) + } + + fn u64_at(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) + } + + #[test] + fn linux_owned_first_render_usc_freelist_record_is_exact() { + let mut record = [0xa5; 0x40]; + encode_g17p_usc_freelist_control( + 0xffff_fc20_c082_8000, + G17P_USC_FREELIST_LOW_VA, + G17P_USC_FREELIST_LOW_END, + G17PUscFreelistSequence { + word_04: 1, + word_0c: 0, + word_30: 0, + }, + &mut record, + ) + .unwrap(); + assert_eq!(u32_at(&record, 0x00), G17P_DEVCTL_USC_FREELIST); + assert_eq!(u32_at(&record, 0x04), 1); + assert_eq!(u32_at(&record, 0x08), 0x3f); + assert_eq!(u32_at(&record, 0x0c), 0); + assert_eq!(u64_at(&record, 0x14), 0xffff_fc20_c082_8000); + assert_eq!(u64_at(&record, 0x1c), G17P_USC_FREELIST_LOW_VA); + assert_eq!(u64_at(&record, 0x24), G17P_USC_FREELIST_LOW_END); + assert_eq!( + u32_at(&record, 0x2c), + G17P_USC_FREELIST_GROW_DESCRIPTOR_BYTES + ); + assert_eq!(u32_at(&record, 0x30), 0); + assert_eq!(u32_at(&record, 0x34), 1); + assert!(record[0x38..].iter().all(|byte| *byte == 0)); + } +} + + +#[cfg(test)] +mod exact_prior_tests { + use super::*; + + #[test] + fn absent_prior_preserves_every_legacy_header_form() { + for base in [0, 1, 100, u64::MAX - 10] { + for preceding in 0..5 { + for barrier in [u16::MAX, 0, 1, 3, 5] { + assert_eq!(g17p_render_order_with_prior(base, preceding, barrier, 0), + g17p_render_order(base, preceding, barrier)); + } + } + } + } + + #[test] + fn exact_old_prefix_does_not_grow_with_newer_unrelated_commands() { + for base in [7, 8, 1000] { + for preceding in [0, 1, 17] { + let order = g17p_render_order_with_prior(base, preceding, u16::MAX, 7).unwrap(); + assert_eq!(order.wait_through, Some(7)); + assert_eq!(order.sequence, base + preceding + 1); + assert!(g17p_render_prefix_contains(order.wait_through, 7)); + assert!(!g17p_render_prefix_contains(order.wait_through, 8)); + } + } + } + + #[test] + fn historical_max_preserves_zero_and_positive_relative_prefixes() { + assert_eq!(g17p_render_order_with_prior(100, 4, 0, 7).unwrap().wait_through, Some(100)); + assert_eq!(g17p_render_order_with_prior(100, 4, 3, 100).unwrap().wait_through, Some(103)); + assert_eq!(g17p_render_order_with_prior(100, 4, u16::MAX, 100).unwrap().wait_through, Some(100)); + assert_eq!(g17p_render_order_with_prior(0, 0, u16::MAX, 0).unwrap().wait_through, None); + } + + #[test] + fn earlier_current_ioctl_commands_are_not_historical() { + for prior in [11, 12, 100, u64::MAX] { + assert_eq!(g17p_render_order_with_prior(10, 100, u16::MAX, prior), + Err(SubmissionError::PriorBarrierFuture)); + } + assert_eq!(g17p_render_order_with_prior(0, 1, u16::MAX, 1), + Err(SubmissionError::PriorBarrierFuture)); + } + + #[test] + fn relative_future_and_counter_overflow_still_reject() { + assert_eq!(g17p_render_order_with_prior(10, 1, 2, 7), + Err(SubmissionError::RenderBarrierFuture)); + assert_eq!(g17p_render_order_with_prior(u64::MAX, 0, u16::MAX, 1), + Err(SubmissionError::RenderBarrierSequenceOverflow)); + assert_eq!(g17p_render_order_with_prior(u64::MAX - 1, 1, 1, 1), + Err(SubmissionError::RenderBarrierSequenceOverflow)); + } + + #[test] + fn split_ioctl_prefix_has_same_members_as_original_relative_prefix() { + let original = g17p_render_order_with_prior(20, 6, 3, 0).unwrap(); + let later_slice = g17p_render_order_with_prior(25, 1, u16::MAX, 23).unwrap(); + assert_eq!(original, later_slice); + for sequence in 0..30 { + assert_eq!(g17p_render_prefix_contains(original.wait_through, sequence), + g17p_render_prefix_contains(later_slice.wait_through, sequence)); + } + } + + #[test] + fn physical_siblings_remain_independent_prefix_members() { + let order = g17p_render_order_with_prior(50, 0, u16::MAX, 12).unwrap(); + let mut deps = [g17p_render_barrier_from_timestamp(0, 0).unwrap(); 4]; + let mut seqs = [0; 4]; + let mut count = 0; + for (sequence, qid, timestamp) in [(10, 2, 99), (11, 3, 55), (12, 2, 100)] { + assert!(g17p_render_prefix_contains(order.wait_through, sequence)); + let dep = g17p_render_barrier_from_timestamp(timestamp, qid).unwrap(); + g17p_insert_prefix_dependency(&mut deps, &mut seqs, &mut count, sequence, dep).unwrap(); + } + assert_eq!(count, 2); + assert_eq!(&seqs[..count], &[12, 11]); + assert_eq!((deps[0].queue_id, deps[1].queue_id), (2, 3)); + // Consuming one destination does not change another's exact prefix. + assert_eq!(g17p_render_order_with_prior(51, 0, u16::MAX, 12).unwrap().wait_through, + order.wait_through); + } +} diff --git a/drivers/gpu/drm/asahi/g18/fw/uma.rs b/drivers/gpu/drm/asahi/g18/fw/uma.rs new file mode 100644 index 0000000000000..ca27f807db358 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/uma.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! T8152 / 26A428 HAL400 UMA hardware descriptor serialization. +//! +//! Native `halLoadUMADesc` at 0xfffffe0008c40980 writes sixteen u64s per +//! hardware slot (0x80 bytes). This differs from M5's four-word format. +//! The caller must supply the owned firmware pool state and the four +//! device configuration words, and validate all addresses and lifetimes. +//! Encoding parity does not establish a usable pool or execute GPU work. + +#![allow(dead_code)] + +pub(crate) const G18_UMA_STATE_BYTES: usize = 0x7c; +pub(crate) const G18_UMA_DESCRIPTOR_BYTES: usize = 0x80; +pub(crate) const G18_POOL_STATE_BYTES: usize = 0xa0; + +/// Resolved inputs to the full native FList firmware state. The directory +/// address includes its sparse page-end adjustment. Accessor/policy values +/// and resource ownership are the caller's responsibility. +pub(crate) struct G18UmaPoolState { + pub(crate) statistics_id: u64, + pub(crate) grow_engine: u32, + pub(crate) priority: u32, + pub(crate) directory_gpu: u64, + pub(crate) directory_stride: u64, + /// Original FList +0xac/+0xb0 order; native rev64 swaps the output lanes. + pub(crate) chunk_counters: [u32; 2], + pub(crate) block_list_gpu: u64, + pub(crate) populated_blocks: u32, + pub(crate) block_units_64k: u32, + pub(crate) sma_mode: u32, + pub(crate) preempt_counter_gpu: u64, + pub(crate) preempt_counter: u64, + pub(crate) compute: bool, + pub(crate) policy_value: u32, +} + +/// Native state initialization at 0xfffffe0008c8d404 and serialization at +/// 0xfffffe0008c8c004. This is the 0xa0-byte firmware state, not the 0x80-byte +/// hardware UMA descriptor. Zero padding matches a fresh native allocation. +pub(crate) fn encode_g18_pool_state(state: &G18UmaPoolState) -> [u8; G18_POOL_STATE_BYTES] { + let mut out = [0; G18_POOL_STATE_BYTES]; + for (offset, value) in [ + (0x00, state.statistics_id), + (0x14, state.directory_gpu), + (0x1c, state.directory_stride), + (0x4c, state.block_list_gpu), + (0x88, state.preempt_counter_gpu), + (0x90, state.preempt_counter), + ] { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [ + (0x08, u32::MAX), + (0x0c, state.grow_engine), + (0x10, state.priority), + (0x44, state.chunk_counters[1]), + (0x48, state.chunk_counters[0]), + (0x54, state.populated_blocks), + (0x58, state.block_units_64k), + (0x7c, state.sma_mode), + (0x98, u32::from(state.compute)), + (0x9c, state.policy_value), + ] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + out +} + +/// Dimensions written by the M6 pool's `computeAndSetUMAPoolVariables`. +pub(crate) struct G18UmaGeometry { + pub(crate) pool_bytes: u64, + pub(crate) block_bytes: u64, + pub(crate) block_count: u32, + pub(crate) directory_bytes: u64, + pub(crate) initial_committed_bytes: u64, + pub(crate) total_1k_chunks: u64, + pub(crate) directory_bytes_per_usc: u64, + pub(crate) chunks_per_usc: u32, +} + +/// Derive allocation geometry after resolving the device's native accessors. +/// +/// Native arithmetic uses 32-bit intermediates for chunk counts and directory +/// sizes. Reject overflow instead of wrapping an allocation. The caller must +/// supply the actual mode and topology; this helper does not select defaults. +pub(crate) fn g18_uma_geometry( + max_bytes: u64, + block_bytes: u64, + uscs: u32, + chunks64k: bool, + page_aligned: bool, + page_shift: u32, +) -> Option { + if uscs == 0 + || uscs > 128 + || !block_bytes.is_power_of_two() + || block_bytes < 0x10000 + || !(12..=16).contains(&page_shift) + { + return None; + } + let pool_bytes = max_bytes & !(block_bytes - 1); + if pool_bytes == 0 { + return None; + } + let units = u32::try_from(pool_bytes >> 16).ok()?; + let per_usc = units.checked_add(uscs - 1)? / uscs; + let raw_directory = per_usc.checked_mul(if chunks64k { 4 } else { 512 })?; + let align = if page_aligned { 1u64 << page_shift } else { 1024 }; + let directory_bytes_per_usc = (u64::from(raw_directory) + align - 1) & !(align - 1); + Some(G18UmaGeometry { + pool_bytes, + block_bytes, + block_count: u32::try_from(pool_bytes / block_bytes).ok()?, + directory_bytes: directory_bytes_per_usc.checked_mul(u64::from(uscs))?, + initial_committed_bytes: 0, + total_1k_chunks: u64::from(units.checked_mul(64)?), + directory_bytes_per_usc, + chunks_per_usc: (units / uscs).checked_mul(64)?, + }) +} + +/// Initial sparse directory layout for the native G18 pool flags value 3. +/// +/// The virtual extent is NOT a request for that much physical backing. Only +/// `initial_page_count` pages are committed initially, at `page_offset(i)`; +/// the owner must reserve the intervening unmapped addresses for later growth. +pub(crate) struct G18UmaSparseDirectory { + pub(crate) reservation_bytes: u64, + pub(crate) page_bytes: u64, + pub(crate) initial_page_count: u32, + pub(crate) gpu_base_offset: u64, + directory_bytes_per_usc: u64, +} + +impl G18UmaSparseDirectory { + pub(crate) fn page_offset(&self, index: u32) -> Option { + if index >= self.initial_page_count { + return None; + } + let offset = self.directory_bytes_per_usc.checked_mul(u64::from(index))?; + if offset.checked_add(self.page_bytes)? > self.reservation_bytes { + return None; + } + Some(offset) + } +} + +/// Native FList init reserves one extra page and backs one page per USC plus +/// the final page. `prepareBufferResources` exposes the directory beginning +/// one KiB before the end of the first page. Other pool flag modes are outside +/// this contract; the caller must first resolve the device's actual flags. +pub(crate) fn g18_uma_sparse_directory( + geometry: &G18UmaGeometry, + uscs: u32, + page_shift: u32, +) -> Option { + if uscs == 0 || uscs > 128 || !(12..=16).contains(&page_shift) { + return None; + } + let page_bytes = 1u64 << page_shift; + let stride = geometry.directory_bytes_per_usc; + if stride < page_bytes + || stride & (page_bytes - 1) != 0 + || stride.checked_mul(u64::from(uscs))? != geometry.directory_bytes + { + return None; + } + Some(G18UmaSparseDirectory { + reservation_bytes: geometry.directory_bytes.checked_add(page_bytes)?, + page_bytes, + initial_page_count: uscs + 1, + gpu_base_offset: page_bytes - 0x400, + directory_bytes_per_usc: stride, + }) +} + +/// Native `halInitSMAEnableMask` at 0xfffffe0008c41bdc splits the active +/// 128-bit mask into sixteen groups, each stored in a 16-bit output lane. +/// The primary mask is accelerator+0x470; +0x480 is used only when it is zero. +/// `group_width` is accelerator+0x4ec. This helper admits widths fitting the +/// output lanes; wider native profiles need a separate qualified contract. +pub(crate) fn pack_g18_sma_masks( + primary: [u64; 2], + fallback: [u64; 2], + group_width: u32, +) -> Option<[u64; 4]> { + if !(1..=16).contains(&group_width) { + return None; + } + let selected = if primary == [0; 2] { fallback } else { primary }; + let mut remaining = u128::from(selected[0]) | (u128::from(selected[1]) << 64); + let mask = (1u128 << group_width) - 1; + let mut words = [0; 4]; + for group in 0..16 { + words[group / 4] |= ((remaining & mask) as u64) << ((group % 4) * 16); + remaining >>= group_width; + } + Some(words) +} + +fn read64(state: &[u8; G18_UMA_STATE_BYTES], offset: usize) -> u64 { + let mut bytes = [0; 8]; + bytes.copy_from_slice(&state[offset..offset + 8]); + u64::from_le_bytes(bytes) +} + +fn read32(state: &[u8; G18_UMA_STATE_BYTES], offset: usize) -> u32 { + let mut bytes = [0; 4]; + bytes.copy_from_slice(&state[offset..offset + 4]); + u32::from_le_bytes(bytes) +} + +/// Pack the exact M6 hardware format. Preserve native masks and truncation; +/// do not substitute M5 directory geometry or infer opaque state values. +/// +/// `device_config` is native accelerator+0x11af8..0x11b10. These four +/// values are supplied by the caller; zero is not a qualified M6 default. +pub(crate) fn pack_g18_uma_descriptor( + state: &[u8; G18_UMA_STATE_BYTES], + device_config: [u64; 4], +) -> [u64; G18_UMA_DESCRIPTOR_BYTES / 8] { + [ + (read64(state, 0x1c) << 40 & 0xfffc_0000_0000_0000) | (read64(state, 0x14) >> 10), + read64(state, 0x24), + read64(state, 0x2c), + device_config[0], + device_config[1], + device_config[2], + device_config[3], + read64(state, 0x34) << 7, + read64(state, 0x3c) << 7, + read64(state, 0x44), + (u64::from(read32(state, 0x58)) << 44) | (read64(state, 0x4c) >> 6), + u64::from(read32(state, 0x54)), + read64(state, 0x5c), + read64(state, 0x64), + read64(state, 0x6c), + read64(state, 0x74), + ] +} + +/// Fixed first-dispatch policy, recovered with the original SMAV1 sizing code. +/// This is deliberately limited to the saved addition workload, not a general +/// replacement for native SMA sizing. Hardware execution remains to be tested. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18ComputeScratchPlan { + pub(crate) block_bytes: u64, + pub(crate) block_count: u32, + pub(crate) register_pair: [u64; 2], + pub(crate) chunk_counters: [u32; 2], +} + +pub(crate) fn g18_addition_scratch(state: &[u8]) -> Option { + const WORDS: [u32; 23] = [ + 32, 10, 0, 0, 192, 16, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 128, 4, 0, 0, + ]; + if state.len() != 0x268 { return None; } + for (index, expected) in WORDS.iter().enumerate() { + let offset = 0x198 + index * 4; + let value = u32::from_le_bytes(state[offset..offset + 4].try_into().ok()?); + if value != *expected { return None; } + } + // Twelve USCs, one compute job, engine/priority 2, flags 3, no preemption, + // 2 GiB maximum and 1 MiB blocks: ideal 5 MiB, minimum 3 MiB. + // Native allocatePagesEnd supplies d550/d558 and FList+0xac/+0xb0 below. + Some(G18ComputeScratchPlan { + block_bytes: 0x10_0000, block_count: 5, + register_pair: [16, 6], chunk_counters: [0, 350], + }) +} + +/// Native 8c8bfc8..8c8bfe0: preserve metadata while encoding a GPU address. +/// Reject truncation; current backing uses one entry per 1 MiB allocation. +pub(crate) fn g18_uma_block_entry(gpu_address: u64, previous: u32) -> Option { + if gpu_address == 0 || gpu_address >= (1u64 << 48) + || gpu_address & 0x0f_ffff != 0 { + return None; + } + Some((previous & 0xf000_0000) | ((gpu_address >> 20) as u32)) +} + +/// Whether the current fixed first-dispatch policy avoids sparse-directory +/// growth. Native 8c8b8f8..8c8b9ac admits 28 one-MiB blocks on twelve USCs; +/// 29 requires new backing and is intentionally refused by this owner. +pub(crate) fn g18_initial_directory_fits( + blocks: u32, block_bytes: u64, uscs: u32, flags: u32, page_shift: u32, +) -> Option { + if blocks == 0 || !matches!(block_bytes, 0x10_0000 | 0x20_0000) + || uscs != 12 || flags != 3 || page_shift != 14 { + return None; + } + let units = (block_bytes >> 16).checked_mul(u64::from(blocks))?; + let per_usc = units.checked_add(u64::from(uscs) - 1)? / u64::from(uscs); + let initial_units = (((1u64 << page_shift) - 0x400) >> 9) + 8; + Some(per_usc <= initial_units) +} diff --git a/drivers/gpu/drm/asahi/g18/fw/vm_layout.rs b/drivers/gpu/drm/asahi/g18/fw/vm_layout.rs new file mode 100644 index 0000000000000..5088901447589 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/fw/vm_layout.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Application VA contract shared with Mesa's agx_m5_va.h. + +use core::ops::Range; + +pub(crate) const DRIVER_BASE: u64 = 0x10_0000_0000; +pub(crate) const DRIVER_END: u64 = DRIVER_BASE + (32u64 << 30); +pub(crate) const COMPACT_END: u64 = DRIVER_BASE + (1u64 << 32); +pub(crate) const DATA_START: u64 = 0x40_0000_0000; + +/// TA deflake fields encode an unsigned 32-bit offset from DRIVER_BASE. +/// Keep the private render arena inside that aperture and the USC backing +/// after it, within the same VM_CREATE reservation. A client may leave a +/// prefix of the compact aperture for its own command data. Mesa's default +/// reservation still begins at DRIVER_BASE and data begins at DATA_START. +pub(crate) fn compact_range(reserved: &Range) -> Option> { + if reserved.start < DRIVER_BASE || reserved.start >= COMPACT_END + || reserved.start & 0x3fff != 0 || reserved.end <= COMPACT_END + || reserved.end > DATA_START || reserved.end & 0x3fff != 0 { + return None; + } + Some(reserved.start..COMPACT_END) +} + +pub(crate) fn auxiliary_range(reserved: &Range) -> Option> { + compact_range(reserved)?; + Some(COMPACT_END..reserved.end) +} + +/// Command pointers must not reach driver-owned mappings, including when +/// the client leaves application data below its private arena reservation. +pub(crate) fn user_range_contains( + reserved: &Range, address: u64, size: u64, user_end: u64, +) -> bool { + let Some(end) = address.checked_add(size) else { return false; }; + size != 0 && address >= DRIVER_BASE && end <= user_end + && (end <= reserved.start || address >= reserved.end) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mesa_reservation_separates_compact_and_auxiliary_storage() { + let reserved = 0x10_0000_0000..0x18_0000_0000; + assert_eq!(compact_range(&reserved), Some(0x10_0000_0000..0x11_0000_0000)); + assert_eq!(auxiliary_range(&reserved), Some(0x11_0000_0000..0x18_0000_0000)); + assert_eq!(COMPACT_END - 1 - DRIVER_BASE, u32::MAX as u64); + } + + #[test] + fn incompatible_reservations_are_rejected() { + for reserved in [ + 0x3f7_ffff_8000..0x3ff_ffff_8000, + DRIVER_BASE..COMPACT_END, + (DRIVER_BASE - 0x4000)..DRIVER_END, + (DRIVER_BASE + 1)..DRIVER_END, + COMPACT_END..DRIVER_END, + DRIVER_BASE..(DATA_START + 0x4000), + DRIVER_BASE..0x18_0000_0001, + ] { + assert_eq!(compact_range(&reserved), None); + assert_eq!(auxiliary_range(&reserved), None); + } + } + + #[test] + fn compact_application_prefix_stays_outside_driver_storage() { + let reserved = (DRIVER_BASE + 0x400000)..DRIVER_END; + assert_eq!(compact_range(&reserved), Some(reserved.start..COMPACT_END)); + assert_eq!(auxiliary_range(&reserved), Some(COMPACT_END..DRIVER_END)); + assert!(user_range_contains(&reserved, DRIVER_BASE, 0x400000, 1 << 42)); + assert!(user_range_contains(&reserved, reserved.end, 0x4000, 1 << 42)); + for (address, size) in [ + (DRIVER_BASE - 1, 1), (DRIVER_BASE, 0), + (reserved.start - 1, 2), (reserved.start, 0x4000), + (reserved.end - 1, 2), (DRIVER_BASE, DRIVER_END), + ((1 << 42) - 1, 2), (u64::MAX, 2), + ] { + assert!(!user_range_contains(&reserved, address, size, 1 << 42)); + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/hw.rs b/drivers/gpu/drm/asahi/g18/hw.rs new file mode 100644 index 0000000000000..efc30b8c15949 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/hw.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! T8152 SoC configuration and the native G18G registry facts. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. +use kernel::prelude::*; +pub(crate) use crate::g18::identity::{GpuGen, GpuHalGeneration, GpuVariant}; + +/// The `hw::agx3` paths used by the carried module-1134 code. +#[allow(unused_imports)] +pub(crate) mod agx3 { + pub(crate) use super::{G18RegistryFacts, SocConfig, T8152, T8152_REGISTRY}; +} + +/// GPU revision enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevision { + A0 = 0x00, + A1 = 0x01, + B0 = 0x10, + B1 = 0x11, + C0 = 0x20, + C1 = 0x21, +} + +/// GPU revision ID. Note: Part of the firmware ABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevisionID { + // Unknown = 0, + A0 = 1, + A1 = 2, + B0 = 3, + B1 = 4, + C0 = 5, + C1 = 6, +} + +/// Specific GPU ID configuration fetched from SGX MMIO registers. +#[derive(Debug)] +pub(crate) struct GpuIdConfig { + /// GPU generation (should match static config). + pub(crate) gpu_gen: GpuGen, + /// GPU variant type (should match static config). + pub(crate) gpu_variant: GpuVariant, + /// Unified shader core generation. + pub(crate) usc_generation: u32, + /// Apple GPU HAL generation (0 for the pre-numbered generations). + pub(crate) gpu_hal_generation: GpuHalGeneration, + /// GPU silicon revision. + pub(crate) gpu_rev: GpuRevision, + /// GPU silicon revision ID (firmware enum). + pub(crate) gpu_rev_id: GpuRevisionID, + /// Number of GPU dies read from the ID register. + pub(crate) num_dies: u32, + /// Total number of GPU clusters. + pub(crate) num_clusters: u32, + /// Maximum number of GPU cores per cluster. + pub(crate) num_cores: u32, + /// Number of frags per cluster. + pub(crate) num_frags: u32, + /// Number of GPs per cluster. + pub(crate) num_gps: u32, + /// Total number of active cores for the whole GPU. + pub(crate) total_active_cores: u32, + /// Mask of active cores per cluster. + pub(crate) core_masks: KVec, + /// Packed mask of all active cores. + pub(crate) core_masks_packed: KVec, + /// GPC performance-state map nibble sampled during the safe SGX ID probe. + /// G17P emits this cached high nibble in fragment register 0x1c838. The + /// 3D descriptor tail at +0x2110 has an independent render-setup source. + pub(crate) gpc_perf_state_map: u32, + /// Low nibble of the same SGX 0xe01480 word, written to 3D tail +0x210c. + pub(crate) gpc_perf_state_map_low: u32, + /// Bit 0 of SGX 0xe0141c, written to 3D tail +0x2104. + pub(crate) gpc_perf_state_control: u32, +} + +/// Identification-only configuration for one AGX3 SoC. +#[derive(Debug)] +pub(crate) struct SocConfig { + /// Chip ID in hex format (e.g. 0x6050 for t6050). + pub(crate) chip_id: u32, + /// Decoded design generation for this chip. + pub(crate) gpu_gen: GpuGen, + /// Decoded variant for this chip. + pub(crate) gpu_variant: GpuVariant, + /// Hardware family byte expected at `0xd04000[31:24]`. + pub(crate) hw_family: u8, + /// Hardware variant byte expected at `0xd04000[23:16]`. + pub(crate) hw_variant: u8, + /// Die count expected at `0xd04010[19:16]`. + pub(crate) num_dies: u32, + /// Input-address bits per UAT root (42 on every AGX3 part). + pub(crate) uat_ias: u8, + /// Output address space (42-bit on every AGX3 part). + pub(crate) uat_oas: u32, +} + +const fn agx3( + chip_id: u32, + gpu_gen: GpuGen, + gpu_variant: GpuVariant, + hw_family: u8, + hw_variant: u8, + num_dies: u32, +) -> SocConfig { + SocConfig { + chip_id, + gpu_gen, + gpu_variant, + hw_family, + hw_variant, + num_dies, + uat_ias: 42, + uat_oas: 42, + } +} + +// G18 (M6 family). Family byte 0xc, variant 2: 26A428 +// `AGX·PI_400·G·A0·Accelerator::readChipInfo` @ 0xfffffe0008c3d3a0 compares +// the family byte with 0xc @ 0xfffffe0008c3d49c and gives variant 2 the +// 12-core G configuration @ 0xfffffe0008c3d4bc..0x4c8. Native J873 registry +// (AGXAcceleratorG18G): gpu_gen=18, gpu_var=G, usc_gen=3, is_sksm=1, +// num_cores=12, num_gps=6, num_mgpus=2, core_mask_list=(63,63), +// num_frags=12, kickid_qid_shift=40, kickid_qid_mask=127. The live ID +// registers have not been read yet (the GPU PSD reads idle at cold boot), so +// the T8152 probe identifies the part by chip id and compatible, and only +// logs a mismatching register decode. The ADT uat-vaddr-size is 43 on both +// J813 and J873: 42 bits per root plus the TTBR-select bit, as on T8142. +/// Apple M6 (G18G, J873 Mac mini). +pub(crate) const T8152: SocConfig = agx3(0x8152, GpuGen::G18, GpuVariant::G, 0xc, 2, 1); + +/// Native J873 `AGXAcceleratorG18G` registry facts (macOS 26A428 ioreg). +/// +/// These are identity facts, not firmware tuning: they size the core-mask +/// decode and cross-check the live ID registers once the GPU is powered. +#[allow(dead_code)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RegistryFacts { + pub(crate) num_cores: u32, + pub(crate) num_gps: u32, + pub(crate) num_mgpus: u32, + /// Per-mGPU (cluster) core masks, `core_mask_list`. + pub(crate) core_mask_list: [u32; 2], + pub(crate) num_frags: u32, + pub(crate) usc_generation: u32, + pub(crate) kickid_qid_shift: u32, + pub(crate) kickid_qid_mask: u32, +} + +/// J873 registry facts (PROVEN from the native ioreg). +#[allow(dead_code)] +pub(crate) const T8152_REGISTRY: G18RegistryFacts = G18RegistryFacts { + num_cores: 12, + num_gps: 6, + num_mgpus: 2, + core_mask_list: [0x3f, 0x3f], + num_frags: 12, + usc_generation: 3, + kickid_qid_shift: 40, + kickid_qid_mask: 127, +}; diff --git a/drivers/gpu/drm/asahi/g18/identity.rs b/drivers/gpu/drm/asahi/g18/identity.rs new file mode 100644 index 0000000000000..ab25cc7803b5d --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/identity.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU identity decoding (family, variant, die count) and the G18 UABI enums. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. + +/// GPU generation enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuGen { + G13 = 13, + G14 = 14, + G15 = 15, + G16 = 16, + G17 = 17, + /// M6 family (T8152 G18G). AGX·PI_400 host HAL. + G18 = 18, +} + +/// GPU variant enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuVariant { + P = 'P' as u32, + G = 'G' as u32, + S = 'S' as u32, + C = 'C' as u32, + D = 'D' as u32, + A = 'A' as u32, +} + +/// Apple's GPU HAL generation. +/// +/// G13 through G15 predate the numbered HAL namespace. This is separate from +/// both the design generation and the USC generation: G17P/G17A use HAL200, +/// while the other G17 parts use HAL300. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuHalGeneration { + Legacy = 0, + Hal200 = 200, + Hal300 = 300, + /// `AGX·PI_400` (26A428 AGXG18G, `__ZTV31AGX·PI_400·G·A0·Accelerator`). + Hal400 = 400, +} + +/// Firmware-role topology presented by Apple's RTBuddy driver. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub(crate) enum FirmwareRoleTopology { + /// One GFX firmware role. + Single, + /// Separate GFX and GFX1 firmware roles. + Dual, +} + +/// Host-to-firmware submission transport present in Apple's driver. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub(crate) enum SubmissionTransport { + /// Classic device-control ring and SKU channels only. + Classic, + /// Both classic and 128-queue SKSM stacks are linked. + ClassicAndSksm, + /// 128-queue SKSM transport only. + Sksm, +} + +/// Identity and userspace-visible architecture axes decoded from GPU ID MMIO. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub(crate) struct GpuIdentity { + pub(crate) gpu_gen: GpuGen, + pub(crate) gpu_variant: GpuVariant, + pub(crate) usc_generation: u32, + pub(crate) gpu_hal_generation: GpuHalGeneration, + pub(crate) firmware_roles: FirmwareRoleTopology, + pub(crate) submission_transport: SubmissionTransport, + /// Input-address bits per UAT root. + pub(crate) uat_input_address_bits: u8, +} + +/// Decode the hardware family/variant/die tuple reported at 0xd04000/0xd04010. +/// +/// This deliberately does not imply that firmware, initdata, command +/// submission, or a core-mask register layout is implemented for the returned +/// identity. +pub(crate) fn decode_gpu_identity(family: u8, variant: u8, num_dies: u8) -> Option { + use GpuGen::*; + use GpuHalGeneration::*; + use GpuVariant::*; + + let (gpu_gen, gpu_variant, usc_generation, gpu_hal_generation) = match (family, variant) { + (0x4, 0) => (G13, P, 2, Legacy), + (0x4, 2) => (G13, G, 2, Legacy), + (0x4, 3) => (G13, S, 2, Legacy), + // G13 has no D core identity; C covers both Max and Ultra. + (0x4, 4) => (G13, C, 2, Legacy), + + (0x5, 0) => (G14, P, 2, Legacy), + (0x5, 2) => (G14, G, 2, Legacy), + (0x6, 3) => (G14, S, 2, Legacy), + (0x6, 4) => match num_dies { + 1 => (G14, C, 2, Legacy), + 2 => (G14, D, 2, Legacy), + _ => return None, + }, + + // G15P is A16, not an M3 GPU. Apple's compiler routes + // applegpu_g15p through the AGX2 backend even though the design + // generation is 15. Keep USC generation independent from gpu_gen. + (0x7, 0) => (G15, P, 2, Legacy), + (0x7, 2) => (G15, G, 3, Legacy), + (0x7, 3) => (G15, S, 3, Legacy), + (0x7, 4) => match num_dies { + 1 => (G15, C, 3, Legacy), + 2 => (G15, D, 3, Legacy), + _ => return None, + }, + + (0x8, 0) => (G16, P, 3, Hal200), + + // Family 0xa crosses the design-generation boundary. + (0xa, 0) => (G17, P, 3, Hal200), + (0xa, 2) => (G16, G, 3, Hal200), + (0xa, 3) => (G16, S, 3, Hal200), + (0xa, 4) => (G16, C, 3, Hal200), + (0xa, 5) => (G17, A, 3, Hal200), + + (0xb, 2) => (G17, G, 3, Hal300), + (0xb, 3) => (G17, S, 3, Hal300), + (0xb, 4) => (G17, C, 3, Hal300), + + // G18 (M6). 26A428 `AGX·PI_400·G·A0·Accelerator::readChipInfo` + // @ 0xfffffe0008c3d3a0 accepts only family 0xc (`lsr w8,w23,#24; + // cmp w8,#0xc` @ 0xfffffe0008c3d498) and decodes variants 0, 2 and 5 + // (@ 0xfffffe0008c3d4a4..0x4b8). Variant 2 stores 12 at +0x78, which + // matches the J873 AGXAcceleratorG18G registry (num_cores=12, + // gpu_var=G, usc_gen=3). Variants 0 and 5 have no named target here + // and stay undecoded. + (0xc, 2) => (G18, G, 3, Hal400), + _ => return None, + }; + + let firmware_roles = if matches!(gpu_gen, G17 | G18) { + FirmwareRoleTopology::Dual + } else { + FirmwareRoleTopology::Single + }; + + let submission_transport = match (family, variant) { + // G16X and G17P/G17A carry both stacks. The runtime selector is not + // yet known, so callers must not silently choose one. + (0xa, 0 | 3 | 4 | 5) => SubmissionTransport::ClassicAndSksm, + // HAL300 removes the classic SKU channel stack. + (0xb, 2 | 3 | 4) => SubmissionTransport::Sksm, + // PI_400 links only `AGX·PI_400·G·A0·{3D,CL,TA}ChannelSKSM`; the + // 26A428 AGXG18G symbol table has no SKU channel class. + (0xc, 2) => SubmissionTransport::Sksm, + _ => SubmissionTransport::Classic, + }; + + let uat_input_address_bits = if matches!(gpu_gen, G13 | G14) { 39 } else { 42 }; + + Some(GpuIdentity { + gpu_gen, + gpu_variant, + usc_generation, + gpu_hal_generation, + firmware_roles, + submission_transport, + uat_input_address_bits, + }) +} diff --git a/drivers/gpu/drm/asahi/g18/initdata/application.rs b/drivers/gpu/drm/asahi/g18/initdata/application.rs new file mode 100644 index 0000000000000..9f0d800f76aee --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/initdata/application.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +use super::*; +use crate::g18::uapi::application_runtime::Frame; +use crate::g18::runtime::graphics_job::application::ApplicationJob; + +impl G18FirmwareMemory { + pub(crate) fn application_aliases(&mut self, vm: &mmu::Vm) -> Result> { + self.uma_pool.application_aliases(&self.uat,vm) + } + + pub(crate) fn audit_application(&mut self, dev: &AsahiDevice, job: &mut ApplicationJob) -> Result { + let frame=job.frame();let context=u32::from(job.job.context_id()); + let pb=job.job.parameter_buffer().verify_resident(&mut self.pbdesc,context,frame.generation)?; + let uma=self.uma_pool.verify_application_descriptor(&mut self.uma,j873_sma_device_words(2,6)?,frame.generation==1)?; + let pages=self.uma_pool.audit_render_mappings(&self.uat,job.job.vm())?; + job.audit(dev,&self.uat,&pb,&uma,pages)?; + if frame.generation==1 { + let state=self.uma_pool.render_state()?;let mut row=[0u8;128]; + for (out,word) in row.chunks_exact_mut(8).zip(uma) {out.copy_from_slice(&word.to_le_bytes());} + for (name,bytes) in [("uma-state",state.as_slice()),("uma-row",row.as_slice())] { + dev_info!(dev.as_ref(),"G18_RENDER_DIGEST name={} fnv={:#018x}\n",name,crate::g18::fw::graphics_payload::fnv64(bytes)); + } + } + self.completion_masks=[0;2];Ok(()) + } + + pub(crate) fn publish_application_outer(&mut self, records: &[[u8;24];2], frame: Frame) -> Result { + for (index,record) in records.iter().enumerate() { + let qid=if index==0 {10} else {9}; + if record[22]!=qid || record[23]!=1 || record[16..20]!=2u32.to_le_bytes() + || record[20..22]!=(frame.inner_after() as u16).to_le_bytes() + || record[8..16]!=(crate::g18::address_map::GRAPHICS_QUEUE_HIGH+if qid==9 {0} else {0x100}).to_le_bytes() {return Err(EINVAL);} + } + self.publish_application_records(records) + } + + /// Graphics and compute share this outer ring but have independent inner + /// sequences. Only publish from a fully retired observed outer position. + pub(crate) fn publish_application_records(&mut self, records: &[[u8;24]]) -> Result { + if records.is_empty() || records.len()>2 || self.control_counters()?!=[[2;3],[1;3]] { + return Err(EINVAL); + } + let counters=self.compute_outer_counters()?;let before=counters[0]; + if before>=256 || counters!=[before;3] {return Err(EBUSY);} + for record in records { + let address=u64::from_le_bytes(record[8..16].try_into().map_err(|_|EINVAL)?); + let expected=match record[22] { + 9=>crate::g18::address_map::GRAPHICS_QUEUE_HIGH, + 10=>crate::g18::address_map::GRAPHICS_QUEUE_HIGH+0x100, + 11=>crate::g18::address_map::COMPUTE_GRAPH, + _=>return Err(EINVAL), + }; + if address!=expected || record[..8]!=[0;8] || record[23]!=1 + || record[16..20]!=2u32.to_le_bytes() + || u16::from_le_bytes(record[20..22].try_into().map_err(|_|EINVAL)?)>=256 { + return Err(EINVAL); + } + } + let after=(before+records.len() as u32)%256; + self.shared.with_bytes_mut(|out| { + for (index,record) in records.iter().enumerate() { + let start=init::G18G_26A428_WORK_RING_BUNDLE_OFFSETS[8]+((before as usize+index)%256)*24; + out.get_mut(start..start+24).ok_or(EINVAL)?.copy_from_slice(record); + } Ok(()) + })?; + if self.compute_outer_counters()?!=[before;3] {return Err(EBUSY);} + fence(Ordering::SeqCst); + self.private.write_u32(PRIVATE_BASE+(init::G18G_26A428_PRIVATE_STATE_GRIDS[0] + +init::WORK_STATE_GRID_OFFSETS[8]+0x20) as u64,after)?; + fence(Ordering::SeqCst);Ok(after) + } + + pub(crate) fn publish_application_compute_outer(&mut self, inner: u32) -> Result { + if inner>=256 {return Err(EINVAL);} + let mut record=[0;24]; + record[8..16].copy_from_slice(&crate::g18::address_map::COMPUTE_GRAPH.to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&(inner as u16).to_le_bytes()); + record[22]=11;record[23]=1; + self.publish_application_records(&[record]) + } +} diff --git a/drivers/gpu/drm/asahi/g18/initdata/application_compute.rs b/drivers/gpu/drm/asahi/g18/initdata/application_compute.rs new file mode 100644 index 0000000000000..0875b61333836 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/initdata/application_compute.rs @@ -0,0 +1,534 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded application CDM owner, separate from the captured addition fixture. +use super::*; +use crate::{g18::uapi::command::TranslatedComputeCommand, g18::address_map as va, g18::fw::compute::*}; +use core::sync::atomic::Ordering; + +// Application-compute-only knobs K1-K5 (compute diagnostics) and +// g18_app_d62a (case 10). Unlike +// kernel::module! parameters (perm 0, set once at load), these are 0644 with +// the C uint ops, so /sys/module/asahi/parameters/g18_app_* may be changed +// between submissions. prepare_application_compute reads each exactly once +// per submission; nothing else reads them, so graphics and the stage8 fixture +// are unaffected. Registration otherwise follows g18_render_policy.rs. +macro_rules! app_knob { + ($module:ident, $name:literal, $default:expr, $description:literal) => { + mod $module { + use kernel::{bindings, module_param::KernelParam}; + pub(super) static VALUE: core::sync::atomic::AtomicU32 = + core::sync::atomic::AtomicU32::new($default); + #[link_section = "__param"] + #[used(compiler)] + static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + kernel::c_str!($name) + } else { kernel::c_str!(concat!("asahi.", $name)) }), + // Module lifetime and parameter serialization are managed by the + // kernel, identically to the kernel::module! registration. A raw + // address avoids a reference to the mutable extern static. + #[cfg(MODULE)] + mod_: &raw mut bindings::__this_module, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + // SAFETY: exported vmlinux ops; they access `arg` as an unsigned + // int under the parameter lock, which AtomicU32 represents exactly. + ops: unsafe { core::ptr::from_ref(&bindings::param_ops_uint) }, + perm: 0o644, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { + arg: core::ptr::from_ref(&VALUE).cast_mut().cast(), + }, + }); + const TYPE: &[u8] = concat!("parmtype=", $name, ":uint\0").as_bytes(); + #[link_section = ".modinfo"] + #[used(compiler)] + static TYPE_INFO: [u8; TYPE.len()] = super::modinfo(TYPE); + const TEXT: &[u8] = concat!("parm=", $name, ":", $description, "\0").as_bytes(); + #[link_section = ".modinfo"] + #[used(compiler)] + static TEXT_INFO: [u8; TEXT.len()] = super::modinfo(TEXT); + } + }; +} + +const fn modinfo(bytes: &[u8]) -> [u8; N] { + let mut out = [0; N]; + let mut i = 0; + while i < N { out[i] = bytes[i]; i += 1; } + out +} + +app_knob!(knob_d514, "g18_app_d514", 0xffff_ffff, + "Application compute command +0xf28 (native desc+0x514). Default 0xffffffff = native, fixture and M5; 0 = candidate160. Read at every compute submission"); +// Defaults are the values that retire on hardware (run 162d: exact integer +// compute and atomics readbacks). The native 0xa731=1 alone stalls the +// dispatch silently, with no fault or watchdog. +app_knob!(knob_a731, "g18_app_a731", 0, + "Application compute first 0xa731: 0 (default) = fixture/M5 (emits 0), proven on hardware; 1 = native integer compute (d420=1,d418=2 emits 1), which stalls the dispatch on Linux. Other values refuse the submission"); +app_knob!(knob_sma_pair, "g18_app_sma_pair", 0x6_0010, + "Application compute 0xa5e1 SMA pair as d550|d558<<16: 0x60010 (16,6) fixture (default, proven on hardware), 0xb001e (30,11) native integer compute, 0xc001e (30,12) candidate160. Must fit the pool plan (30,12); other values refuse the submission"); +app_knob!(knob_sma_units, "g18_app_sma_units", 0, + "Application compute 0x10241[63:16] SMA result (AGXUMADataRec+0x100). Default 0 (proven on hardware); 0xae = native integer compute. Independent of the pool binding generation"); +app_knob!(knob_prepare_test, "g18_app_prepare_test", 0, + "Unpublished preparation rollback E2E: 0 disabled, 1 inject four allocation failures before the next compute command is encoded, 2 do this for the next two commands to cover reuse after retirement. Verify rollback then prepare each original command. Other values refuse before preparation"); +app_knob!(knob_pool_test, "g18_app_pool_test", 0, + "Pool replacement E2E: arm 1 before the first compute. Starts with the qualified 8-MiB pool, then runs original non-spill commands with 30,40,30-MiB shared pools; internal phases 2-4 replace only after verified retirement. Checks staged-owner rollback and reused addresses. Does not admit scratch shaders"); +app_knob!(knob_preempt_mode, "g18_app_preempt_mode", 1, + "Application compute 0x1a510 object: 1 (default) = fresh zeroed 0xc000 bytes mapped only in the job VM at 0x1100000000 (driver arena); 0 = candidate160 16 KiB page at 0x7050000000. Other values refuse the submission"); +app_knob!(knob_d62a, "g18_app_d62a", 1, + "Application compute desc+0x62a (0x1a458 0x14c08a60 texture mode instead of 0x10c08860): 1 (default) = from the command's DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE flag, as native; 0 = never; 2 = always. Other values refuse the submission"); + +/// Every SMA pair witnessed in a completed native or fixture compute command. +const SMA_PAIRS: [[u64; 2]; 3] = [[30, 11], [30, 12], [16, 6]]; + +/// One submission's knob values, read once and logged with its kick. +#[derive(Clone, Copy)] +pub(crate) struct ApplicationComputeKnobs { + d514: u32, + a731: u32, + sma_pair: u32, + sma_units: u32, + metadata_plan: bool, + preempt_mode: u32, + d62a_mode: u32, + preempt: u64, + preempt_bytes: u64, + pool_slot: u32, + /// The command's USES_TEXTURE_STATE flag and the d62a it produced. + texture_state: bool, + d62a: u8, +} + +impl ApplicationComputeKnobs { + fn read() -> Self { + Self { + d514: knob_d514::VALUE.load(Ordering::Relaxed), + a731: knob_a731::VALUE.load(Ordering::Relaxed), + sma_pair: knob_sma_pair::VALUE.load(Ordering::Relaxed), + sma_units: knob_sma_units::VALUE.load(Ordering::Relaxed), + metadata_plan: false, + preempt_mode: knob_preempt_mode::VALUE.load(Ordering::Relaxed), + d62a_mode: knob_d62a::VALUE.load(Ordering::Relaxed), + preempt: 0, preempt_bytes: 0, pool_slot: 0, + texture_state: false, d62a: 0, + } + } + + /// Native run112 sets desc+0x62a (0x1a458 = 0x14c08a60) on every compute + /// that uses texture or image instructions and on none of the + /// buffer-only ones; Mesa reports that per command. + fn resolve_d62a(&mut self, texture_state: bool) { + self.texture_state = texture_state; + self.d62a = match self.d62a_mode { + 0 => 0, + 2 => 1, + _ => u8::from(texture_state), + }; + } + + fn pair(&self) -> [u64; 2] { + [u64::from(self.sma_pair & 0xffff), u64::from(self.sma_pair >> 16)] + } + + fn apply_plan(&mut self, plan: crate::g18::uapi::compute_plan::G18ComputePlan, + dev: &AsahiDevice) -> Result { + // Fixed knobs remain available for legacy diagnostics. Never combine + // them with shader-derived ownership or silently ignore an override. + if self.sma_pair != 0x6_0010 || self.sma_units != 0 { + dev_err!(dev.as_ref(), "G18_APP_COMPUTE_REFUSED metadata with SMA overrides\n"); + return Err(EINVAL); + } + self.sma_pair = plan.ideal.register_pair[0] | plan.ideal.register_pair[1] << 16; + self.sma_units = plan.sma_units; + self.metadata_plan = true; + Ok(()) + } + + /// Refuse, before any owner changes, values that would make the command + /// inconsistent. Nothing is published for a refused submission. + fn validate(&self, dev: &AsahiDevice) -> Result { + let refusal = if self.a731 > 1 { + Some(("g18_app_a731", self.a731, "admitted 0 (fixture) or 1 (native)")) + } else if !SMA_PAIRS.contains(&self.pair()) { + Some(("g18_app_sma_pair", self.sma_pair, + "admitted 0xb001e (30,11), 0xc001e (30,12) or 0x60010 (16,6)")) + } else if self.preempt_mode > 1 { + Some(("g18_app_preempt_mode", self.preempt_mode, + "admitted 0 (0x7050000000 page) or 1 (driver arena)")) + } else if self.d62a_mode > 2 { + Some(("g18_app_d62a", self.d62a_mode, + "admitted 0 (never), 1 (from the command flag) or 2 (always)")) + } else { None }; + if let Some((knob, value, reason)) = refusal { + dev_err!(dev.as_ref(), + "G18_APP_COMPUTE_REFUSED knob={} value={:#x}: {}; command not sent\n", + knob, value, reason); + return Err(EINVAL); + } + Ok(()) + } +} + +impl kernel::fmt::Display for ApplicationComputeKnobs { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let [d550, d558] = self.pair(); + write!(f, "d514={:#x} a731={} sma_pair=({},{}) reg_a5e1={:#x} sma_units={:#x} reg_10241={:#x} preempt_mode={} preempt={:#x}+{:#x} texture_state={} d62a_mode={} d62a={} metadata_plan={}", + self.d514, self.a731, d550, d558, d550 | d558 << 16, self.sma_units, + (self.pool_slot as i32 as i64 as u64) | u64::from(self.sma_units) << 16, + self.preempt_mode, self.preempt, self.preempt_bytes, + u8::from(self.texture_state), self.d62a_mode, self.d62a, u8::from(self.metadata_plan)) + } +} + +pub(super) struct ApplicationCompute { + guard: Option, + aliases: KVec, + // g18_app_preempt_mode=1 object, mapped only in the submitting job VM. + // Replaced only after the previous generation retired. + driver_preempt: Option, + pool: G18UmaPoolMemory, + binding: G18UmaPoolBinding, + previous_pool: Option, + previous_binding: Option, + bound_context: u32, + successful_retirement: bool, + /// Distinguish production shader plans from the legacy replacement probe. + metadata_pool: bool, + preempt: MappedObject, + generation: u32, + retired: bool, +} + +impl G18FirmwareMemory { + pub(crate) fn prepare_application_compute(&mut self, dev: &AsahiDevice, + vm: &mmu::Vm, command: &TranslatedComputeCommand) + -> Result<(G18PreparedComputeCommand, G18UmaPoolBinding, u32, ApplicationComputeKnobs)> { + if command.sampler_heap != 0 || command.sampler_count != 0 + || command.g17p_add3_buffers.is_some() + || command.control_stream_end.checked_sub(command.control_stream_base) + .map_or(true, |n| n > crate::g18::drm::G18_COMPUTE_STREAM_BYTES) { + pr_err!("G18 compute prepare rejected: sampler_heap={:#x} samplers={} add3={} stream {:#x}..{:#x} (admits at most {} bytes)\n", + command.sampler_heap, command.sampler_count, command.g17p_add3_buffers.is_some(), + command.control_stream_base, command.control_stream_end, + crate::g18::drm::G18_COMPUTE_STREAM_BYTES); + return Err(ENOTSUPP); + } + let mut knobs = ApplicationComputeKnobs::read(); + knobs.validate(dev)?; + knobs.resolve_d62a(command.g18_texture_state); + let diagnostic = knob_prepare_test::VALUE.swap(0, Ordering::Relaxed); + if diagnostic > 2 { return Err(EINVAL); } + let pool_test = knob_pool_test::VALUE.swap(0, Ordering::Relaxed); + if pool_test > 4 || (pool_test != 0 && diagnostic != 0) { return Err(EINVAL); } + let metadata_plan = match command.g18_resource_descriptor.as_ref() { + Some(words) => Some(crate::g18::uapi::compute_plan::G18ComputePlan::from_descriptor(words) + .ok_or(EINVAL)?), + None => None, + }; + if let Some(plan) = metadata_plan { + if diagnostic != 0 || pool_test != 0 { return Err(EINVAL); } + knobs.apply_plan(plan, dev)?; + } + if self.application_compute.as_ref().is_some_and(|job| job.metadata_pool) + && (diagnostic != 0 || pool_test != 0) { return Err(EINVAL); } + let pool_plan = if pool_test != 0 { + let mut words = [48,12,0,0,2304,24,0,0,2,1,0,0,0,0,0,0,1,0,0,128,4,0,0]; + if pool_test == 3 { words[4] = 3072; } + Some(crate::g18::uapi::compute_plan::G18ComputePlan::from_descriptor(&words).ok_or(EINVAL)?) + } else { None }; + if pool_test == 1 && self.application_compute.is_some() + || pool_test > 1 && self.application_compute.is_none() { return Err(EINVAL); } + if self.application_compute.is_none() { + let high = va::DYNAMIC_HIGH_START..va::DYNAMIC_HIGH_END; + let mut pool = if let Some(plan) = metadata_plan { + G18UmaPoolMemory::new_shared_compute(dev, &self.uat, high, plan)? + } else { + G18UmaPoolMemory::new_application_compute(dev, &self.uat, high)? + }; + let binding = pool.prepare_binding()?; + let preempt = MappedObject::new_wc_in_range(dev, self.uat.kernel_lower_vm(), + 0x70_5000_0000..0x70_5100_0000, 0x4000, 0x4000, + mmu::PROT_GPU_FW_SHARED_RW)?; + let mut owner = KBox::new(ApplicationCompute { + guard: None, aliases: KVec::new(), driver_preempt: None, pool, binding, + previous_pool: None, previous_binding: None, bound_context: 0, + successful_retirement: true, + metadata_pool: metadata_plan.is_some(), + preempt, generation: 0, retired: true, + }, GFP_KERNEL)?; + self.scheduler.prepare(0, 1)?; + owner.pool.load_hardware_descriptor(dev, &mut self.uma, binding, + j873_sma_device_words(2, 6)?)?; + self.application_compute = Some(owner); + } + if diagnostic != 0 { + let before_generation = self.application_compute_generation(); + let before = self.scheduler.snapshot(0)?; + for checkpoint in 1..=4 { + let result = self.prepare_application_compute_attempt(dev, vm, command, knobs, checkpoint, None); + if !matches!(result, Err(error) if error == ENOMEM) { return Err(EIO); } + let owner = self.application_compute.as_ref().ok_or(EIO)?; + let after = self.scheduler.snapshot(0)?; + if !owner.retired || owner.guard.is_some() || !owner.aliases.is_empty() + || owner.driver_preempt.is_some() || owner.generation != before_generation + || !self.compute.unpublished_idle() + || (before.submitted, before.completed, before.linked) + != (after.submitted, after.completed, after.linked) { return Err(EIO); } + dev_info!(dev.as_ref(), + "G18_COMPUTE_PREPARE_E2E checkpoint={} rollback=pass generation={} scheduler={}/{}/{} published=0\n", + checkpoint, before_generation, after.submitted, after.completed, after.linked); + } + } + if pool_test > 1 { + let job = &mut **self.application_compute.as_mut().ok_or(EIO)?; + let scheduler = self.scheduler.snapshot(0)?; + if !job.retired || !job.successful_retirement || job.previous_pool.is_some() + || job.generation == 0 || scheduler.submitted != job.generation + || scheduler.completed != job.generation || scheduler.linked != 0 { + return Err(EBUSY); + } + job.pool.verify_replacement_rollback(dev, &self.uat, pool_plan.ok_or(EINVAL)?)?; + } + let replacement = if let Some(plan) = metadata_plan { + let job = self.application_compute.as_ref().ok_or(EIO)?; + (job.pool.shared_compute_plan() != Some(plan)).then_some(plan) + } else if pool_test > 1 { pool_plan } else { None }; + let prepared = self.prepare_application_compute_attempt(dev, vm, command, knobs, 0, replacement)?; + if pool_test != 0 && pool_test < 4 { + let _ = knob_pool_test::VALUE.compare_exchange(0, pool_test + 1, + Ordering::Relaxed, Ordering::Relaxed); + } + if diagnostic == 2 { + // The next submission still requires this command to retire. + // Preserve a concurrent explicit parameter change, and never add + // a fallible operation after the prepared owner was committed. + let _ = knob_prepare_test::VALUE.compare_exchange(0, 1, + Ordering::Relaxed, Ordering::Relaxed); + } + Ok(prepared) + } + + fn prepare_application_compute_attempt(&mut self, dev: &AsahiDevice, + vm: &mmu::Vm, command: &TranslatedComputeCommand, + mut knobs: ApplicationComputeKnobs, fail_checkpoint: u32, + replacement: Option) + -> Result<(G18PreparedComputeCommand, G18UmaPoolBinding, u32, ApplicationComputeKnobs)> { + let job = &mut **self.application_compute.as_mut().ok_or(EIO)?; + if !job.retired || job.guard.is_some() || !self.compute.unpublished_idle() + || job.previous_pool.is_some() || job.previous_binding.is_some() { return Err(EBUSY); } + let generation = job.generation.checked_add(1).ok_or(EOVERFLOW)?; + if generation > 0x00ff_ffff { return Err(EOVERFLOW); } + let previous = self.scheduler.snapshot(0)?; + if previous.submitted != job.generation || previous.completed != job.generation + || previous.linked != 0 { return Err(EBUSY); } + // The pool is sized once for its plan pair; a larger pair would name + // backing that does not exist. The slot must not reach the SMA half. + let mut candidate = if let Some(plan) = replacement { + if !job.successful_retirement || job.generation == 0 { return Err(EBUSY); } + let mut pool = job.pool.replacement_shared_compute(dev, &self.uat, + va::DYNAMIC_HIGH_START..va::DYNAMIC_HIGH_END, plan)?; + let mut binding = pool.prepare_binding()?; + binding.generation = job.binding.generation.checked_add(1).ok_or(EOVERFLOW)?; + if binding.generation == u64::MAX { return Err(EOVERFLOW); } + Some((pool, binding)) + } else { None }; + let (pool, binding) = match candidate.as_mut() { + Some((pool, binding)) => (pool, *binding), + None => (&mut job.pool, job.binding), + }; + let plan = pool.scratch_plan().ok_or(EINVAL)?; + let pair = knobs.pair(); + if let Some(expected) = command.g18_resource_descriptor.as_ref() { + let expected = crate::g18::uapi::compute_plan::G18ComputePlan::from_descriptor(expected) + .ok_or(EINVAL)?; + if pool.shared_compute_plan() != Some(expected) + || pair != expected.ideal.register_pair.map(u64::from) + || knobs.sma_units != expected.sma_units || !knobs.metadata_plan { + return Err(EIO); + } + crate::g18_kick_info!(dev.as_ref(), + "G18_COMPUTE_RESOURCE_PLAN bytes={} blocks={} pair={:?} counters={:?} sma_units={} replacement={}\n", + expected.ideal.bytes, expected.ideal.block_count, pair, + expected.ideal.chunk_counters, expected.sma_units, u8::from(replacement.is_some())); + } else if job.metadata_pool + && (plan.block_bytes != 0x20_0000 || plan.block_count < 4 + || plan.register_pair[0] < 30 || plan.register_pair[1] < 12 + || plan.chunk_counters[1] < 260) + { + dev_err!(dev.as_ref(), "G18_APP_COMPUTE_REFUSED legacy command exceeds retained native pool\n"); + return Err(ENOTSUPP); + } + if pair[0] > plan.register_pair[0] || pair[1] > plan.register_pair[1] { + dev_err!(dev.as_ref(), + "G18_APP_COMPUTE_REFUSED knob=g18_app_sma_pair value={:#x}: exceeds pool plan {:?}; command not sent\n", + knobs.sma_pair, plan.register_pair); + return Err(EINVAL); + } + if binding.slot >= 0x1_0000 { + dev_err!(dev.as_ref(), + "G18_APP_COMPUTE_REFUSED knob=g18_app_sma_units value={:#x}: pool slot {} overlaps 0x10241[63:16]; command not sent\n", + knobs.sma_units, binding.slot); + return Err(EINVAL); + } + knobs.pool_slot = binding.slot; + let driver = va::APPLICATION_COMPUTE_PREEMPT_DRIVER; + let driver_bytes = va::APPLICATION_COMPUTE_PREEMPT_DRIVER_BYTES; + let driver_range = driver..driver + driver_bytes + mmu::UAT_PGSZ as u64; + let reserved = vm.kernel_range(); + if knobs.preempt_mode == 1 + && (driver_range.start < reserved.start || driver_range.end > reserved.end) { + dev_err!(dev.as_ref(), + "G18_APP_COMPUTE_REFUSED knob=g18_app_preempt_mode value=1: VM_CREATE driver range {:#x}..{:#x} does not contain {:#x}..{:#x}; command not sent\n", + reserved.start, reserved.end, driver_range.start, driver_range.end); + return Err(EINVAL); + } + let guard = vm.retain_t8140_job()?; + job.aliases.clear(); + job.driver_preempt = None; + let attempt = (|| { + if fail_checkpoint == 1 { return Err(ENOMEM); } + let context = self.compute.bind_process(dev, &self.uat, vm)?; + if fail_checkpoint == 2 { return Err(ENOMEM); } + let mut aliases = pool.compute_application_aliases(dev, &self.uat, vm)?; + if fail_checkpoint == 3 { return Err(ENOMEM); } + let mut driver_preempt = None; + let preempt = if knobs.preempt_mode == 0 { + let preempt = job.preempt.iova(); + job.preempt.with_bytes_mut(|bytes| { bytes.fill(0); Ok(()) })?; + let alias = job.preempt.map_alias_at(vm, preempt, mmu::PROT_GPU_SHARED_RW)?; + aliases.push(alias, GFP_KERNEL)?; + if vm.translate_iova(preempt)? != self.uat.kernel_lower_vm().translate_iova(preempt)? { + return Err(EFAULT); + } + knobs.preempt_bytes = 0x4000; + preempt + } else { + // Native keeps this object in the process VM only (0x100_003f_0000, + // 48 KiB). The driver arena is refused to userspace binds, and the + // range admits exactly one placement: the object plus one guard. + let object = MappedObject::new_wc_in_range_with_guards(dev, vm, driver_range, + driver_bytes as usize, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_SHARED_RW, 1)?; + let preempt = object.iova(); + driver_preempt = Some(object); + if preempt != driver { return Err(EFAULT); } + for offset in (0..driver_bytes).step_by(mmu::UAT_PGSZ) { + vm.translate_iova(preempt + offset)?; + } + if vm.translate_iova(preempt + driver_bytes).is_ok() { return Err(EFAULT); } + knobs.preempt_bytes = driver_bytes; + preempt + }; + knobs.preempt = preempt; + if fail_checkpoint == 4 { return Err(ENOMEM); } + let inputs = G18ComputeCommandInputs { + sequence: u64::from(generation), d100: 1, + completion: G18ComputeCompletionState { + stamp: generation << 8, index: 11, entries: 256, generation: 0, base: 1, + }, + d4e0: command.control_stream_terminator, d7b8: binding.state, + ..Default::default() + }; + // G18 preemption pointers are +0x1b80..+0x1b98 (native gallery), + // not the M5 +0x1480 layout. These are fresh driver-owned zero pages. + let registers = G18ComputeRegisterInputs { + program: [preempt, command.control_stream_base, preempt + 0x1b80, + preempt + 0x1b88, preempt + 0x1b90, preempt + 0x1b98], + d550: pair[0], d558: pair[1], d774: 0x1c, d5c4: 8, + d420: knobs.a731 as u8, d418: 2, d50c: 1, d514: knobs.d514, + d62a: knobs.d62a, + context_id: context, pool_slot: binding.slot, + sma_units: u64::from(knobs.sma_units), + a10aac: 1, a6d0: 1, clusters: 2, cluster_width: 6, + ..Default::default() + }; + let prepared = self.compute.prepare(self.scheduler.binding(0)?, &inputs, ®isters)?; + Ok((prepared, aliases, driver_preempt, knobs)) + })(); + let (prepared, aliases, driver_preempt, knobs) = match attempt { + Ok(value) => value, + Err(error) => { + // The local VM guard outlives process/alias cleanup. A cleanup + // failure remains fatal; it is never called a clean rollback. + self.compute.abort_unpublished_process()?; + return Err(error); + } + }; + job.guard = Some(guard); + job.aliases = aliases; + job.driver_preempt = driver_preempt; + job.retired = false; + job.generation = generation; + if command.g18_resource_descriptor.is_some() { job.metadata_pool = true; } + if let Some((pool, binding)) = candidate { + job.previous_pool = Some(core::mem::replace(&mut job.pool, pool)); + job.previous_binding = Some(job.binding); + job.binding = binding; + } + self.completion_masks = [0; 2]; + Ok((prepared, job.binding, generation, knobs)) + } + + /// The binding the firmware must still own before the next tag15. + pub(crate) fn application_compute_previous_binding(&self) -> Result<(G18UmaPoolBinding, u32)> { + let job = self.application_compute.as_ref().ok_or(EIO)?; + Ok((job.previous_binding.unwrap_or(job.binding), job.bound_context)) + } + + /// Called only after the session checked the old firmware binding and the + /// retired scheduler. Both views remain owned if any later step fails. + pub(crate) fn install_application_compute_pool(&mut self, dev: &AsahiDevice) -> Result { + let job = &mut **self.application_compute.as_mut().ok_or(EIO)?; + if let Some(previous) = job.previous_pool.as_mut() { + let old = job.previous_binding.ok_or(EIO)?; + let scheduler = self.scheduler.snapshot(0)?; + if !job.successful_retirement || job.generation < 2 + || scheduler.submitted != job.generation - 1 + || scheduler.completed != job.generation - 1 || scheduler.linked != 0 { + return Err(EBUSY); + } + job.pool.replace_hardware_descriptor(dev, &mut self.uma, job.binding, + previous, old, j873_sma_device_words(2, 6)?)?; + crate::g18_kick_info!(dev.as_ref(), + "G18_COMPUTE_POOL_SWITCH compute={} old_state={:#x} new_state={:#x} old_generation={} new_generation={} slot={} old_retained=1\n", + job.generation, old.state, job.binding.state, old.generation, + job.binding.generation, job.binding.slot); + } + Ok(()) + } + + /// The caller must first verify the new firmware binding, including both + /// directions of queue membership. A completion stamp alone is insufficient. + pub(crate) fn retire_application_compute(&mut self, generation: u32, context: u32) -> Result { + let job = &mut **self.application_compute.as_mut().ok_or(EIO)?; + if job.retired || job.generation != generation { return Err(EINVAL); } + self.compute.retire_application(generation << 8)?; + job.bound_context = context; + job.successful_retirement = true; + job.retired = true; + job.guard = None; + job.previous_pool = None; + job.previous_binding = None; + Ok(()) + } + + /// The last compute stamp kicked to QID 11 (0 before the first dispatch). + pub(crate) fn application_compute_generation(&self) -> u32 { + self.application_compute.as_ref().map_or(0, |job| job.generation) + } + + /// As [`Self::retire_application_compute`] for a dispatch a recovery ended. + pub(crate) fn retire_recovered_application_compute(&mut self, generation: u32) -> Result { + let job = &mut **self.application_compute.as_mut().ok_or(EIO)?; + if job.retired || job.generation != generation { return Err(EINVAL); } + self.compute.retire_recovered()?; + job.successful_retirement = false; + job.retired = true; + job.guard = None; + Ok(()) + } + + pub(crate) fn quarantine_application_compute(&mut self) { + if let Some(job) = &self.application_compute { + if let Some(guard) = &job.guard { guard.quarantine(); } + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/initdata/mod.rs b/drivers/gpu/drm/asahi/g18/initdata/mod.rs new file mode 100644 index 0000000000000..3d3c195002a24 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/initdata/mod.rs @@ -0,0 +1,948 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![allow(dead_code)] + +//! Retained RAM for the J873 / T8152 / G18G / 26A428 initialization graph +//! (plan M1, steps 1.1-1.7). +//! +//! Construction only. Nothing here starts a processor, publishes on a +//! mailbox, reads or writes MMIO, requests PMGR/PSD power, touches the ASC +//! wrappers or the post-office bank, or submits work: every write lands in +//! host-allocated GEM memory or in UAT page tables. Register windows are +//! mapped for the firmware (page-table writes only); their contents are never +//! accessed from the host. +//! +//! The G18 live-boot session owns this object. It must build the UAT with +//! `mmu::Uat::new_t8152` (plan step 0.6) before calling +//! [`G18FirmwareMemory::new_with_uat`], and must retain both the UAT and this +//! owner until BOTH firmware processors are stopped, including on failure. +//! +//! Derived from `m5_resources.rs` with the M2/M3 compute and graphics owners +//! removed (they depend on the unported HAL400 submission ABI). The firmware +//! VA plan is the J813 one: the addresses are host-chosen and both shared +//! allocations keep their 25G76 page totals (0x28000 / 0x178000) in 26A428. +//! Every object inside the allocations follows the 26A428 kext layout from +//! `g17_initdata` (`G18G_26A428_*`). Values that are not proven for 26A428 +//! are listed in `g17_initdata::G18G_26A428_UNPROVEN`. +//! +//! Integration (outside this file): include it from `g17_resources.rs` next +//! to the M5 module, so it can use the private `MappedObject`: +//! +//! ```ignore +//! #[cfg(not(test))] +//! #[path = "g18_resources.rs"] +//! pub(crate) mod g18; +//! ``` + +#[path = "application.rs"] +mod application; +#[path = "application_compute.rs"] +mod application_compute; +use application_compute::ApplicationCompute; + +use crate::g18::buffer::object::MappedObject; +use crate::g18::buffer::pool::{G18UmaPoolBinding, G18UmaPoolMemory}; +use crate::g18::queue::scheduler::G18SchedulerMemory; +use crate::g18::runtime::compute::{G18ComputeMemory, G18PreparedComputeCommand}; +use crate::g18::runtime::compute_payload::G18ComputePayloadMemory; +use crate::{driver::AsahiDevice, g18::fw::initdata as init, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; +use kernel::sync::Arc; + +const PROFILE: init::RootProfile = init::RootProfile::J873G18G26A428; + +const KERNEL_BASE: u64 = 0xffff_fc20_0000_0000; +/// Shared allocation #2 (the private cluster, 0x178000 bytes). +const PRIVATE_BASE: u64 = KERNEL_BASE + 0x2_0000; +const REGION_C_BASE: u64 = KERNEL_BASE + 0x1a_0000; +/// The secondary root sits [`init::SECONDARY_ROOT_DELTA`] above the primary. +const ROOTS_BASE: u64 = KERNEL_BASE + 0x1a_8000; +const REGION_A_BASE: u64 = KERNEL_BASE + 0x1b_8000; +/// ArmFirmware fwctl allocation: 0x20-byte state, then the ring at +0x40. +const FWCTL_BASE: u64 = KERNEL_BASE + 0x1c_0000; +const FWCTL_SIZE: usize = 0x4000; +/// Shared allocation #1 (hw_data, policy blocks, rings, main configs). +const SHARED_BASE: u64 = KERNEL_BASE + 0xc078_8000; +/// Main `+0x2d0` blank sentinel; unmapped at handoff, as on J813. +const SENTINEL_BASE: u64 = KERNEL_BASE + 0xc07e_0000; +/// PB descriptor and UMA pool tables, aliased at [`init::REGION_VIEW_LOW_ADDRS`]. +const PBDESC_BASE: u64 = KERNEL_BASE + 0x15e_0000; +const UMA_BASE: u64 = KERNEL_BASE + 0x15e_8000; +const PBDESC_TABLE_SIZE: usize = 0x4000; +/// 26A428 createHardwareMappings at 0xfffffe0008c41978 requests 0x8000 +/// bytes: 256 native 0x80-byte UMA descriptors, rather than the M5 page. +const UMA_TABLE_SIZE: usize = 0x8000; +/// The four KSM completion queues (low context-0 alias / firmware-high). +const COMPLETION_LOW_BASE: u64 = 0x70_0184_8000; +const COMPLETION_HIGH_BASE: u64 = KERNEL_BASE + 0xc07e_8000; +const COMPLETION_STRIDE: u64 = 0x2_8000; +/// Allocator-selected objects (QoS block, AON PTD) stay between the PB/UMA +/// tables and the fixed high graph. +const DYNAMIC_START: u64 = KERNEL_BASE + 0x168_0000; +const DYNAMIC_END: u64 = KERNEL_BASE + 0x1000_0000; + +const _: () = { + assert!(PBDESC_BASE + PBDESC_TABLE_SIZE as u64 <= UMA_BASE); + assert!(UMA_BASE + UMA_TABLE_SIZE as u64 <= DYNAMIC_START); + assert!(init::REGION_VIEW_LOW_ADDRS[0] + PBDESC_TABLE_SIZE as u64 + <= init::REGION_VIEW_LOW_ADDRS[1]); + assert!(init::REGION_VIEW_LOW_ADDRS[1] + UMA_TABLE_SIZE as u64 <= COMPLETION_LOW_BASE); +}; + +/// Report rings keep their 25G76 shape (FW_PROVEN: event producer b000 +/// 0x7f14, 256 x 0x48; trace producer b000 0x19984, 512 x 0x48; host owns the +/// cursor at +0, firmware the producer at +0x20). +const REPORT_RECORD_SIZE: usize = 0x48; +const REPORT_PRODUCER_OFFSET: u64 = 0x20; +const REPORT_CAPACITY: [u32; 2] = [256, 512]; +const REPORT_DRAIN_BUDGET: usize = 128; +const TRACE_PRINT_BUDGET: [u32; 2] = [512, 16]; +/// Host-only storage; do not change the native firmware ring geometry. +/// Roughly 5 MiB for both roles, allocated before publication. +const TRACE_JOURNAL_CAPACITY: usize = 32768; +/// b000 0x7de8 constructs type4 with counter at +4 and stamp index at +12. +const EVENT_TYPE_RECOVERY: u32 = 4; + +/// `halLoadUMADesc` device words (accelerator+0x11af8..0x11b10) as native +/// `halInitSMAEnableMask` 0xfffffe0008c41bdc derives them on J873. +/// getProbeScore stores readChipInfo's 0xe01500 core-enable word as the +/// accelerator+0x480 fallback (0xfffffe0008c3d6c4, 0xfffffe0008c002d8); the +/// +0x470 override stays zero without gpu-fstp-downgrade properties, which the +/// J873 ADT lacks. Width +0x4ec is ID_COUNTS_1[7:0] (0xfffffe0008c3d554) and +/// +0x4d4 the cluster count; generateRegisterList reads the same two fields, +/// so they must equal the command topology. getPropertyCoreConfigImpl splits +/// the same selected mask into the native registry core_mask_list, so every +/// 16-bit lane must equal that list (zero beyond it). +fn j873_sma_device_words(clusters: u32, width: u32) -> Result<[u64; 4]> { + let facts = crate::g18::hw::T8152_REGISTRY; + if clusters != facts.num_mgpus || facts.core_mask_list.len() != clusters as usize + || !(1..=16).contains(&width) || width.checked_mul(clusters) != Some(facts.num_cores) { + return Err(EINVAL); + } + let mut mask = 0u64; + for (index, &cores) in facts.core_mask_list.iter().enumerate() { + if u64::from(cores) >> width != 0 { return Err(EINVAL); } + mask |= u64::from(cores) << (index as u32 * width); + } + let words = crate::g18::fw::uma::pack_g18_sma_masks([0; 2], [mask, 0], width).ok_or(EINVAL)?; + for group in 0..16 { + let lane = (words[group / 4] >> ((group % 4) * 16)) & 0xffff; + if lane != facts.core_mask_list.get(group).map_or(0, |&cores| u64::from(cores)) { + return Err(EINVAL); + } + } + Ok(words) +} + +struct G18TraceRecord { + ordinal: u32, + raw: [u8; REPORT_RECORD_SIZE], +} + +/// Owning mappings are declared before the UAT so they drop first. Aliases +/// retain the SAME GEM backing; no firmware pointer names a copied snapshot. +pub(crate) struct G18FirmwareMemory { + primary_root: MappedObject, + secondary_root: MappedObject, + shared: MappedObject, + private: MappedObject, + region_a: MappedObject, + region_c: MappedObject, + fwctl: MappedObject, + aon: MappedObject, + qos: MappedObject, + pbdesc: MappedObject, + uma: MappedObject, + uma_pool: G18UmaPoolMemory, + scheduler: G18SchedulerMemory, + compute: G18ComputeMemory, + compute_payload: Option, + application_compute: Option>, + completions: KVec, + aliases: KVec, + registers: KVec, + report_counts: [[u32; 2]; 2], + completion_masks: [u64; 2], + trace_printed: [u32; 2], + trace_journal: [KVVec; 2], + trace_dropped: [u32; 2], + uat: Arc, +} + +/// Read-back of the new 26A428 status cross-links (plan step 1.3 validation): +/// status B +0xe714 header, +0xe71c word and +0xe720 pointer; status config +/// +0x6c4 word and +0x6c8 pointer. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18StatusLinks { + pub(crate) status_b_header: u32, + pub(crate) status_b_word: u32, + pub(crate) status_b_pointer: u64, + pub(crate) status_config_word: u32, + pub(crate) status_config_pointer: u64, +} + +/// The status-B recovery handshake words. +#[derive(Debug, Copy, Clone, Default)] +pub(crate) struct G18RecoveryWord { + pub(crate) epoch: u64, + pub(crate) state: u32, + pub(crate) host: u32, +} + +/// What the firmware published about one recovery: the status-B cause words, +/// the valid recovery-info entries, and the shared recovery-request block. +#[derive(Debug, Copy, Clone, Default)] +pub(crate) struct G18RecoveryCause { + pub(crate) emitted: u32, + pub(crate) source_count: u32, + pub(crate) command: u32, + pub(crate) reason: u32, + pub(crate) detail: u64, + pub(crate) slot_mask: u64, + pub(crate) valid_entries: u32, + pub(crate) first_entry: Option<(u32, u64)>, + /// The first valid entries as (index, raw). Upstream's 13.5 layout for + /// the same 256 x 8 table (`pending_stamps`) is info u32 (bit 0 valid, + /// flags 3:0, stamp slot 10:4) then the awaited stamp value u32. + pub(crate) pending: [(u16, u64); G18_RECOVERY_PENDING_LOGGED], + pub(crate) request: [u32; init::G18G_26A428_RECOVERY_REQUEST_WORDS], +} + +pub(crate) const G18_RECOVERY_PENDING_LOGGED: usize = 8; + +impl G18FirmwareMemory { + /// Allocate, map and encode the complete G18G init-data graph while both + /// firmware processors are stopped. All fallible work happens before the + /// caller publishes anything. The caller keeps its own `Arc` of the UAT + /// until both processors stop, including when this construction fails. + #[inline(never)] + pub(crate) fn new_with_uat( + dev: &AsahiDevice, + uat: Arc, + boot_timestamp: u64, + render: bool, + ) -> Result> { + if !uat.is_t8152_geometry() { + return Err(ENODEV); + } + match *crate::module_parameters::g18_compute_owner_test.value() { + 0 => {}, + 1 => G18UmaPoolMemory::verify_shared_compute_owner(dev, &uat)?, + _ => return Err(EINVAL), + } + let trace_journal = [ + KVVec::with_capacity(TRACE_JOURNAL_CAPACITY, GFP_KERNEL)?, + KVVec::with_capacity(TRACE_JOURNAL_CAPACITY, GFP_KERNEL)?, + ]; + let mut registers = KVec::new(); + for window in init::G18G_26A428_REGISTER_WINDOWS { + let (physical, virtual_address, span) = window.mapped_pages().map_err(|_| EINVAL)?; + // Flag 2 windows are firmware read-write; flag 0 windows are + // mapped read-only, the J813 rule (INFERRED for 26A428). + let protection = match window.flag { + 0 => mmu::PROT_FW_MMIO_RO, + 2 => mmu::PROT_FW_MMIO_RW, + _ => return Err(EINVAL), + }; + // Page-table writes only. No register contents are accessed. + registers.push( + uat.kernel_vm().map_io(virtual_address, physical.try_into()?, span.try_into()?, + protection)?, + GFP_KERNEL, + )?; + } + + let fixed = |address, size, protection| { + MappedObject::new_wc_at(dev, uat.kernel_vm(), address, size, protection) + }; + // Install every fixed address before allocator-selected objects. + let mut private = fixed(PRIVATE_BASE, init::G18G_26A428_PRIVATE_CLUSTER_SIZE, + mmu::PROT_FW_SHARED_RW)?; + // Both 26A428 roots are 0xc8 bytes. Separate mappings keep the empty + // page between them. + let mut primary_root = fixed(ROOTS_BASE, init::G18G_26A428_ROOT_SIZE, mmu::PROT_FW_SHARED_RO)?; + let mut secondary_root = fixed(ROOTS_BASE + init::SECONDARY_ROOT_DELTA, + init::G18G_26A428_ROOT_SIZE, mmu::PROT_FW_SHARED_RO)?; + let mut shared = fixed(SHARED_BASE, init::G18G_26A428_SHARED_CLUSTER_SIZE, mmu::PROT_FW_PRIV_RW)?; + let region_a = fixed(REGION_A_BASE, init::REGION_A_SIZE, mmu::PROT_FW_SHARED_RO)?; + let mut region_c = fixed(REGION_C_BASE, init::REGION_C_SIZE, mmu::PROT_FW_SHARED_RO)?; + let fwctl = fixed(FWCTL_BASE, FWCTL_SIZE, mmu::PROT_FW_SHARED_RW)?; + // Zero-backed at handoff; owned by later (M2/M3) publication. + let mut pbdesc = fixed(PBDESC_BASE, PBDESC_TABLE_SIZE, mmu::PROT_FW_SHARED_RW)?; + let mut uma = fixed(UMA_BASE, UMA_TABLE_SIZE, mmu::PROT_FW_SHARED_RW)?; + let mut aliases = KVec::new(); + for (object, address, size) in [ + (&mut pbdesc, init::REGION_VIEW_LOW_ADDRS[0], PBDESC_TABLE_SIZE), + (&mut uma, init::REGION_VIEW_LOW_ADDRS[1], UMA_TABLE_SIZE), + ] { + aliases.push(object.map_alias_at(uat.kernel_lower_vm(), address, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + for offset in (0..size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(object.iova() + offset as u64)? + != uat.kernel_lower_vm().translate_iova(address + offset as u64)? + { + return Err(EFAULT); + } + } + } + dev_info!(dev.as_ref(), "G18 UMA descriptor table: bytes={:#x} slots=256 stride=0x80 alias_pages={} verified\n", + UMA_TABLE_SIZE, UMA_TABLE_SIZE / mmu::UAT_PGSZ); + + let mut completions = KVec::new(); + let mut completion_bindings = + [init::G17PKsmCompletionAliases { low: 0, high: 0 }; init::M5_25G76_COMPLETION_QUEUE_COUNT]; + for (ordinal, binding) in completion_bindings.iter_mut().enumerate() { + // Host-chosen pairs, the J813 placement (INFERRED for 26A428). + let low = COMPLETION_LOW_BASE + ordinal as u64 * COMPLETION_STRIDE; + let high = COMPLETION_HIGH_BASE + ordinal as u64 * COMPLETION_STRIDE; + let mut backing = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, + init::M5_25G76_COMPLETION_BACKING_SIZE, mmu::PROT_FW_PRIV_RW)?; + aliases.push(backing.map_alias_at(uat.kernel_lower_vm(), low, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + for offset in (0..init::M5_25G76_COMPLETION_BACKING_SIZE).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset as u64)? + != uat.kernel_lower_vm().translate_iova(low + offset as u64)? + { + return Err(EFAULT); + } + } + *binding = init::G17PKsmCompletionAliases { low, high }; + completions.push(backing, GFP_KERNEL)?; + } + let dynamic = DYNAMIC_START..DYNAMIC_END; + let qos = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), dynamic.clone(), + init::G17P_QOS_RESOURCE_SIZE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)?; + let mut aon = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), dynamic, + init::G18G_26A428_AON_PTD_SIZE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_PRIV_RW)?; + let uma_pool = if render { G18UmaPoolMemory::new_render(dev, &uat)? } else { + G18UmaPoolMemory::new(dev, &uat, DYNAMIC_START..DYNAMIC_END, + crate::g18::hw::T8152_REGISTRY.num_cores)? + }; + let scheduler = G18SchedulerMemory::new(dev, &uat, DYNAMIC_START..DYNAMIC_END)?; + let facts = crate::g18::hw::T8152_REGISTRY; + let compute = G18ComputeMemory::new(dev, &uat, DYNAMIC_START..DYNAMIC_END, + facts.num_mgpus, facts.num_gps)?; + + aon.with_bytes_mut(|out| init::encode_g18g_26a428_aon_ptd(out).map_err(|_| EINVAL))?; + region_c.with_bytes_mut(|out| { + init::encode_region_c_for_profile(PROFILE, out).map_err(|_| EINVAL) + })?; + private.with_bytes_mut(|out| { + init::encode_g18g_26a428_private_cluster(PRIVATE_BASE, FWCTL_BASE, out).map_err(|_| EINVAL) + })?; + + let layout = init::G18G_26A428_PRIVATE_LAYOUT; + let addresses = init::SharedAddresses { + region_a: region_a.iova(), + region_c: region_c.iova(), + hw_data_bundle: SHARED_BASE, + hw_data_bundle_alloc: init::G18G_26A428_SHARED_CLUSTER_SIZE, + }; + let primary = init::InstanceAddresses { + root: ROOTS_BASE, + main_config: SHARED_BASE + init::G18G_26A428_MAIN_BUNDLE_OFFSETS[0] as u64, + status_a: PRIVATE_BASE + init::G18G_26A428_PRIVATE_STATUS_A[0] as u64, + status_b: PRIVATE_BASE + layout.status_b as u64, + secondary_extras: [0, 0], + }; + let secondary = init::InstanceAddresses { + root: ROOTS_BASE + init::SECONDARY_ROOT_DELTA, + main_config: SHARED_BASE + init::G18G_26A428_MAIN_BUNDLE_OFFSETS[1] as u64, + status_a: PRIVATE_BASE + init::G18G_26A428_PRIVATE_STATUS_A[1] as u64, + status_b: 0, + // Secondary root +0xb8 = status config, +0xc0 = the 0x60 object + // (KEXT_PROVEN, g18g_key.c:3748-3812). + secondary_extras: [ + PRIVATE_BASE + layout.status_config as u64, + PRIVATE_BASE + layout.secondary_extra_1 as u64, + ], + }; + init::validate_pair(&primary, &secondary).map_err(|_| EINVAL)?; + let views = init::PrimaryRegionViews { + sentinel_high: SENTINEL_BASE, + alias_high: [PBDESC_BASE, UMA_BASE], + }; + // GFX traces remain disabled here. Stage8 explicitly enables only + // re-derived b000 command/scheduler classes before root publication. + shared.with_bytes_mut(|out| { + init::encode_g18g_26a428_hw_data(completion_bindings, qos.iova(), + &mut out[..init::G18G_26A428_HW_DATA_SIZE]).map_err(|_| EINVAL)?; + init::encode_g18g_26a428_bundle_policy(init::M5PolicyBindings { + state: PRIVATE_BASE + layout.policy_state as u64, + aon_ptd: aon.iova(), + boot_timestamp, + }, out).map_err(|_| EINVAL)?; + for (role, instance, views) in [ + (init::InstanceRole::Primary, &primary, Some(&views)), + (init::InstanceRole::Secondary, &secondary, None), + ] { + let grid = PRIVATE_BASE + init::G18G_26A428_PRIVATE_STATE_GRIDS[role as usize] as u64; + let channels = init::derive_channel_table_for_profile(PROFILE, role, grid, + instance.status_a, instance.main_config, SHARED_BASE); + let offset = (instance.main_config - SHARED_BASE) as usize; + init::encode_main_config_for_profile(PROFILE, role, &addresses, instance, &channels, + views, &mut out[offset..offset + init::MAIN_CONFIG_SIZE]).map_err(|_| EINVAL)?; + } + Ok(()) + })?; + for (root, role, instance) in [ + (&mut primary_root, init::InstanceRole::Primary, &primary), + (&mut secondary_root, init::InstanceRole::Secondary, &secondary), + ] { + root.with_bytes_mut(|out| { + init::encode_root_for_profile(PROFILE, role, &addresses, instance, out) + .map_err(|_| EINVAL)?; + // The two firmware gates (version word, root+0x2c) must pass + // before anything is published. + init::check_firmware_root_gates_for_profile(PROFILE, out).map_err(|_| EINVAL) + })?; + } + // Unlike PB/UMA, the blank sentinel is unmapped at native handoff. + if uat.kernel_vm().translate_iova(SENTINEL_BASE).is_ok() { + return Err(EINVAL); + } + // Writes to all CPU WC mappings precede any future handoff. Only the + // two opening control records are staged; work rings stay empty. + fence(Ordering::SeqCst); + // Keep the retained graph off the kernel stack. + KBox::init(try_init!(Self { + primary_root, secondary_root, shared, private, region_a, region_c, fwctl, aon, qos, + pbdesc, uma, uma_pool, scheduler, compute, compute_payload: None, + application_compute: None, + completions, aliases, registers, + report_counts: [[0; 2]; 2], completion_masks: [0; 2], trace_printed: [0; 2], uat, + trace_journal, trace_dropped: [0; 2], + }), GFP_KERNEL) + } + + pub(crate) fn uma_pool(&self) -> &G18UmaPoolMemory { &self.uma_pool } + + pub(crate) fn prepare_render(&mut self, dev: &AsahiDevice, + job: &mut crate::g18::runtime::graphics_job::G18GraphicsJob) -> Result { + self.uma_pool.prepare_render_backing(dev, &self.uat, job.vm())?; + let pool = self.uma_pool.prepare_binding()?; + self.uma_pool.load_hardware_descriptor(dev, &mut self.uma, pool, j873_sma_device_words(2, 6)?)?; + job.parameter_buffer().load_row(&mut self.pbdesc)?; + job.encode_h1(dev)?; + Ok(pool) + } + + pub(crate) fn audit_render(&mut self, dev: &AsahiDevice, + job: &mut crate::g18::runtime::graphics_job::G18GraphicsJob) -> Result { + let pb = job.parameter_buffer().verify_before_kick(&mut self.pbdesc)?; + if pb.counter != 1 { return Err(EIO); } + let uma = self.uma_pool.verify_render_descriptor(&mut self.uma, j873_sma_device_words(2, 6)?)?; + let pages = self.uma_pool.audit_render_mappings(&self.uat, job.vm())?; + job.audit_h1(dev, &self.uat, &pb, &uma)?; + let state = self.uma_pool.render_state()?; + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=uma-state fnv={:#018x}\n", + crate::g18::fw::graphics_payload::fnv64(&state)); + dev_info!(dev.as_ref(), "G18_RENDER_AUDIT_UMA pages={} row=match state-slot=0\n", pages); + Ok(()) + } + + /// Batched CL_2 publication. Both queue records become visible through one + /// producer write, after every byte and every counter has been validated. + /// Used only by stage 9; the existing compute publication stays unchanged. + pub(crate) fn publish_outer_records(&mut self, records: &[[u8; 24]]) -> Result { + if records.len() != 2 || self.control_counters()? != [[2; 3], [1; 3]] { return Err(EINVAL); } + let before = self.compute_outer_counters()?; + let head = before[0]; + if before != [head; 3] || head > 6 || head & 1 != 0 { return Err(EBUSY); } + for (i, record) in records.iter().enumerate() { + let qid = record[22]; + let address = u64::from_le_bytes(record[8..16].try_into().map_err(|_| EINVAL)?); + if qid != if i == 0 {10} else {9} + || address != crate::g18::address_map::GRAPHICS_QUEUE_HIGH + if qid == 9 {0} else {0x100} + || record[16..20] != 2u32.to_le_bytes() || record[23] != 1 { return Err(EINVAL); } + } + self.shared.with_bytes_mut(|out| { + let start = init::G18G_26A428_WORK_RING_BUNDLE_OFFSETS[8] + head as usize * 24; + let slots = out.get_mut(start..start + 48).ok_or(EINVAL)?; + if slots.iter().any(|&b| b != 0) { return Err(EBUSY); } + for (out, record) in slots.chunks_exact_mut(24).zip(records) { out.copy_from_slice(record); } + Ok(()) + })?; + // No producer moved while the batch was being stored. + if self.compute_outer_counters()? != before { return Err(EBUSY); } + fence(Ordering::SeqCst); + let address = PRIVATE_BASE + (init::G18G_26A428_PRIVATE_STATE_GRIDS[0] + + init::WORK_STATE_GRID_OFFSETS[8] + 0x20) as u64; + self.private.write_u32(address, head + 2)?; + fence(Ordering::SeqCst); + Ok(head + 2) + } + + pub(crate) fn scheduler(&mut self) -> &mut G18SchedulerMemory { &mut self.scheduler } + + pub(crate) fn compute(&mut self) -> &mut G18ComputeMemory { &mut self.compute } + + /// Select the firmware-owned KSMFE queue path before root publication. + /// b000 0x216f8..0x21700 forms x10=global+0x1093e0, then 0x21884 + /// copies hwdata+0x2724 to [x10+0xb7c] (global+0x109f5c); + /// 0x12cdc..0x12de8 skips tag15 hardware queue setup when it is nonzero. + /// Native macOS uses host queue programming. This first Linux submission + /// sends tag15/tag14 and performs no host KSMFE MMIO, so choose one owner. + pub(crate) fn use_firmware_queue_programming(&mut self, dev: &AsahiDevice) -> Result { + let policy = SHARED_BASE + 0x2724; + let trace = PRIVATE_BASE + init::G18G_26A428_PRIVATE_STATUS_A[0] as u64; + let progress = REGION_C_BASE + 0x9ac; + if self.shared.read_u32(policy)? != 1 || self.private.read_u32(trace)? != 0 + || self.region_c.read_u32(progress)? != 0x333 { + return Err(EINVAL); + } + self.shared.write_u32(policy, 0)?; + // b000 0x216b8 obtains GFX status-A; 0x3508 loads its first word + // into global 0x1093e4. Bit0 gates commands/completion; bit1 gates + // scheduling/recovery. audit-g18-trace-mask-native.py executes these + // exact branches. Opt-in graphics diagnosis additionally enables + // bit9 (global 0x1093e5 bit1), the native stall diagnostic category. + // audit-g18-graphics-stall.py executes that original predicate. + let graphics_stall_trace = *crate::module_parameters::g18_graphics_stall_trace.value() == 1; + let trace_mask = if graphics_stall_trace { 0x203 } else { 3 }; + self.private.write_u32(trace, trace_mask)?; + // b000 refresh 0x352c/0x3688 copies region-C+0x9ac into + // global 0x109488. Its compute nibble (bits8..11) selects native + // progress sampling: mode3 only checks busy; mode2 takes the + // detailed sampler. TA (nibble 0) also takes mode2: b000 0x890c + // then enters the original sampler at 0x8924 instead of the + // busy-bit-only path. The busy-only path recovers any TA busy for + // region-C+0x9a0 (=0xa) one-tick scans, ~12.8 ms after the kick, + // which legitimate single-invocation vertex/geometry work reaches + // (image-op-rootcause-1113). Mode2 still recovered a real TA hang + // (linux-workloads-123, ~32 ms). Keep 3D and all timeout bounds. + let progress_policy = 0x232; + self.region_c.write_u32(progress, progress_policy)?; + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), + "G18_QUEUE_OWNER firmware: hwdata+0x2724=0 before root publication; no host KSMFE writes\n"); + dev_info!(dev.as_ref(), "G18_TRACE_MASK GFX status-A={:#x} before root publication\n", trace_mask); + dev_info!(dev.as_ref(), "G18_PROGRESS_POLICY region-C+0x9ac={:#x} before publication; timeout bounds unchanged\n", progress_policy); + Ok(()) + } + + /// Retain a private shader/input/output VM. This does not select a process + /// context or make its roots visible to firmware, and never queues work. + pub(crate) fn prepare_compute_payload(&mut self, dev: &AsahiDevice) + -> Result<&mut G18ComputePayloadMemory> { + if self.compute_payload.is_some() { return Err(EBUSY); } + self.compute_payload = Some(G18ComputePayloadMemory::load(dev, &self.uat)?); + Ok(self.compute_payload.as_mut().unwrap()) + } + + /// Connect the retained payload VM to the command/channel owner. Context + /// roots remain unpublished until the complete command has been prepared. + pub(crate) fn bind_compute_process(&mut self, dev: &AsahiDevice) -> Result { + let payload = self.compute_payload.as_ref().ok_or(EINVAL)?; + self.compute.bind_process(dev, &self.uat, payload.vm()) + } + + /// Allocate the captured job's scratch pool while its state is unpublished. + /// The returned register inputs are valid only with this retained owner. + pub(crate) fn prepare_compute_scratch(&mut self, dev: &AsahiDevice) + -> Result { + let plan = self.compute_payload.as_ref().ok_or(EINVAL)?.scratch_plan()?; + self.uma_pool.prepare_compute_backing(dev, &self.uat, DYNAMIC_START..DYNAMIC_END, plan)?; + Ok(plan) + } + + /// Assemble the single captured addition with retained Linux resources. + /// Every step is still unpublished: the caller must subsequently bind the + /// pool, publish the process roots, encode the owned SKSM entry and only + /// then publish a kick. Errors leave any partial owners retained here. + pub(crate) fn prepare_addition(&mut self, dev: &AsahiDevice) + -> Result<(G18PreparedComputeCommand, G18UmaPoolBinding)> { + use crate::g18::fw::compute::G18ComputeCompletionState; + use crate::g18::queue::probe::QUEUE_ID; + + let (mut command, mut registers) = self.prepare_compute_payload(dev)?.addition_inputs()?; + let context_id = self.bind_compute_process(dev)?; + let scratch = self.prepare_compute_scratch(dev)?; + // The hardware reads the pool through the job's context (stage27 + // GMMU fault in vm_slot 3), so the pool must be mapped in that VM. + let payload_vm = self.compute_payload.as_ref().ok_or(EINVAL)?.vm(); + self.uma_pool.alias_into_process(dev, &self.uat, payload_vm)?; + let pool = self.prepare_uma_pool_binding()?; + // Native prepareLocked loads the hardware descriptor for the same slot + // before the submit's tag15 names the pool; bind_pool publishes later. + let sma = j873_sma_device_words(registers.clusters, registers.cluster_width)?; + self.uma_pool.load_hardware_descriptor(dev, &mut self.uma, pool, sma)?; + let scheduler = self.scheduler.prepare(0, 1)?; + + // One fresh CL queue, first work ordinal 0, hardware stamp 1. Native + // event-machine conversion with 256 entries and base 1 yields 1 for + // event stamp 0x100; no previous completion is imported from macOS. + command.sequence = 1; + command.d100 = 1; + command.d7b8 = pool.state; + command.completion = G18ComputeCompletionState { + stamp: 0x100, index: u32::from(QUEUE_ID), entries: 256, + generation: 0, base: 1, + }; + registers.context_id = context_id; + registers.pool_slot = pool.slot; + // The passing fixture ran with a zero SMA result (0x10241 = slot). + // The pool binding generation is the separate FList field (tag15). + registers.sma_units = 0; + registers.d550 = scratch.register_pair[0]; + registers.d558 = scratch.register_pair[1]; + let prepared = self.compute.prepare(scheduler, &command, ®isters)?; + dev_info!(dev.as_ref(), + "G18 addition prepared: command={:#x}/{:#x} context={} scheduler={:#x} stamp={:#x} pool={}/{} registers={:?}; unpublished\n", + prepared.gpu_address, prepared.firmware_address, context_id, + scheduler.record, prepared.expected_stamp, pool.slot, pool.generation, + prepared.registers); + Ok((prepared, pool)) + } + + pub(crate) fn addition_output(&mut self) -> Result { + self.compute_payload.as_mut().ok_or(EINVAL)?.output_snapshot() + } + + pub(crate) fn triangle_output(&mut self) -> Result { + self.compute_payload.as_mut().ok_or(EINVAL)?.triangle_output_snapshot() + } + + pub(crate) fn prepare_uma_pool_binding(&mut self) -> Result { + self.uma_pool.prepare_binding() + } + + /// FW VA of one role's root, for the EP 0x20 0x81 doorbell. + pub(crate) fn root(&self, secondary: bool) -> u64 { + if secondary { self.secondary_root.iova() } else { self.primary_root.iova() } + } + + pub(crate) fn compute_outer_counters(&mut self) -> Result<[u32; 3]> { + let base = PRIVATE_BASE + (init::G18G_26A428_PRIVATE_STATE_GRIDS[0] + + init::WORK_STATE_GRID_OFFSETS[8]) as u64; + Ok([self.private.read_u32(base)?, self.private.read_u32(base + 0x10)?, + self.private.read_u32(base + 0x20)?]) + } + + /// G18 native 24-byte CL_2 record, limited to the registration and + /// scheduler-bind records. Each new producer is published exactly once. + pub(crate) fn publish_queue_probe(&mut self, address: u64, queue_id: u8, producer: u32) -> Result { + if !(1..=3).contains(&producer) { return Err(EINVAL); } + self.publish_compute_outer(address, queue_id, producer, producer) + } + + /// After the three setup records, publish one outer item covering the + /// command, context configuration and kick (inner producer 6). + pub(crate) fn publish_compute_queue(&mut self, address: u64, queue_id: u8) -> Result { + self.publish_compute_outer(address, queue_id, 4, 6) + } + + fn publish_compute_outer(&mut self, address: u64, queue_id: u8, + producer: u32, inner_producer: u32) -> Result { + if address < KERNEL_BASE || address & 0x3f != 0 || queue_id > 127 + || !(1..=4).contains(&producer) { + return Err(EINVAL); + } + if self.compute_outer_counters()? != [producer - 1; 3] + || self.control_counters()? != [[2; 3], [1; 3]] { + return Err(EBUSY); + } + self.shared.with_bytes_mut(|out| { + let start = init::G18G_26A428_WORK_RING_BUNDLE_OFFSETS[8] + (producer as usize - 1) * 24; + let record = out.get_mut(start..start + 24).ok_or(EINVAL)?; + if record.iter().any(|byte| *byte != 0) { return Err(EBUSY); } + record[8..16].copy_from_slice(&address.to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&(inner_producer as u16).to_le_bytes()); + record[22] = queue_id; + record[23] = 1; + Ok(()) + })?; + fence(Ordering::SeqCst); + let producer_address = PRIVATE_BASE + (init::G18G_26A428_PRIVATE_STATE_GRIDS[0] + + init::WORK_STATE_GRID_OFFSETS[8] + 0x20) as u64; + self.private.write_u32(producer_address, producer)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// The live device-control counters (consumer, CFI, producer) of both + /// roles, read from the shared state the firmware consumes. + pub(crate) fn control_counters(&mut self) -> Result<[[u32; 3]; 2]> { + self.private.with_bytes_mut(|bytes| { + let mut counters = [[0; 3]; 2]; + for (role, grid) in init::G18G_26A428_PRIVATE_STATE_GRIDS.into_iter().enumerate() { + let base = grid + init::WORK_STATE_GRID_OFFSETS[init::CONTROL_CHANNEL_INDEX]; + for (index, value) in counters[role].iter_mut().enumerate() { + let offset = base + index * init::CHANNEL_STATE_SPACING; + let word = bytes.get(offset..offset + 4).ok_or(EINVAL)?; + // SAFETY: the grid offsets are u32 aligned inside this + // bounded WC mapping, which is retained while firmware runs. + *value = unsafe { core::ptr::read_volatile(word.as_ptr().cast::()) }; + } + } + fence(Ordering::Acquire); + Ok(counters) + }) + } + + /// Stage device-control record 1 of role 0, IdlePowerOff(0) (opcode 0x0a, + /// FW_PROVEN unchanged: b000 0x4ef0), and publish it. Only valid after + /// both opening records retired, i.e. every counter reads 1. + pub(crate) fn publish_idle_wake(&mut self) -> Result { + if self.control_counters()? != [[1; 3]; 2] { + return Err(EBUSY); + } + let record = init::G18G_26A428_CONTROL_RING_BUNDLE_OFFSETS[0] + init::CONTROL_RECORD_SIZE; + self.shared.with_bytes_mut(|out| { + let record = out.get_mut(record..record + init::CONTROL_RECORD_SIZE).ok_or(EINVAL)?; + if record.iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + init::encode_device_control_record( + init::G18G_26A428_DEVICE_CONTROL_OPCODE_IDLE_POWER_OFF, + init::DEVICE_CONTROL_IDLE_POWER_OFF_ARG_DISABLE_IDLE_AND_WAKE, + record, + ) + .map_err(|_| EINVAL) + })?; + fence(Ordering::SeqCst); + let producer = init::G18G_26A428_PRIVATE_STATE_GRIDS[0] + + init::WORK_STATE_GRID_OFFSETS[init::CONTROL_CHANNEL_INDEX] + + init::CHANNEL_STATE_PRODUCER * init::CHANNEL_STATE_SPACING; + self.private.write_u32(PRIVATE_BASE + producer as u64, init::CONTROL_OPENING_PRIMARY_PRODUCER + 1)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// The four firmware-owned KSM completion cursors (hw_data +0x2734 + + /// ordinal * 0x20 + 0x1c; FW_PROVEN b000 0x1cbc0..0x1cc48). + pub(crate) fn completion_cursors(&mut self) -> Result<[u32; init::M5_25G76_COMPLETION_QUEUE_COUNT]> { + let mut cursors = [0; init::M5_25G76_COMPLETION_QUEUE_COUNT]; + for (ordinal, cursor) in cursors.iter_mut().enumerate() { + *cursor = self.shared.read_u32( + SHARED_BASE + + (init::G18G_26A428_KSM_COMPLETION_TABLE_OFFSET + + ordinal * init::G17P_KSM_COMPLETION_DESCRIPTOR_STRIDE + + init::G18G_26A428_KSM_COMPLETION_CURSOR) as u64, + )?; + } + Ok(cursors) + } + + /// Current recovery handshake words. Host DRAM only; no SGX MMIO. + pub(crate) fn recovery_word(&mut self) -> Result { + let status_b = PRIVATE_BASE + init::G18G_26A428_PRIVATE_LAYOUT.status_b as u64; + fence(Ordering::Acquire); + Ok(G18RecoveryWord { + epoch: self.private.read_u64(status_b + init::G18G_26A428_STATUS_B_RECOVERY_EPOCH as u64)?, + state: self.private.read_u32(status_b + init::G18G_26A428_STATUS_B_RECOVERY_STATE as u64)?, + host: self.private.read_u32(status_b + init::G18G_26A428_STATUS_B_RECOVERY_HOST as u64)?, + }) + } + + /// The cause and info the firmware published for the current recovery. + /// Like macOS, count entries by their valid bit rather than trusting the + /// emitted count. + pub(crate) fn recovery_cause(&mut self) -> Result { + let status_b = PRIVATE_BASE + init::G18G_26A428_PRIVATE_LAYOUT.status_b as u64; + fence(Ordering::Acquire); + let word = |memory: &mut Self, offset: usize| memory.private.read_u32(status_b + offset as u64); + let mut cause = G18RecoveryCause { + emitted: word(self, init::G18G_26A428_STATUS_B_RECOVERY_EMITTED)?, + source_count: word(self, init::G18G_26A428_STATUS_B_RECOVERY_SOURCE_COUNT)?, + command: word(self, init::G18G_26A428_STATUS_B_RECOVERY_COMMAND)?, + reason: word(self, init::G18G_26A428_STATUS_B_RECOVERY_REASON)?, + detail: self.private.read_u64(status_b + init::G18G_26A428_STATUS_B_RECOVERY_DETAIL as u64)?, + slot_mask: self.private.read_u64(status_b + init::G18G_26A428_STATUS_B_RECOVERY_SLOT_MASK as u64)?, + ..Default::default() + }; + for index in 0..init::G18G_26A428_STATUS_B_RECOVERY_INFO_ENTRIES { + let entry = self.private.read_u64(status_b + + (init::G18G_26A428_STATUS_B_RECOVERY_INFO + index * 8) as u64)?; + if entry & 1 != 0 { + if let Some(slot) = cause.pending.get_mut(cause.valid_entries as usize) { + *slot = (index as u16, entry); + } + cause.valid_entries += 1; + cause.first_entry.get_or_insert((index as u32, entry)); + } + } + let request = SHARED_BASE + init::G18G_26A428_SHARED_LAYOUT.block_68 as u64; + for (index, value) in cause.request.iter_mut().enumerate() { + *value = self.shared.read_u32(request + (index * 4) as u64)?; + } + Ok(cause) + } + + /// One QID's last-submitted entry: (valid, stamp). + pub(crate) fn queue_progress(&mut self, qid: usize) -> Result<(u32, u64)> { + if qid >= init::G18G_26A428_QUEUE_PROGRESS_ENTRIES { + return Err(EINVAL); + } + let entry = SHARED_BASE + (init::G18G_26A428_SHARED_LAYOUT.block_800 + + qid * init::G18G_26A428_QUEUE_PROGRESS_STRIDE) as u64; + fence(Ordering::Acquire); + Ok((self.shared.read_u32(entry)?, self.shared.read_u64(entry + 8)?)) + } + + /// Publish one QID's last-submitted entry for the firmware's restart. + pub(crate) fn publish_queue_progress(&mut self, qid: usize, valid: u32, stamp: u64) -> Result { + if qid >= init::G18G_26A428_QUEUE_PROGRESS_ENTRIES { + return Err(EINVAL); + } + let entry = SHARED_BASE + (init::G18G_26A428_SHARED_LAYOUT.block_800 + + qid * init::G18G_26A428_QUEUE_PROGRESS_STRIDE) as u64; + self.shared.write_u32(entry, valid)?; + self.shared.write_u32(entry + 4, 0)?; + self.shared.write_u32(entry + 8, stamp as u32)?; + self.shared.write_u32(entry + 12, (stamp >> 32) as u32)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Take the host's half of one handshake edge: 1 -> 2 lets the halted + /// firmware recover, 3 -> 0 is the final clear. Any other edge, or a + /// state other than `expected`, is refused without writing. + pub(crate) fn acknowledge_recovery(&mut self, expected: u32, next: u32) -> Result { + if !matches!((expected, next), (1, 2) | (3, 0)) { + return Err(EINVAL); + } + let state = PRIVATE_BASE + (init::G18G_26A428_PRIVATE_LAYOUT.status_b + + init::G18G_26A428_STATUS_B_RECOVERY_STATE) as u64; + fence(Ordering::Acquire); + if self.private.read_u32(state)? != expected { + return Err(EBUSY); + } + fence(Ordering::SeqCst); + self.private.write_u32(state, next)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Read back the new status cross-links from the host mapping (plan step + /// 1.3 validation after the 0x81 ACKs). + pub(crate) fn status_links(&mut self) -> Result { + let layout = init::G18G_26A428_PRIVATE_LAYOUT; + let status_b = PRIVATE_BASE + layout.status_b as u64; + let config = PRIVATE_BASE + layout.status_config as u64; + Ok(G18StatusLinks { + status_b_header: self.private.read_u32(status_b + init::G18G_26A428_STATUS_B_CONFIG_HEADER as u64)?, + status_b_word: self.private.read_u32(status_b + init::G18G_26A428_STATUS_B_LINK_WORD as u64)?, + status_b_pointer: self.private.read_u64(status_b + init::G18G_26A428_STATUS_B_LINK_PTR as u64)?, + status_config_word: self.private.read_u32(config + init::G18G_26A428_STATUS_CONFIG_LINK_WORD as u64)?, + status_config_pointer: self.private.read_u64(config + init::G18G_26A428_STATUS_CONFIG_LINK_PTR as u64)?, + }) + } + + /// Drain both roles' event (channel 13) and trace (channel 14) report + /// rings. Records are copied before their slot is released; each drain is + /// bounded. Returns whether an event record of the J813 recovery type was + /// seen. This is the milestone-1 heartbeat (plan step 1.8, stage 4). + pub(crate) fn drain_reports(&mut self, dev: &AsahiDevice) -> Result { + let mut recovery = false; + for (role, status_a) in init::G18G_26A428_PRIVATE_STATUS_A.into_iter().enumerate() { + for channel in 0..2 { + let [cursor_offset, ring_offset, _] = init::REPORT_STATE_STATUS_A_OFFSETS[channel]; + let capacity = REPORT_CAPACITY[channel]; + let cursor = PRIVATE_BASE + (status_a + cursor_offset) as u64; + let ring = status_a + ring_offset; + for _ in 0..REPORT_DRAIN_BUDGET { + let consumer = self.private.read_u32(cursor)?; + let producer = self.private.read_u32(cursor + REPORT_PRODUCER_OFFSET)?; + if consumer >= capacity || producer >= capacity { + return Err(EIO); + } + if consumer == producer { + break; + } + fence(Ordering::Acquire); + let offset = ring + consumer as usize * REPORT_RECORD_SIZE; + let mut raw = [0u8; REPORT_RECORD_SIZE]; + self.private.with_bytes_mut(|bytes| { + raw.copy_from_slice(bytes.get(offset..offset + REPORT_RECORD_SIZE).ok_or(EINVAL)?); + Ok(()) + })?; + let word = |off: usize| u32::from_le_bytes([raw[off], raw[off + 1], raw[off + 2], raw[off + 3]]); + let qword = |off: usize| u64::from(word(off)) | (u64::from(word(off + 4)) << 32); + let count = self.report_counts[role][channel]; + if channel == 1 { + // Preserve all bytes before firmware may reuse this slot. + // No allocation, MMIO, or extra logging in the hot path. + if self.trace_journal[role].len() >= TRACE_JOURNAL_CAPACITY + || self.trace_journal[role] + .push_within_capacity(G18TraceRecord { ordinal: count, raw }) + .is_err() + { + self.trace_dropped[role] = self.trace_dropped[role].saturating_add(1); + } + } + if channel == 0 { + crate::g18_event_info!(role == 0 && word(0) == 1, dev.as_ref(), "G18_EVENT role={} ordinal={} type={} payload={:x?}\n", + role, count, word(0), [qword(4), qword(12), qword(20), qword(28), qword(36)]); + recovery |= word(0) == EVENT_TYPE_RECOVERY; + if role == 0 && word(0) == 1 { + // Original G18 b000 233a8..233d4 uses this exact + // type-1 prefix. Copy/validate before releasing the + // slot; the remaining bytes are not completion proof. + let event = crate::g18::fw::events::decode_g17p_firmware_stamp_event(&raw) + .map_err(|_| EIO)?; + self.completion_masks[0] |= event.masks[0]; + self.completion_masks[1] |= event.masks[1]; + } + } else if self.trace_printed[role] < TRACE_PRINT_BUDGET[role] { + dev_info!(dev.as_ref(), + "G18_TRACE role={} ordinal={} timestamp={} type={} id={:#x} args={:x?}\n", + role, count, qword(4), word(0), word(0x2c), + [qword(12), qword(20), qword(28), qword(36)]); + self.trace_printed[role] += 1; + } + self.report_counts[role][channel] = count.saturating_add(1); + fence(Ordering::SeqCst); + self.private.write_u32(cursor, (consumer + 1) % capacity)?; + } + } + } + Ok(recovery) + } + + /// Report-record totals per role and channel since construction. + pub(crate) fn report_counts(&self) -> [[u32; 2]; 2] { + self.report_counts + } + + /// Only primary firmware type-1 records contribute to these fresh-boot bits. + pub(crate) fn completion_masks(&self) -> [u64; 2] { + self.completion_masks + } + + /// Export after the transaction and stop requests, never while polling a + /// GPU job. Completeness covers consumed records, not a possibly live producer. + pub(crate) fn log_trace_journal(&self, dev: &AsahiDevice) { + for role in 0..2 { + let journal = &self.trace_journal[role]; + let consumed = self.report_counts[role][1]; + dev_info!(dev.as_ref(), + "G18_TRACE_JOURNAL_BEGIN role={} consumed={} stored={} dropped={} capacity={}\n", + role, consumed, journal.len(), self.trace_dropped[role], TRACE_JOURNAL_CAPACITY); + for record in journal.iter() { + let words: [u64; 9] = core::array::from_fn(|index| { + let mut bytes = [0; 8]; + bytes.copy_from_slice(&record.raw[index * 8..index * 8 + 8]); + u64::from_le_bytes(bytes) + }); + dev_info!(dev.as_ref(), "G18_TRACE_JOURNAL role={} ordinal={} words={:016x?}\n", + role, record.ordinal, words); + } + dev_info!(dev.as_ref(), + "G18_TRACE_JOURNAL_END role={} consumed={} stored={} dropped={}\n", + role, consumed, journal.len(), self.trace_dropped[role]); + } + } + + /// One-line construction summary for the bring-up log. + pub(crate) fn log_summary(&self, dev: &AsahiDevice) { + dev_info!(dev.as_ref(), + "G18G initdata: roots {:#x}/{:#x} version {:#x}; shared {:#x}+{:#x}; private {:#x}+{:#x}; {} register windows; {} unproven fields\n", + self.root(false), self.root(true), init::G18G_26A428_INITDATA_VERSION_WORD, + SHARED_BASE, init::G18G_26A428_SHARED_CLUSTER_SIZE, + PRIVATE_BASE, init::G18G_26A428_PRIVATE_CLUSTER_SIZE, + self.registers.len(), init::G18G_26A428_UNPROVEN.len()); + } +} diff --git a/drivers/gpu/drm/asahi/g18/mod.rs b/drivers/gpu/drm/asahi/g18/mod.rs new file mode 100644 index 0000000000000..14317928821e2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/mod.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 (T8152, Mac mini M6 / J873) backend. +//! +//! Probe, firmware start and DRM registration follow the J873 module 1134 +//! path; the backend plugs into the shared driver through `gpu::Gpu` and +//! `queue::Queue`. + +#[allow(dead_code)] +pub(crate) mod address_map; +#[allow(dead_code)] +pub(crate) mod boot; +pub(crate) mod buffer; +pub(crate) mod compat; +#[allow(dead_code)] +pub(crate) mod drm; +pub(crate) mod fixture; +pub(crate) mod fw; +#[allow(dead_code)] +pub(crate) mod initdata; +pub(crate) mod queue; +pub(crate) mod runtime; +pub(crate) mod uapi; +#[allow(dead_code)] +pub(crate) mod debug; +#[allow(dead_code)] +pub(crate) mod hw; +#[allow(dead_code)] +pub(crate) mod identity; +#[allow(dead_code)] +pub(crate) mod probe; +#[allow(dead_code)] +pub(crate) mod regs; diff --git a/drivers/gpu/drm/asahi/g18/probe.rs b/drivers/gpu/drm/asahi/g18/probe.rs new file mode 100644 index 0000000000000..9bae60ce5a16d --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/probe.rs @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! T8152 probe: parameter gates, device preparation and the stage-10 DRM path. +//! +//! Items copied from the J873 module-1134 driver.rs (provenance in +//! the port's extraction ledger), then adapted to +//! this driver's core: the DRM device is allocated with +//! `Device::new_uninit` and its data published with `init_data` (module 1134 +//! used a no-op initializer the release path skipped), `AsahiData` is this +//! driver's (no G17 work items, no SGX resources) and the G18 adapter is a +//! `gpu::Gpu`. The parameter gates and every log line are unchanged. + +use kernel::{ + c_str, + device::Core, + dma::{Device, DmaMask}, + drm, + of, + platform, + prelude::*, + sync::{aref::ARef, Arc}, +}; + +use crate::g18::boot as g18_live_boot; +use crate::g18::{hw, identity, regs}; +use crate::{ + driver::{AsahiData, AsahiDevice}, + gpu, +}; + +/// Owner object returned by the G18 live-boot module. +/// +/// The live-boot module implements this for its session type. The driver +/// keeps it for the device's lifetime and calls [`G18Runtime::shutdown`] on +/// unbind, before devres releases the SGX mapping and the mailbox links. +pub(crate) trait G18Runtime: Send + Sync { + /// Quiesce the firmware, both ASC processors and every host mapping. + fn shutdown(&self); +} + +/// Everything the T8152 probe gate hands to the G18 live-boot entry point. +/// +/// Built only by [`prepare_t8152_device`], and only when `g18_initdata_test` +/// is non-zero. Holding one means: +/// - the three GPU PSD services (sgx 3, gfx-asc 81, gfx1-asc 82) were read +/// once, read-only, and their samples are in [`T8152Probe::psd`]; +/// - the SGX window is mapped in `registers`. Its ID block was read only if +/// the sgx PSD sample proved the block powered: `observed` is `Some` +/// exactly then; +/// - for the firmware stages 2..4 the loader's G18G firmware receipt matched +/// and the 42-bit DMA mask is set. Stages 1 and 90 require neither. +/// +/// No firmware, ASC CPU, PSD request, DMA or UAT state has been touched. +#[allow(dead_code)] +pub(crate) struct T8152Probe { + /// The matched SoC entry (`hw::agx3::T8152`). + pub(crate) soc: &'static hw::agx3::SocConfig, + /// Static identity decoded from the SoC entry (G18G, HAL400, SKSM). + pub(crate) identity: identity::GpuIdentity, + /// The `g18_initdata_test` stage, always non-zero here. + pub(crate) stage: u32, + /// Whether /chosen `aurora,g18g-gpu-firmware-profile` matched. + pub(crate) firmware_profile_admitted: bool, + /// Read-only samples of PSD services 3, 81 and 82, in that order. + pub(crate) psd: [regs::T8152PsdSample; 3], + /// SGX mapping. Every further SGX read must again follow a powered + /// `regs::t8152_prove_sgx_powered()` result. + pub(crate) registers: regs::Resources, + /// Hardware identity, only when the SGX PSD proved the block powered. + pub(crate) observed: Option, + /// Unregistered DRM device whose data slot is still uninitialized, as + /// on M5. The live-boot module initializes it before any registration. + pub(crate) drm: ARef, +} + +/// Loader receipt the G18G firmware stages require. The loader writes it +/// only after hashing both loaded TEXT images (b000 GFX 0x50000, b010 GFX1 +/// 0x54000) against the 26A428 profile (plan step 0.2). +const G18G_FIRMWARE_PROFILE: &[u8] = b"G18G-RTKit-3514.0.15.release\0"; + +/// T8152 probe gate. With `g18_initdata_test=0` (the default) this is a +/// complete no-op: it logs and refuses with ENODEV before any MMIO, DMA +/// mask, reserved-memory, mailbox or processor access, just as M5 refuses by +/// default. Unlike the M5 default it also skips the read-only SGX identity +/// probe, because J873 cold-boots with the GPU PSD services idle, where an +/// SGX read can raise an SError. +pub(crate) fn probe_t8152( + pdev: &platform::Device, + soc: &'static hw::agx3::SocConfig, +) -> Result> { + let stage = *crate::module_parameters::g18_initdata_test.value(); + if stage == 0 { + dev_info!( + pdev.as_ref(), + "G18G: T8152 matched; g18_initdata_test=0 refuses before any MMIO, DMA, mailbox or processor access\n" + ); + return Err(ENODEV); + } + if !g18_live_boot::stage_is_known(stage) { + dev_err!( + pdev.as_ref(), + "G18G: g18_initdata_test={} is not a known stage (1-9, 90-93); refused before any MMIO\n", + stage + ); + return Err(EINVAL); + } + let variant = *crate::module_parameters::g18_compute_variant.value(); + if variant > 2 || (variant != 0 && stage != g18_live_boot::STAGE_COMPUTE_ADD) { + dev_err!( + pdev.as_ref(), + "G18G: g18_compute_variant={} needs 0, or 1/2 with g18_initdata_test=8; refused before any MMIO\n", + variant + ); + return Err(EINVAL); + } + let render_variant = *crate::module_parameters::g18_render_variant.value(); + let render_profile = *crate::module_parameters::g18_render_profile.value(); + let sma_minimum = *crate::module_parameters::g18_sma_minimum.value(); + let sma_ideal = *crate::module_parameters::g18_sma_ideal.value(); + let native_barrier = *crate::module_parameters::g18_native_barrier.value(); + if native_barrier > 1 || (native_barrier != 0 && (sma_ideal != 1 || sma_minimum != 0 || + stage != crate::g18::boot::render_stage::STAGE_RENDER || render_variant != 0 || render_profile != 4)) { + dev_err!(pdev.as_ref(), "G18G: native barrier probe requires ideal SMA stage9 variant0 profile4; refused before MMIO\n"); + return Err(EINVAL); + } + if sma_ideal > 1 || (sma_ideal != 0 && (sma_minimum != 0 || + stage != crate::g18::boot::render_stage::STAGE_RENDER || render_variant != 0 || render_profile != 4)) { + dev_err!(pdev.as_ref(), "G18G: SMA ideal probe requires stage9 variant0 profile4 and minimum=0; refused before MMIO\n"); + return Err(EINVAL); + } + // Explicit fixed-minimum comparison for the retained first application + // draw, as well as the original H1 witness. This is not dynamic sizing. + let minimum_probe = render_variant == 0 && + ((stage == crate::g18::boot::render_stage::STAGE_RENDER && render_profile == 4) || + (stage == g18_live_boot::STAGE_DRM && render_profile == 1)); + let stall_trace = *crate::module_parameters::g18_graphics_stall_trace.value(); + if stall_trace > 1 || (stall_trace != 0 && !minimum_probe) { + dev_err!(pdev.as_ref(), "G18G: graphics stall tracing requires stage9/profile4 or stage10/profile1, variant0; refused before MMIO\n"); + return Err(EINVAL); + } + let high_vdm = *crate::module_parameters::g18_vdm_high_alias.value(); + if high_vdm > 1 || (high_vdm != 0 && + (stage != crate::g18::boot::render_stage::STAGE_RENDER || render_variant != 0 || + render_profile != 4 || sma_minimum != 1 || sma_ideal != 0 || stall_trace != 1)) { + dev_err!(pdev.as_ref(), "G18G: high VDM alias requires stage9/profile4/variant0, minimum SMA and stall tracing; refused before MMIO\n"); + return Err(EINVAL); + } + let high_ppp = *crate::module_parameters::g18_ppp_high_alias.value(); + if high_ppp > 1 || (high_ppp != 0 && high_vdm != 1) { + dev_err!(pdev.as_ref(), "G18G: high PPP aliases require the bounded high VDM control; refused before MMIO\n"); + return Err(EINVAL); + } + if sma_minimum > 1 || (sma_minimum != 0 && !minimum_probe) { + dev_err!(pdev.as_ref(), "G18G: SMA minimum probe requires stage9/profile4 or stage10/profile1, variant0; refused before MMIO\n"); + return Err(EINVAL); + } + let usc_statistics = *crate::module_parameters::g18_usc_statistics.value(); + if usc_statistics > 1 || (usc_statistics != 0 && + (stage != crate::g18::boot::render_stage::STAGE_RENDER || render_variant != 0)) { + dev_err!(pdev.as_ref(), "G18G: USC statistics require the stage9 colour fixture; refused before MMIO\n"); + return Err(EINVAL); + } + if stage == crate::g18::boot::render_stage::STAGE_RENDER { + crate::g18::boot::render_stage::admit(pdev.as_ref(), render_variant, render_profile)?; + } else if render_variant != 0 || render_profile != 1 { + return Err(EINVAL); + } + let admit_msaa4 = *crate::module_parameters::g18_admit_msaa4.value(); + if admit_msaa4 > 1 || (admit_msaa4 != 0 && stage != g18_live_boot::STAGE_DRM) { + dev_err!(pdev.as_ref(), "G18G: g18_admit_msaa4={} needs 0, or 1 with g18_initdata_test=10; refused before MMIO\n", admit_msaa4); + return Err(EINVAL); + } + if stage == g18_live_boot::STAGE_DRM { + if *crate::module_parameters::g18_admit_compute.value() > 1 {return Err(EINVAL);} + return probe_t8152_drm(pdev, soc, stage); + } + let probe = prepare_t8152_device(pdev, soc, stage)?; + g18_live_boot_start(pdev, probe) +} + +/// Register the G18 DRM render node over a started G18 session. +fn probe_t8152_drm( + pdev: &platform::Device, + soc: &'static hw::agx3::SocConfig, + stage: u32, +) -> Result> { + let mut probe = prepare_t8152_device(pdev, soc, stage)?; + let drm = probe.drm.clone(); + let observed = probe.observed.take(); + let (owner, runtime) = g18_live_boot::start_drm(pdev, probe)?; + let config = crate::g18::drm::G18DrmConfig { + admit_compute: *crate::module_parameters::g18_admit_compute.value() != 0, + admit_msaa4: *crate::module_parameters::g18_admit_msaa4.value() != 0, + admit_graphics_scratch: + *crate::module_parameters::g18_admit_graphics_scratch.value() != 0, + }; + let gpu: Arc = Arc::new( + crate::g18::drm::G18DrmGpu::new(runtime, observed.as_ref(), config)?, + GFP_KERNEL, + )?; + let data = try_pin_init!(AsahiData { gpu, pdev: pdev.into(), resources: None }); + // SAFETY: prepare_t8152_device leaves this data slot uninitialized. This + // is its sole initialization, before registration exposes file operations. + unsafe { drm.init_data(data)? }; + (*drm).gpu.init()?; + crate::g18::drm::note_g18_parent(pdev.as_ref().as_raw()); + drm::driver::Registration::new_foreign_owned(&drm, pdev.as_ref(), 0)?; + dev_info!(pdev.as_ref(), "G18_DRM registered render node (asahi, apple,agx-t8152)\n"); + Ok(owner) +} + +fn prepare_t8152_device( + pdev: &platform::Device, + soc: &'static hw::agx3::SocConfig, + stage: u32, +) -> Result { + let identity = + identity::decode_gpu_identity(soc.hw_family, soc.hw_variant, soc.num_dies as u8) + .ok_or(EINVAL)?; + if soc.chip_id != 0x8152 + || identity.gpu_gen != hw::GpuGen::G18 + || identity.gpu_variant != identity::GpuVariant::G + || identity.uat_input_address_bits != soc.uat_ias + { + return Err(EINVAL); + } + dev_info!( + pdev.as_ref(), + "G18G: T8152 stage {}: {:?}{:?} USC gen {} HAL {:?} {:?} roles {:?} submission, {}-bit UAT roots\n", + stage, + identity.gpu_gen, + identity.gpu_variant, + identity.usc_generation, + identity.gpu_hal_generation, + identity.firmware_roles, + identity.submission_transport, + soc.uat_ias + ); + + let firmware_profile_admitted = match of::chosen() { + Some(chosen) => { + let profile: Result> = + chosen.get_property(c_str!("aurora,g18g-gpu-firmware-profile")); + matches!(profile, Ok(profile) if profile.as_slice() == G18G_FIRMWARE_PROFILE) + } + None => false, + }; + dev_info!( + pdev.as_ref(), + "G18G: firmware receipt {}\n", + (if firmware_profile_admitted { "admitted" } else { "absent or different" }) + ); + if g18_live_boot::stage_runs_firmware(stage) && !firmware_profile_admitted { + dev_err!( + pdev.as_ref(), + "G18G: stage {} needs /chosen aurora,g18g-gpu-firmware-profile = G18G-RTKit-3514.0.15.release\n", + stage + ); + return Err(ENODEV); + } + + // PSD state first, read-only. Nothing below reads SGX unless the sgx + // service proves the block active and acknowledged. + let mut psd = [regs::T8152PsdSample { + service: regs::T8152PsdService::Sgx, + state: 0, + ack: 0, + }; 3]; + for (slot, service) in psd.iter_mut().zip(regs::T8152PsdService::ALL) { + *slot = regs::t8152_read_psd(pdev, service)?; + dev_info!( + pdev.as_ref(), + "G18G: PSD service {} ({:#x}) state={:#x} power state {} ack={:#x} -> {}\n", + service.service_id(), + service.physical_base(), + slot.state, + slot.power_state(), + slot.ack, + (if slot.powered() { "powered" } else { "not powered" }) + ); + } + + let registers = regs::Resources::new(pdev)?; + let observed = match regs::t8152_prove_sgx_powered(pdev) { + Ok(powered) => Some(registers.get_gpu_id_t8152(&powered, soc)?), + Err(error) if error == regs::t8152_not_powered() => { + dev_info!( + pdev.as_ref(), + "G18G: SGX identity not read (sgx PSD idle); identified by chip id {:#x} and compatible\n", + soc.chip_id + ); + None + } + Err(error) => return Err(error), + }; + + if g18_live_boot::stage_runs_firmware(stage) { + // J873 DRAM ends at 0x106_0000_0000, inside a 42-bit output space. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(soc.uat_oas)?)? }; + } + + // This device is deliberately unregistered and its data slot stays + // uninitialized until the stage-10 DRM path calls `init_data`; the DRM + // release path drops only initialized data. + // SAFETY: nothing reads the data slot before `init_data`. + let drm: ARef = unsafe { drm::device::Device::new_uninit(pdev.as_ref())? }; + Ok(T8152Probe { + soc, + identity, + stage, + firmware_profile_admitted, + psd, + registers, + observed, + drm, + }) +} + +/// G18 live-boot entry point: the staged bring-up in `g18_live_boot.rs`. +/// +/// `probe.stage` is `g18_initdata_test`, already checked by +/// [`g18_live_boot::stage_is_known`]: +/// - 1: read-only probe (PSD, SGX ID, wrapper CPU state, post-office +/// status), then ENODEV; +/// - 2, 3, 4, 5, 6 (REQUIRE USER APPROVAL, plus the mailbox provider's own +/// `apple_mailbox.ascwrap_v8_cpu_control=1`): ASC CPU start and RTKit, +/// init-data ACKs, openings + idle wake + heartbeat. Stages 2 and 3 end in +/// ENODEV with every firmware owner retained; stages 4/5/6 return the owner. +/// Stage 5 registers one empty queue; stage 6 binds its scheduler. No GPU jobs; +/// - 90 (REQUIRES USER APPROVAL): PMGR2 PSD power-on only, then ENODEV. +/// +/// On success the driver keeps the owner and calls +/// [`G18Runtime::shutdown`] on unbind. +fn g18_live_boot_start( + pdev: &platform::Device, + probe: T8152Probe, +) -> Result> { + g18_live_boot::start(pdev, probe) +} diff --git a/drivers/gpu/drm/asahi/g18/queue/graphics.rs b/drivers/gpu/drm/asahi/g18/queue/graphics.rs new file mode 100644 index 0000000000000..9a7b53295190a --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/queue/graphics.rs @@ -0,0 +1,1077 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! J873 (26A428) TA/3D queue registration, binding and one render pass. +//! +//! Two SKSM kick queues, TA qid 9 (stage 0) and 3D qid 10 (stage 1), follow +//! the proven compute protocol of g18_queue_probe.rs (qid 11) step by step: +//! install (inner producer 1), scheduler bind (2), UMA pool + PB bind (3), +//! then one render pass [command, context config, kick] (6). Both queues share +//! ONE scheduler control record and ONE job list (M5 "Shared graphics +//! completion list"), and every inner command is encoded by +//! g18_render_command.rs, whose bytes equal the original TA/3D submitBuffer. +//! +//! Firmware references (GFX b000, SHA 2c63b343...): tag15 handler 0x12824 +//! (0x98 bytes), queue slot table 0xb90b8 + qid*0x28 (stage word at +0x20), +//! scheduler table 0xb7468 + qid*8, UMA pool slots 0x5f660 + slot*0x100, +//! per-queue pool table 0x6f660 + qid*0x38, PB slot table 0x71420 + slot*0x50. +//! +//! Outer CL_2 records (24 bytes each) are returned, not published: the +//! firmware-memory owner (g18_resources.rs) owns the CL_2 ring and producer. +//! Addresses come from the address map; nothing here chooses a VA. + +#![allow(dead_code)] + +#[cfg(not(test))] +#[path = "graphics_application.rs"] +mod application; + +use crate::g18::fw::render_command::{self as cmd, ChannelForm, G18ChannelConfig, G18ConfigUma, + G18ConfigParameterBuffer, G18RenderStage, G18_CHANNEL_CONFIG_BYTES, G18_KICK_BYTES}; + +pub(crate) const G18_GRAPHICS_QUEUE_IDS: [u8; 2] = [9, 10]; +pub(crate) const G18_GRAPHICS_STAGES: [G18RenderStage; 2] = [G18RenderStage::Ta, G18RenderStage::ThreeD]; +pub(crate) const G18_ENTRY_RING_BYTES: usize = 0x20000; // b000 0x1257c: 256 * 0x200 +pub(crate) const G18_GRAPH_BYTES: usize = 0x8000; +pub(crate) const G18_OUTER_RECORD_BYTES: usize = 24; +const PRIORITY: u32 = 2; +/// AGXChannel::getSubPriority value the compute path proved (config +0x5e). +const SUB_PRIORITY: u32 = 2; + +// Graph layout (one firmware-private object; offsets are this module's own). +const CHANNEL: [usize; 2] = [0x0, 0x100]; // 0x80-byte M6 channel state +const POINTERS: [usize; 2] = [0x200, 0x300]; // pointer state (+0 consumer, +0x40 producer) +const JOB_LIST: usize = 0x400; // shared scheduler job list +const CONTROL: usize = 0x500; // shared scheduler control (0x38) +const ITEMS: [usize; 2] = [0x800, 0x1000]; // u64 command-pointer rings +const CAPACITY: u32 = 0x100; +const CONFIGS: [usize; 2] = [0x1800, 0x2000]; // install/bind/pool/draw tag15 at 0x100 stride +const KICKS: [usize; 2] = [0x2800, 0x2c00]; // 0x40 stride + +const _: () = assert!(ITEMS[1] + CAPACITY as usize * 8 <= CONFIGS[0]); +const _: () = assert!(CONFIGS[1] + 0x400 <= KICKS[0] && KICKS[1] + 0x400 <= G18_GRAPH_BYTES); + +/// Addresses from the address map (see the G18 render-command notes, section 5). +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18GraphicsQueueAddresses { + /// SKSM entry rings: firmware VA (kernel_vm, PRIV_RW) and the kernel low + /// alias (kernel_lower_vm, GPU_SHARED_RW) of the same pages, per stage. + pub entry_high: [u64; 2], + pub entry_low: [u64; 2], + /// Queue graph, firmware VA (kernel_vm, PRIV_RW), G18_GRAPH_BYTES. + pub graph: u64, +} + +/// One 24-byte outer CL_2 record: b000 walkers 0x274cc..0x27598 read +8 +/// queue address, +0x10 ring DM (2), +0x14 inner producer, +0x16 qid, +0x17 +/// cache flag. Both stage queues travel on the CL_2 ring. +pub(crate) fn encode_g18_outer_record(queue_address: u64, queue_id: u8, inner_head: u16) + -> [u8; G18_OUTER_RECORD_BYTES] { + let mut record = [0; G18_OUTER_RECORD_BYTES]; + record[8..16].copy_from_slice(&queue_address.to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&inner_head.to_le_bytes()); + record[22] = queue_id; + record[23] = 1; + record +} + +/// Everything one render pass needs beyond the queues themselves. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18QueuedPass { + /// TA and 3D command firmware VAs (SKSM payload[0], inner item 0). + pub command_high: [u64; 2], + pub context_id: u32, + pub scheduler_control: u64, + pub scheduler_index: u32, + pub uma: G18ConfigUma, + /// TA only; `None` disables the PB tuple (3D never carries one). + pub parameter_buffer: Option, + /// RCE lists returned by encode_g18_ta_command / encode_g18_3d_command. + pub rce: [[m5_rce::M5SksmRce; 4]; 2], + /// Event stamp of each stage (0x100 per generation) and the queues' + /// event-machine generation for it. + pub stamps: [u32; 2], + pub event_generation: [u64; 2], + /// TA barrier record flag for the 3D's dependency (0: kind 1, the + /// M5-proven choice; 1: HAL400 3D dependency type 3). + pub ta_barrier_flag: u32, +} + +use crate::g18::fw::sksm as m5_rce; + +/// Kick-queue state the kernel mirrors (native queue +0x48/+0x98/+0xa0/+0x90). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct G18KickQueueState { + pub current_stamp: u64, + pub previous_stamp: Option, + pub base: u64, + /// Passes submitted on this queue (host bookkeeping only). + pub submitted_kicks: u16, +} + +/// Build one stage's SKSM entry and kick for a pass. Pure; the owner stores +/// them. `queue_address` is the stage's channel state (SKSM payload[1], the +/// native channel +0x80). +pub(crate) fn g18_stage_kick( + stage: usize, + pass: &G18QueuedPass, + queue: G18KickQueueState, + queue_address: u64, +) -> Result<(m5_rce::PreparedM5SksmEntry, [u8; G18_KICK_BYTES]), cmd::G18RenderCommandError> { + let qid = G18_GRAPHICS_QUEUE_IDS[stage]; + let mut barriers = [cmd::G18Barrier { stamp: 0, queue_index: 0, flag: 0, generation: 0 }; 1]; + let barrier_count = if stage == 1 { + barriers[0] = cmd::G18Barrier { + stamp: pass.stamps[0], queue_index: G18_GRAPHICS_QUEUE_IDS[0], + flag: pass.ta_barrier_flag, generation: pass.event_generation[0], + }; + cmd::g18_order_3d_barriers(&mut barriers, u32::from(G18_GRAPHICS_QUEUE_IDS[0])); + 1 + } else { 0 }; + let entry = cmd::encode_g18_kick_entry(&cmd::G18KickEntryInputs { + stage: G18_GRAPHICS_STAGES[stage], + queue: m5_rce::M5SksmQueue { id: qid, priority: PRIORITY as u8, + current_stamp: queue.current_stamp, submitted: 0, previous_stamp: queue.previous_stamp }, + entries: 256, base: queue.base, stamp: pass.stamps[stage], + generation: pass.event_generation[stage], barriers: &barriers[..barrier_count], + pipeline_dependency: 0, scheduler_index: pass.scheduler_index as u8, qos_word: 0xffff, + payload: [pass.command_high[stage], queue_address], rce: pass.rce[stage], + flag_bit1: false, flag_bit61: false, flag_bit63: false, event_flags: 0, + auxiliary: 0x3f_ffff_ffff_ffff, + })?; + // Native kick count = this descriptor's new entries (NOPs + 1); Linux + // never needs NOPs because every stamp is consecutive (encodeNops 0). + let kick = cmd::encode_g18_kick(G18_GRAPHICS_STAGES[stage], qid, PRIORITY as u8, + queue.current_stamp, 1)?; + Ok((entry, kick)) +} + +/// The tag15 configuration for a stage and protocol step (compute-proven +/// staging: 0 install, 1 scheduler bind, 2 pool/PB bind, 3 pass context). +pub(crate) fn g18_stage_config( + stage: usize, + step: u32, + entry: (u64, u64), + pass: &G18QueuedPass, +) -> [u8; G18_CHANNEL_CONFIG_BYTES] { + let mut out = [0u8; G18_CHANNEL_CONFIG_BYTES]; + let bound = step >= 2; + let config = G18ChannelConfig { + ring_low: entry.0, ring_high: entry.1, queue_id: u16::from(G18_GRAPHICS_QUEUE_IDS[stage]), + install: step == 0, priority: PRIORITY, data_master: stage as u32, + state_update_flag: false, state_update: 0, context_id: pass.context_id, + uma: if bound { pass.uma } else { G18ConfigUma { state: 0, generation: None, slot: u32::MAX } }, + channel_control: pass.scheduler_control, update_scheduler: step == 1, + scheduler_index: 0, pid: 0, sub_priority: SUB_PRIORITY, + update_context: step == 3, gart_flag: step == 3, + extension_flag: false, extension: 0, + parameter_buffer: if bound && stage == 0 { pass.parameter_buffer } else { None }, + }; + // Infallible: the output is exactly one M6 record. + let _ = cmd::encode_channel_config(ChannelForm::M6, &config, &mut out); + out +} + +// ---- Kernel owner ------------------------------------------------------------- + +#[cfg(not(test))] +use crate::g18::buffer::object::MappedObject; +#[cfg(not(test))] +use crate::{driver::AsahiDevice, mmu}; +#[cfg(not(test))] +use core::sync::atomic::{fence, Ordering}; +#[cfg(not(test))] +use kernel::{c_str, io, prelude::*}; + +#[cfg(not(test))] +const DATA_SIZE: usize = 0xbc000; +#[cfg(not(test))] +const DATA_BASE: usize = 0x50000; +#[cfg(not(test))] +const SLOT_TABLE: usize = 0xb90b8 - DATA_BASE; +#[cfg(not(test))] +const SCHEDULER_TABLE: usize = 0xb7468 - DATA_BASE; +#[cfg(not(test))] +const POOL_TABLE: usize = 0x5f660 - DATA_BASE; +#[cfg(not(test))] +const QUEUE_POOL_TABLE: usize = 0x6f660 - DATA_BASE; +#[cfg(not(test))] +const PB_TABLE: usize = 0x71420 - DATA_BASE; + +/// Read-only view of reserved GFX DATA (same rules as g18_queue_probe.rs: +/// invalidate before reading, never clean firmware lines). +#[cfg(not(test))] +struct DataView { + map: io::mem::Mem, + cache_line: usize, +} +// SAFETY: bounded volatile reads of a fixed reserved mapping only. +#[cfg(not(test))] +unsafe impl Send for DataView {} +// SAFETY: concurrent reads cannot mutate host-owned state. +#[cfg(not(test))] +unsafe impl Sync for DataView {} + +#[cfg(not(test))] +impl DataView { + fn new(dev: &AsahiDevice) -> Result { + let resource = dev.as_ref().of_node().ok_or(EINVAL)? + .reserved_mem_region_to_resource_byname(c_str!("gfx-data"))?; + if resource.size() != DATA_SIZE as u64 || resource.start() < 0x100_0000_0000 + || resource.start() & 0x3fff != 0 + || resource.start().checked_add(resource.size()).ok_or(EOVERFLOW)? > 0x104_0000_0000 { + return Err(EINVAL); + } + let ctr: u64; + // SAFETY: architectural read-only cache geometry register. + unsafe { core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)) }; + let cache_line = 4usize << ((ctr >> 16) & 0xf); + if cache_line > 0x4000 { return Err(EINVAL); } + // SAFETY: WB matches reserved System RAM; the host never dirties it. + let map = unsafe { io::mem::Mem::try_new(resource, io::mem::MemFlag::WB.into()) }?; + Ok(Self { map, cache_line }) + } + + fn words(&self, offset: usize) -> Result<[u64; N]> { + let start = offset & !(self.cache_line - 1); + let end = (offset + N * 8 + self.cache_line - 1) & !(self.cache_line - 1); + if offset % 8 != 0 || end > DATA_SIZE { return Err(EINVAL); } + for line in (start..end).step_by(self.cache_line) { + // SAFETY: bounds checked above; invalidate only. + unsafe { core::arch::asm!("dc ivac, {addr}", + addr = in(reg) self.map.ptr().add(line), options(nostack, preserves_flags)) }; + } + // SAFETY: complete invalidation before observing firmware stores. + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; + let mut out = [0; N]; + for (index, word) in out.iter_mut().enumerate() { + // SAFETY: aligned reads inside the checked extent. + *word = unsafe { core::ptr::read_volatile( + self.map.ptr().add(offset + index * 8).cast::()) }; + } + fence(Ordering::Acquire); + Ok(out) + } +} + +/// Firmware-side observations for both queues. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18GraphicsFirmwareStatus { + /// Queue slot (5 words, +0x20 low half is the stage word) per stage. + pub slots: [[u64; 5]; 2], + /// Scheduler table entry per stage (the control pointer). + pub schedulers: [u64; 2], + /// Control header, membership words. + pub control: [u64; 3], +} + +#[cfg(not(test))] +pub(crate) struct G18GraphicsQueues { + aliases: KVec, + entries: KVec, + graph: MappedObject, + firmware: DataView, + addresses: G18GraphicsQueueAddresses, + step: u32, + queues: [G18KickQueueState; 2], +} + +#[cfg(not(test))] +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +#[cfg(not(test))] +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +#[cfg(not(test))] +impl G18GraphicsQueues { + /// Allocate both entry rings (dual-mapped, same pages proven) and the + /// graph; initialize channel/pointer states (AGXChannel::resetChannelState + /// 0xfffffe0008bdd178), the shared job list and the shared control + /// (AGXCommandQueueReclaimableState::init 0xfffffe0008bf9014). Nothing is + /// published: all inner producers stay 0. + pub(crate) fn new(dev: &AsahiDevice, uat: &mmu::Uat, a: G18GraphicsQueueAddresses) -> Result { + if !uat.is_t8152_geometry() { return Err(ENODEV); } + let mut entries = KVec::new(); + let mut aliases = KVec::new(); + for stage in 0..2 { + let mut entry = MappedObject::new_wc_at(dev, uat.kernel_vm(), a.entry_high[stage], + G18_ENTRY_RING_BYTES, mmu::PROT_FW_PRIV_RW)?; + aliases.push(entry.map_alias_at(uat.kernel_lower_vm(), a.entry_low[stage], + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..G18_ENTRY_RING_BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(a.entry_high[stage] + offset as u64)? + != uat.kernel_lower_vm().translate_iova(a.entry_low[stage] + offset as u64)? { + return Err(EFAULT); + } + } + entries.push(entry, GFP_KERNEL)?; + } + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), a.graph, G18_GRAPH_BYTES, + mmu::PROT_FW_PRIV_RW)?; + graph.with_bytes_mut(|out| { + for stage in 0..2 { + let channel = CHANNEL[stage]; + put64(out, channel, a.graph + POINTERS[stage] as u64); + put64(out, channel + 8, a.graph + ITEMS[stage] as u64); + // One job list for both queues (M5 lesson). + put64(out, channel + 0x10, a.graph + JOB_LIST as u64); + put32(out, channel + 0x24, u32::MAX); + put32(out, channel + 0x28, 4); + put32(out, channel + 0x40, u32::MAX); + put32(out, channel + 0x44, 0); + put64(out, channel + 0x70, a.graph + CONTROL as u64); + put32(out, POINTERS[stage] + 0x50, u32::MAX); + put32(out, POINTERS[stage] + 0x60, CAPACITY); + } + put64(out, JOB_LIST + 8, a.graph + JOB_LIST as u64); + out[CONTROL..CONTROL + 2].fill(0xff); + out[CONTROL + 5] = 1; + out[CONTROL + 0x33] = 0xff; + out[CONTROL + 0x26] = 2; + Ok(()) + })?; + fence(Ordering::SeqCst); + Ok(Self { aliases, entries, graph, firmware: DataView::new(dev)?, addresses: a, step: 0, + queues: [G18KickQueueState { current_stamp: 1, previous_stamp: None, base: 1, + submitted_kicks: 0 }; 2] }) + } + + /// Channel-state firmware VA: outer record +8 and SKSM payload[1]. + pub(crate) fn queue_address(&self, stage: usize) -> u64 { + self.addresses.graph + CHANNEL[stage & 1] as u64 + } + + pub(crate) fn scheduler_control(&self) -> u64 { self.addresses.graph + CONTROL as u64 } + + pub(crate) fn job_list(&self) -> u64 { self.addresses.graph + JOB_LIST as u64 } + + pub(crate) fn counters(&mut self) -> Result<[[u32; 3]; 2]> { + let mut result = [[0; 3]; 2]; + for stage in 0..2 { + let base = self.addresses.graph + POINTERS[stage] as u64; + result[stage] = [self.graph.read_u32(base)?, self.graph.read_u32(base + 0x30)?, + self.graph.read_u32(base + 0x40)?]; + } + Ok(result) + } + + pub(crate) fn completion_list(&mut self) -> Result<[u64; 2]> { + let list = self.job_list(); + Ok([self.graph.read_u64(list)?, self.graph.read_u64(list + 8)?]) + } + + pub(crate) fn firmware_status(&mut self) -> Result { + let mut slots = [[0; 5]; 2]; + let mut schedulers = [0; 2]; + for stage in 0..2 { + let qid = G18_GRAPHICS_QUEUE_IDS[stage] as usize; + slots[stage] = self.firmware.words(SLOT_TABLE + qid * 0x28)?; + schedulers[stage] = self.firmware.words::<1>(SCHEDULER_TABLE + qid * 8)?[0]; + } + let control = self.scheduler_control(); + Ok(G18GraphicsFirmwareStatus { slots, schedulers, + control: [self.graph.read_u64(control)?, self.graph.read_u64(control + 0x12)?, + self.graph.read_u64(control + 0x1a)?] }) + } + + /// Both firmware slots name our rings with the right stage words + /// (b000 0x12cb4..0x12cd4; +0x20 = tag15 +0x22). + pub(crate) fn installed(status: &G18GraphicsFirmwareStatus, a: &G18GraphicsQueueAddresses) -> bool { + (0..2).all(|stage| { + let slot = status.slots[stage]; + slot[0] & 1 != 0 && slot[1] == a.entry_high[stage] && slot[2] == a.entry_low[stage] + && slot[3] == 4u64 << 32 && slot[4] as u32 == stage as u32 + }) + } + + /// Scheduler bound for both queues: shared control published in both + /// scheduler entries, allocated indices, state 4, membership bits 9/10. + pub(crate) fn scheduler_bound(&self, status: &G18GraphicsFirmwareStatus) -> bool { + let header = status.control[0]; + let members = (1u64 << G18_GRAPHICS_QUEUE_IDS[0]) | (1u64 << G18_GRAPHICS_QUEUE_IDS[1]); + status.schedulers.iter().all(|&pointer| pointer == self.scheduler_control()) + && header & 0xff != 0xff && (header >> 8) & 0xff != 0xff + && (header >> 16) & 0xff == 4 && status.control[1] == members && status.control[2] == 0 + } + + /// Firmware UMA-pool and PB slot entries, for the pool/PB bind gate. + pub(crate) fn pool_and_pb_status(&mut self, uma_slot: u32, pb_slot: u32) -> Result<([u64; 5], [u64; 6])> { + let pool = self.firmware.words::<3>(POOL_TABLE + uma_slot as usize * 0x100)?; + let members = self.firmware.words::<2>(POOL_TABLE + uma_slot as usize * 0x100 + 0x78)?; + let pb = self.firmware.words::<6>(PB_TABLE + pb_slot as usize * 0x50)?; + Ok(([pool[0], pool[1], pool[2], members[0], members[1]], pb)) + } + + pub(crate) fn stage_gate(&mut self, phase: u32, pass: &G18QueuedPass) -> Result { + let status = self.firmware_status()?; + if self.counters()? != [[phase; 3]; 2] + || !Self::installed(&status, &self.addresses) { return Ok(false); } + if phase >= 2 && !self.scheduler_bound(&status) { return Ok(false); } + if phase >= 3 { + let pb = pass.parameter_buffer.ok_or(EINVAL)?; + let (pool, slot) = self.pool_and_pb_status(pass.uma.slot, pb.slot)?; + // b000 0x13708..0x137c0 sets the queue bit at slot +0x18 and the + // manager bit at +0x28; +0x20 stays clear. Stage33b hardware read + // exactly [state, 1, 3, 0x200, 0, 0x400] after the pool records. + if pool != [pass.uma.state, pass.uma.generation.ok_or(EINVAL)?, (2u64 << 32) | 3, + (1 << 9) | (1 << 10), 0] + || slot != [pb.state, pb.generation, u64::from(pass.context_id), 1 << 9, 0, 1 << 10] { + return Ok(false); + } + } + Ok(true) + } + + /// Gate before preparing any work item. Both rows, context and resource + /// fields are checked by the job/table owners; these are the queue checks. + pub(crate) fn audit_before_pass(&mut self, uat: &mmu::Uat, pass: &G18QueuedPass) -> Result { + if self.step != 3 || !self.stage_gate(3, pass)? { return Err(EIO); } + let a = self.addresses; + for stage in 0..2 { + self.graph.with_bytes_mut(|out| { + if out[ITEMS[stage]+24..ITEMS[stage]+48].iter().any(|&b| b != 0) { return Err(EBUSY); } + for phase in 0..3 { + let expected = g18_stage_config(stage, phase, (a.entry_low[stage], a.entry_high[stage]), pass); + let off = CONFIGS[stage]+phase as usize*0x100; + if out[off..off+G18_CHANNEL_CONFIG_BYTES] != expected { return Err(EIO); } + } + Ok(()) + })?; + self.entries[stage].with_bytes_mut(|out| { + if out[0x200..0x400].iter().any(|&b| b != 0) { return Err(EBUSY); } + Ok(()) + })?; + for off in (0..G18_ENTRY_RING_BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(a.entry_high[stage] + off as u64)? + != uat.kernel_lower_vm().translate_iova(a.entry_low[stage] + off as u64)? { return Err(EFAULT); } + } + } + for off in (0..G18_GRAPH_BYTES).step_by(mmu::UAT_PGSZ) { + uat.kernel_vm().translate_iova(a.graph + off as u64)?; + } + Ok(()) + } + + /// Compare the retained command records to the shared pure encoders and + /// print their actual bytes' digest before producer 6 can be published. + pub(crate) fn audit_prepared_pass(&mut self, dev: &AsahiDevice, pass: &G18QueuedPass) -> Result { + if self.step != 4 || self.counters()? != [[3; 3]; 2] { return Err(EBUSY); } + let a = self.addresses; + for stage in 0..2 { + for phase in 0..4 { + let expected = g18_stage_config(stage, phase, (a.entry_low[stage], a.entry_high[stage]), pass); + self.graph.with_bytes_mut(|out| { + let off = CONFIGS[stage] + phase as usize*0x100; + let raw = &out[off..off+G18_CHANNEL_CONFIG_BYTES]; + if raw != expected { return Err(EIO); } + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=tag15-{}-{} fnv={:#018x}\n", stage, phase, + crate::g18::fw::graphics_payload::fnv64(raw)); + Ok(()) + })?; + let outer = encode_g18_outer_record(self.queue_address(stage), G18_GRAPHICS_QUEUE_IDS[stage], + [1,2,3,6][phase as usize]); + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=outer-{}-{} fnv={:#018x}\n", stage, phase, + crate::g18::fw::graphics_payload::fnv64(&outer)); + } + let (entry, kick) = g18_stage_kick(stage, pass, + G18KickQueueState { current_stamp: 1, previous_stamp: None, base: 1, submitted_kicks: 0 }, + self.queue_address(stage)).map_err(|_| EINVAL)?; + self.entries[stage].with_bytes_mut(|out| { + let raw = &out[0x200..0x400]; + if raw != entry.bytes { return Err(EIO); } + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=sksm-{} fnv={:#018x}\n",stage, + crate::g18::fw::graphics_payload::fnv64(raw)); + Ok(()) + })?; + self.graph.with_bytes_mut(|out| { + let raw = &out[KICKS[stage]..KICKS[stage]+G18_KICK_BYTES]; + if raw != kick { return Err(EIO); } + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=kick-{} fnv={:#018x}\n",stage, + crate::g18::fw::graphics_payload::fnv64(raw)); + Ok(()) + })?; + } + Ok(()) + } + + fn stage_step(&mut self, step: u32, pass: &G18QueuedPass) -> Result<[[u8; G18_OUTER_RECORD_BYTES]; 2]> { + if self.step != step || step > 2 { return Err(EBUSY); } + let expected = [step; 3]; + if self.counters()? != [expected; 2] { return Err(EBUSY); } + let a = self.addresses; + let mut records = [[0; G18_OUTER_RECORD_BYTES]; 2]; + for stage in 0..2 { + let config = g18_stage_config(stage, step, (a.entry_low[stage], a.entry_high[stage]), pass); + let at = CONFIGS[stage] + step as usize * 0x100; + let item = ITEMS[stage] + step as usize * 8; + self.graph.with_bytes_mut(|out| { + if out[at..at + 0x100].iter().any(|b| *b != 0) + || out[item..item + 8].iter().any(|b| *b != 0) { return Err(EBUSY); } + out[at..at + G18_CHANNEL_CONFIG_BYTES].copy_from_slice(&config); + put64(out, item, a.graph + at as u64); + Ok(()) + })?; + records[stage] = encode_g18_outer_record(self.queue_address(stage), + G18_GRAPHICS_QUEUE_IDS[stage], step as u16 + 1); + } + fence(Ordering::SeqCst); + for stage in 0..2 { + self.graph.write_u32(a.graph + POINTERS[stage] as u64 + 0x40, step + 1)?; + } + fence(Ordering::SeqCst); + self.step = step + 1; + Ok([records[1], records[0]]) + } + + /// Step 1: install both queues (tag15 install=1, stage words 0/1). + pub(crate) fn install_records(&mut self, pass: &G18QueuedPass) -> Result<[[u8; G18_OUTER_RECORD_BYTES]; 2]> { + self.stage_step(0, pass) + } + + /// Step 2 (after `installed`): bind both queues to the shared control. + pub(crate) fn scheduler_records(&mut self, pass: &G18QueuedPass) -> Result<[[u8; G18_OUTER_RECORD_BYTES]; 2]> { + self.stage_step(1, pass) + } + + /// Step 3 (after `scheduler_bound`): UMA pool on both, PB tuple on TA. + /// The UMA descriptor and PB row must already be loaded and read back. + pub(crate) fn pool_records(&mut self, pass: &G18QueuedPass) -> Result<[[u8; G18_OUTER_RECORD_BYTES]; 2]> { + if pass.uma.slot == u32::MAX || pass.uma.state == 0 { return Err(EINVAL); } + if let Some(pb) = pass.parameter_buffer { + if pb.generation == 0 || pb.state == 0 { return Err(EINVAL); } + } + self.stage_step(2, pass) + } + + /// Step 4: store both SKSM entries and [command, context config, kick] + /// per queue, unpublished. The job must have encoded both commands, + /// incremented the PB counter, loaded its roots and passed every + /// pre-kick check before `publish_pass`. + pub(crate) fn prepare_pass(&mut self, pass: &G18QueuedPass) -> Result { + if self.step != 3 || self.counters()? != [[3; 3]; 2] { return Err(EBUSY); } + let a = self.addresses; + for stage in [1, 0] { + let (entry, kick) = g18_stage_kick(stage, pass, self.queues[stage], + self.queue_address(stage)).map_err(|_| EINVAL)?; + let offset = entry.offset as usize; + self.entries[stage].with_bytes_mut(|out| { + let slot = out.get_mut(offset..offset + entry.bytes.len()).ok_or(EINVAL)?; + if slot.iter().any(|b| *b != 0) { return Err(EBUSY); } + slot.copy_from_slice(&entry.bytes); + Ok(()) + })?; + let config = g18_stage_config(stage, 3, (a.entry_low[stage], a.entry_high[stage]), pass); + let at = CONFIGS[stage] + 0x300; + self.graph.with_bytes_mut(|out| { + if out[at..at + 0x100].iter().any(|b| *b != 0) + || out[KICKS[stage]..KICKS[stage] + 0x40].iter().any(|b| *b != 0) + || out[ITEMS[stage] + 24..ITEMS[stage] + 48].iter().any(|b| *b != 0) { + return Err(EBUSY); + } + out[at..at + G18_CHANNEL_CONFIG_BYTES].copy_from_slice(&config); + out[KICKS[stage]..KICKS[stage] + G18_KICK_BYTES].copy_from_slice(&kick); + for (index, address) in [pass.command_high[stage], a.graph + at as u64, + a.graph + KICKS[stage] as u64].into_iter().enumerate() { + put64(out, ITEMS[stage] + (index + 3) * 8, address); + } + Ok(()) + })?; + self.queues[stage] = G18KickQueueState { + current_stamp: (self.queues[stage].current_stamp + 1) & ((1 << 40) - 1), + previous_stamp: Some(self.queues[stage].current_stamp), + base: self.queues[stage].base, + submitted_kicks: self.queues[stage].submitted_kicks + 1, + }; + } + fence(Ordering::SeqCst); + self.step = 4; + Ok(()) + } + + /// Publish both inner producers (6) and return the two outer records. + /// A later failure never authorizes resetting rings or entries. + pub(crate) fn publish_pass(&mut self) -> Result<[[u8; G18_OUTER_RECORD_BYTES]; 2]> { + if self.step != 4 || self.counters()? != [[3; 3]; 2] { return Err(EBUSY); } + fence(Ordering::SeqCst); + let mut records = [[0; G18_OUTER_RECORD_BYTES]; 2]; + for stage in [1, 0] { + self.graph.write_u32(self.addresses.graph + POINTERS[stage] as u64 + 0x40, 6)?; + records[stage] = encode_g18_outer_record(self.queue_address(stage), + G18_GRAPHICS_QUEUE_IDS[stage], 6); + } + fence(Ordering::SeqCst); + self.step = 5; + Ok([records[1], records[0]]) + } +} + +// ---- Parameter-buffer kernel owner ------------------------------------------- +// +// The PM object graph below keeps the layout native AGXParameterManagement +// init (0xfffffe0008c5eee4) builds and that M5 proved on hardware +// (m5_graphics_job.rs PM_BLOCKS/PM_SHARED/PM_COUNTER/SCENES): a per-block page +// base table (state +0x44 = PM +0xc0, u32 at 8-byte stride = PM +0x90), a +// shared header (state +0x4c = PM +0xc8; scene records link to its +0x40 = +// PM +0xa0), the submitted-operations counter (state +0x64) and 0x80-byte scene +// records (TA +0x28 = PM +0xa8[scene], TA +0x30 = PM +0xb8 = the record after +// the last). Scene record fields follow the native init loop (0x8c5f850.. +// 0x8c5f874) and setupSceneState (0x8c61ba8..0x8c61bc8); the shared header +// words are M5-derived and marked [OPEN] in the G18 render-command notes. + +use crate::g18::fw::parameter_buffer::*; + +const PM_GRAPH_BYTES: usize = 0x8000; +const BLOCK_TABLE: usize = 0; +const SHARED: usize = 0x800; +const COUNTER: usize = 0x900; +const SCENES: usize = 0x1000; +const SCENE_BYTES: usize = 0x80; + +fn pb_put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn pb_put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} +/// Native metrics spacing: device PM config word 1 (0x40) rounded (0x8c5f7b0). +const METRICS_STRIDE: u64 = 0x40; +const METRICS_BYTES: usize = 0x4000; +const SCENE_SCRATCH_BYTES: usize = 0x4000; +const DISCARD_BYTES: usize = 0x8000; +pub(crate) const G18_PB_MAX_SCENES: u32 = ((PM_GRAPH_BYTES - SCENES) / SCENE_BYTES - 1) as u32; + +/// Fixed addresses from the address map. `*_high` are firmware VAs (global, +/// kernel_vm); `*_low` are GPU VAs that the TA/3D hardware dereferences in +/// the job's context: they are created in the caller's low VM (the job VM) +/// and may be re-mapped into further windows with `alias_low_window`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18PbAddresses { + /// PB state object (0x4000 allocation, 0xc0 used). Firmware writes +0x08. + pub state_high: u64, + /// PM graph: block table, shared header, counter, scene records (0x8000). + pub graph_high: u64, + pub page_list_high: u64, + pub page_list_low: u64, + /// The page-list VA written to PB state +0x28 and the PB row: the job + /// context's view the PM decodes (see `g18_pb_row_address_valid`). + pub page_list_row: u64, + pub metrics_high: u64, + pub metrics_low: u64, + pub scene_scratch_low: u64, + /// Block 0; blocks are contiguous at 0x80000 stride. + pub blocks_low: u64, + pub discard_low: u64, + /// Added (wrapping) to a low VA to form the user-process view native + /// uses via getGPUVirtualAddressForUserProcess (block page bases, discard + /// page, scene cycle). 0 when the job maps the PB unbiased; the address + /// map's COMPACT_BIAS when the page list must name the biased window. + pub user_view_delta: u64, +} + +/// Host policy for one PB. Every field is a documented choice, not a default +/// silently inherited from M5 (see the G18 render-command notes, section 4). +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18PbPolicy { + /// Backed blocks (native initial allocation); must be < ring capacity. + pub blocks: u32, + /// PM +0x148 ring capacity, +0x154/+0x15c, +0x1d4. + pub capacity: u32, + pub pm_154: u32, + pub pm_15c: u32, + pub pm_1d4: u32, + /// Scene records and the selected scene (descriptor PM +0x2c). + pub scene_count: u32, + pub scene: u32, + /// PB table slot (TA HardwareBufferID). + pub slot: u32, + /// tag15 +0x82. Must differ from the firmware slot's cached generation + /// (0 on a fresh boot) or firmware never fetches the PB state. + pub generation: u64, +} + +/// Values the TA/3D commands take from the PB. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18PbCommandFields { + pub pb_slot: u32, + pub pm_1d4: u32, + pub pb_state: u64, + pub pm_scene: u64, + pub pm_b8: u64, +} + +/// Pre-kick readback of the published PB objects. +#[derive(Clone, Copy, Debug)] +pub(crate) struct G18PbReadback { + pub row: [u8; G18_PB_ROW_BYTES], + pub state: [u8; G18_PB_STATE_FIRMWARE_BYTES], + pub counter: u32, +} + +#[cfg(not(test))] +pub(crate) struct G18ParameterBuffer { + // Window aliases go first: that VM's roots may still be live. + window_aliases: KVec, + low_aliases: KVec, + state: MappedObject, + graph: MappedObject, + page_list: MappedObject, + metrics: MappedObject, + scene_scratch: MappedObject, + blocks: KVec, + discard: MappedObject, + addresses: G18PbAddresses, + policy: G18PbPolicy, + inputs: G18PbStateInputs, + row: Option<[u8; G18_PB_ROW_BYTES]>, +} + +#[cfg(not(test))] +impl G18ParameterBuffer { + /// Allocate, map and encode every PB object. Nothing is published to + /// firmware: the PB-table row is written separately by `load_row`, and + /// the tag15 that names the PB comes later still. + pub(crate) fn new( + dev: &AsahiDevice, + uat: &mmu::Uat, + low_vm: &mmu::Vm, + a: G18PbAddresses, + policy: G18PbPolicy, + ) -> Result { + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 { + return Err(ENODEV); + } + if policy.blocks == 0 || policy.blocks >= policy.capacity + || policy.scene_count == 0 || policy.scene_count > G18_PB_MAX_SCENES + || policy.scene >= policy.scene_count || policy.slot >= G18_PB_TABLE_ROWS + || policy.generation == 0 + || u64::from(policy.scene_count) * METRICS_STRIDE > METRICS_BYTES as u64 + || a.blocks_low & (G18_PB_BLOCK_BYTES - 1) != 0 || a.discard_low & 0x7fff != 0 + || a.user_view_delta & 0x7fff != 0 + || !g18_pb_row_address_valid(a.page_list_row) { + return Err(EINVAL); + } + let list_entries = usize::try_from(policy.capacity)? + .checked_mul(G18_PB_PAGES_PER_BLOCK as usize).ok_or(EOVERFLOW)?; + let list_bytes = list_entries.checked_mul(4).ok_or(EOVERFLOW)? + .checked_add(mmu::UAT_PGSZ - 1).ok_or(EOVERFLOW)? & !(mmu::UAT_PGSZ - 1); + let high = |address, size| MappedObject::new_wc_at(dev, uat.kernel_vm(), address, size, + mmu::PROT_FW_SHARED_RW); + let low = |address, size| MappedObject::new_wc_at(dev, low_vm, address, + size, mmu::PROT_GPU_SHARED_RW); + let state = high(a.state_high, mmu::UAT_PGSZ)?; + let graph = high(a.graph_high, PM_GRAPH_BYTES)?; + let mut page_list = high(a.page_list_high, list_bytes)?; + let mut metrics = high(a.metrics_high, METRICS_BYTES)?; + let mut low_aliases = KVec::new(); + for (object, address, size) in [(&mut page_list, a.page_list_low, list_bytes), + (&mut metrics, a.metrics_low, METRICS_BYTES)] { + low_aliases.push(object.map_alias_at(low_vm, address, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(object.iova() + offset as u64)? + != low_vm.translate_iova(address + offset as u64)? { + return Err(EFAULT); + } + } + } + let scene_scratch = low(a.scene_scratch_low, SCENE_SCRATCH_BYTES)?; + let discard = low(a.discard_low, DISCARD_BYTES)?; + let mut blocks = KVec::new(); + for index in 0..u64::from(policy.blocks) { + let address = a.blocks_low + index * G18_PB_BLOCK_BYTES; + blocks.push(low(address, G18_PB_BLOCK_BYTES as usize)?, GFP_KERNEL)?; + } + let config = G18PmConfig { + pm_148: policy.capacity, + pm_138: u64::from(policy.blocks) << 19, + pm_1d8: G18_PB_PAGES_PER_BLOCK, + pm_70: G18_PB_BLOCK_BYTES, + pm_c0: a.graph_high + BLOCK_TABLE as u64, + pm_c8: a.graph_high + SHARED as u64, + pm_154: policy.pm_154, + pm_15c: policy.pm_15c, + pm_1d4: policy.pm_1d4, + }; + let inputs = G18PbStateInputs { + config, + page_list_firmware: a.page_list_high, + page_list_gpu: a.page_list_row, + page_list_bytes: u32::try_from(list_bytes)?, + counter_gpu: a.graph_high + COUNTER as u64, + discard_gpu: a.discard_low.wrapping_add(a.user_view_delta), + slot: policy.slot, + start_block: 0, + end_block: policy.blocks, + }; + let mut owner = Self { + window_aliases: KVec::new(), low_aliases, state, graph, page_list, metrics, + scene_scratch, blocks, discard, addresses: a, policy, inputs, row: None, + }; + owner.encode()?; + dev_info!(dev.as_ref(), + "G18 PB: state={:#x} graph={:#x} list={:#x}/{:#x} blocks={}x{:#x}@{:#x} slot={} gen={}; unpublished\n", + a.state_high, a.graph_high, a.page_list_high, a.page_list_low, policy.blocks, + G18_PB_BLOCK_BYTES, a.blocks_low, policy.slot, policy.generation); + Ok(owner) + } + + fn page_bases(&self) -> Result> { + let mut bases = KVec::with_capacity(self.policy.capacity as usize, GFP_KERNEL)?; + for slot in 0..u64::from(self.policy.capacity) { + // Unbacked ring slots are never reached before growth, which is + // not implemented; they repeat block 0 so no stray page appears. + let block = if slot < u64::from(self.policy.blocks) { slot } else { 0 }; + let va = (self.addresses.blocks_low + block * G18_PB_BLOCK_BYTES) + .wrapping_add(self.addresses.user_view_delta); + let base = u32::try_from(va >> 15).map_err(|_| ERANGE)?; + if base >> 25 != 0 { return Err(ERANGE); } + bases.push(base, GFP_KERNEL)?; + } + Ok(bases) + } + + fn scene_cycle_at(&self, scene: u32) -> u64 { + // Scene scratch in the user-process view (setupSceneState 0x8c61bb8). + self.addresses.scene_scratch_low.wrapping_add(self.addresses.user_view_delta) + 0x40 + + u64::from(scene) * 0x40 + } + + fn encode(&mut self) -> Result { + let bases = self.page_bases()?; + let inputs = self.inputs; + let blocks = self.policy.blocks as usize; + self.page_list.with_bytes_mut(|out| { + g18_pb_page_list(out, &inputs.config, &bases, 0, inputs.end_block) + .map_err(|_| EINVAL)?; + Ok(()) + })?; + let a = self.addresses; + let policy = self.policy; + let mut cycles = KVec::with_capacity(policy.scene_count as usize, GFP_KERNEL)?; + for scene in 0..policy.scene_count { + cycles.push(self.scene_cycle_at(scene), GFP_KERNEL)?; + } + self.graph.with_bytes_mut(|out| { + for (index, base) in bases.iter().take(blocks).enumerate() { + pb_put32(out, BLOCK_TABLE + index * 8, *base); + } + // [OPEN] M5-proven shared header; +8 is buildPageList's first block. + pb_put32(out, SHARED, policy.blocks); + pb_put32(out, SHARED + 4, policy.blocks); + pb_put32(out, SHARED + 8, 0); + pb_put32(out, SHARED + 0x60, 1); + pb_put32(out, COUNTER, 0); + for scene in 0..policy.scene_count as usize { + let record = SCENES + scene * SCENE_BYTES; + pb_put64(out, record, a.metrics_low + scene as u64 * METRICS_STRIDE); + pb_put64(out, record + 8, a.metrics_high + scene as u64 * METRICS_STRIDE); + pb_put64(out, record + 0x28, cycles[scene]); + pb_put64(out, record + 0x40, a.graph_high + SHARED as u64 + 0x40); + } + Ok(()) + })?; + self.state.with_bytes_mut(|out| { + let state: &mut [u8; G18_PB_STATE_BYTES] = + (&mut out[..G18_PB_STATE_BYTES]).try_into().map_err(|_| EINVAL)?; + encode_g18_pb_state(state, &inputs, true); + Ok(()) + })?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Map every low object again into `vm` at `low VA + delta` (delta 0: + /// another VM at the same addresses; COMPACT_BIAS: the biased window of + /// the dual low map). Verifies every page against `low_vm`, the VM the + /// objects were created in. Call before the job's roots are published. + pub(crate) fn alias_low_window(&mut self, low_vm: &mmu::Vm, vm: &mmu::Vm, delta: u64) -> Result { + let a = self.addresses; + let mut aliases = KVec::new(); + let mut ranges = KVec::new(); + let list_bytes = self.inputs.page_list_bytes as u64; + let at = |address: u64| address.checked_add(delta).ok_or(EOVERFLOW); + aliases.push(self.page_list.map_alias_at(vm, at(a.page_list_low)?, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + ranges.push((a.page_list_low, list_bytes), GFP_KERNEL)?; + aliases.push(self.metrics.map_alias_at(vm, at(a.metrics_low)?, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + ranges.push((a.metrics_low, METRICS_BYTES as u64), GFP_KERNEL)?; + aliases.push(self.scene_scratch.map_alias_at(vm, at(a.scene_scratch_low)?, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + ranges.push((a.scene_scratch_low, SCENE_SCRATCH_BYTES as u64), GFP_KERNEL)?; + aliases.push(self.discard.map_alias_at(vm, at(a.discard_low)?, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL)?; + ranges.push((a.discard_low, DISCARD_BYTES as u64), GFP_KERNEL)?; + for (index, block) in self.blocks.iter_mut().enumerate() { + let address = a.blocks_low + index as u64 * G18_PB_BLOCK_BYTES; + aliases.push(block.map_alias_at(vm, at(address)?, mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + ranges.push((address, G18_PB_BLOCK_BYTES), GFP_KERNEL)?; + } + for &(address, bytes) in ranges.iter() { + for offset in (0..bytes).step_by(mmu::UAT_PGSZ) { + if vm.translate_iova(at(address)? + offset)? + != low_vm.translate_iova(address + offset)? { + return Err(EFAULT); + } + } + } + fence(Ordering::SeqCst); + for alias in aliases { + self.window_aliases.push(alias, GFP_KERNEL)?; + } + Ok(()) + } + + /// The row halLoadPBDesc would write over `old` (ring zero after + /// buildPageList). + pub(crate) fn pb_row(&self, old: [u8; G18_PB_ROW_BYTES]) -> [u8; G18_PB_ROW_BYTES] { + g18_pb_row(self.inputs.page_list_gpu, g18_pb_page_count(&self.inputs.config), + G18PbRing::default(), old) + } + + /// The complete encoded PB state (firmware owns +0x08 after binding). + pub(crate) fn encode_pm_state(&self, out: &mut [u8; G18_PB_STATE_BYTES]) { + encode_g18_pb_state(out, &self.inputs, true); + } + + /// Native loadPBDesc order: state +0x0c (already encoded) then the row. + /// Refuses an occupied row or a second load; plain stores, then a fence, + /// before any tag15 names this PB. + pub(crate) fn load_row(&mut self, table: &mut MappedObject) -> Result<[u8; G18_PB_ROW_BYTES]> { + if self.row.is_some() { return Err(EBUSY); } + let offset = self.policy.slot as usize * G18_PB_ROW_BYTES; + let row = table.with_bytes_mut(|out| { + if out.len() != G18_PB_TABLE_ROWS as usize * G18_PB_ROW_BYTES { return Err(EINVAL); } + let old: [u8; G18_PB_ROW_BYTES] = out[offset..offset + G18_PB_ROW_BYTES] + .try_into().map_err(|_| EINVAL)?; + if old != [0; G18_PB_ROW_BYTES] { return Err(EBUSY); } + let row = self.pb_row(old); + out[offset..offset + G18_PB_ROW_BYTES].copy_from_slice(&row); + Ok(row) + })?; + fence(Ordering::SeqCst); + self.row = Some(row); + Ok(row) + } + + /// Pre-kick self-check: re-read the table row, the firmware-visible state + /// prefix (except the firmware-owned context word) and the page list + /// head, and compare with the encoders. Any difference refuses the kick. + pub(crate) fn verify_before_kick(&mut self, table: &mut MappedObject) -> Result { + let expected_row = self.row.ok_or(EINVAL)?; + let offset = self.policy.slot as usize * G18_PB_ROW_BYTES; + let row: [u8; G18_PB_ROW_BYTES] = table.with_bytes_mut(|out| { + out.get(offset..offset + G18_PB_ROW_BYTES).ok_or(EINVAL)?.try_into().map_err(|_| EINVAL) + })?; + if row != expected_row || row == [0; G18_PB_ROW_BYTES] { return Err(EIO); } + if g18_pb_row_decoded_address(&row) != Some(self.inputs.page_list_gpu) { return Err(EIO); } + let mut expected = [0u8; G18_PB_STATE_BYTES]; + self.encode_pm_state(&mut expected); + let state: [u8; G18_PB_STATE_FIRMWARE_BYTES] = self.state.with_bytes_mut(|out| { + out.get(..G18_PB_STATE_FIRMWARE_BYTES).ok_or(EINVAL)?.try_into().map_err(|_| EINVAL) + })?; + for (index, (actual, wanted)) in state.iter().zip(expected.iter()).enumerate() { + let firmware_owned = (8..12).contains(&index); + let unwritten = (0x1c..0x20).contains(&index); + if !firmware_owned && !unwritten && actual != wanted { return Err(EIO); } + } + let bases = self.page_bases()?; + let head = self.page_list.read_u32(self.addresses.page_list_high)?; + if head != bases[0] { return Err(EIO); } + for index in 0..self.policy.blocks as usize { + if self.graph.read_u32(self.addresses.graph_high + (index * 8) as u64)? != bases[index] { + return Err(EIO); + } + for page in 0..G18_PB_PAGES_PER_BLOCK as usize { + if self.page_list.read_u32(self.addresses.page_list_high + ((index * 16 + page) * 4) as u64)? + != bases[index] + page as u32 { return Err(EIO); } + } + } + let counter = self.graph.read_u32(self.addresses.graph_high + COUNTER as u64)?; + Ok(G18PbReadback { row, state, counter }) + } + + /// Native incrementPMSubmittedOperations (0x8bbcd50): one per 3D submit, + /// before the 3D command is published. + pub(crate) fn count_submitted_render(&mut self) -> Result { + let address = self.addresses.graph_high + COUNTER as u64; + let value = self.graph.read_u32(address)?.checked_add(1).ok_or(EOVERFLOW)?; + self.graph.write_u32(address, value)?; + fence(Ordering::SeqCst); + Ok(value) + } + + pub(crate) fn command_fields(&self) -> G18PbCommandFields { + let a = self.addresses; + G18PbCommandFields { + pb_slot: self.policy.slot, + pm_1d4: self.policy.pm_1d4, + pb_state: a.state_high, + pm_scene: a.graph_high + (SCENES + self.policy.scene as usize * SCENE_BYTES) as u64, + pm_b8: a.graph_high + (SCENES + self.policy.scene_count as usize * SCENE_BYTES) as u64, + } + } + + pub(crate) fn audit_job_mappings(&self, uat: &mmu::Uat, vm: &mmu::Vm, bias: u64) -> Result { + let a = self.addresses; + let mut pages = 0; + for (address, bytes) in [(a.page_list_low, self.inputs.page_list_bytes as usize), + (a.metrics_low, METRICS_BYTES), (a.scene_scratch_low, SCENE_SCRATCH_BYTES), + (a.discard_low, DISCARD_BYTES), (a.blocks_low, self.policy.blocks as usize * G18_PB_BLOCK_BYTES as usize)] { + for offset in (0..bytes).step_by(mmu::UAT_PGSZ) { + if vm.translate_iova(address + offset as u64)? != vm.translate_iova(bias + address + offset as u64)? { return Err(EFAULT); } + pages += 1; + } + } + for object in [&self.state, &self.graph, &self.page_list, &self.metrics] { + for off in (0..object.logical_size).step_by(mmu::UAT_PGSZ) { uat.kernel_vm().translate_iova(object.iova() + off as u64)?; } + } + Ok(pages) + } + + pub(crate) fn print_h1_digests(&mut self, dev: &AsahiDevice, row: &[u8;16]) -> Result { + // Context +8 is firmware-owned and separately checked. Normalize only + // that field and the unwritten +1c word for host reproducibility. + self.state.with_bytes_mut(|out| { + let mut bytes = [0u8; G18_PB_STATE_BYTES]; + bytes.copy_from_slice(&out[..G18_PB_STATE_BYTES]); + bytes[8..12].fill(0); bytes[0x1c..0x20].fill(0); + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=pb-state fnv={:#018x}\n", crate::g18::fw::graphics_payload::fnv64(&bytes)); + Ok(()) + })?; + self.page_list.with_bytes_mut(|bytes| { + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=pb-page-list fnv={:#018x}\n", crate::g18::fw::graphics_payload::fnv64(bytes)); Ok(()) + })?; + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=pb-row fnv={:#018x}\n", crate::g18::fw::graphics_payload::fnv64(row)); + Ok(()) + } + + /// tag15 PB tuple for the TA queue; `manager` is the linked 3D queue index. + pub(crate) fn binding(&self, manager: u32) -> G18ConfigParameterBuffer { + G18ConfigParameterBuffer { + state: self.addresses.state_high, generation: self.policy.generation, + slot: self.policy.slot, manager, + } + } + + /// Register inputs owned by the PB (G17pRenderParameters.cycle and the + /// metrics record for register 0x1c910), for render-regs. + pub(crate) fn scene_cycle(&self) -> u64 { self.scene_cycle_at(self.policy.scene) } + + pub(crate) fn metrics_record(&self) -> u64 { + self.addresses.metrics_low + u64::from(self.policy.scene) * METRICS_STRIDE + } + + /// Firmware PB slot table entry (b000 0x71420 + slot * 0x50) expected + /// after the TA pool bind: state, generation, context, queue bit, manager bit. + pub(crate) fn expected_firmware_slot(&self, context: u32, ta_qid: u8, manager: u32) + -> [u64; 6] { + [self.addresses.state_high, self.policy.generation, u64::from(context), + 1u64 << ta_qid, 0, 1u64 << manager] + } +} diff --git a/drivers/gpu/drm/asahi/g18/queue/graphics_application.rs b/drivers/gpu/drm/asahi/g18/queue/graphics_application.rs new file mode 100644 index 0000000000000..ab15b357a87ed --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/queue/graphics_application.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retired-frame reuse. Ring slots wrap, but native event identities do not. +use super::*; +use crate::g18::uapi::application_runtime::Frame; + +impl G18GraphicsQueues { + pub(crate) fn prepare_application_pass(&mut self, dev: &AsahiDevice, + pass: &G18QueuedPass, frame: Frame) -> Result { + let before = frame.inner_before(); + if self.counters()? != [[before;3];2] + || self.step != if frame.generation == 1 {3} else {5} + || self.completion_list()? != [0,self.job_list()] {return Err(EBUSY);} + let status=self.firmware_status()?; + if !Self::installed(&status,&self.addresses) || !self.scheduler_bound(&status) {return Err(EIO);} + let pb=pass.parameter_buffer.ok_or(EINVAL)?; + let (pool,slot)=self.pool_and_pb_status(pass.uma.slot,pb.slot)?; + if pool != [pass.uma.state,pass.uma.generation.ok_or(EINVAL)?,(2u64<<32)|3,0x600,0] + || slot != [pb.state,pb.generation,u64::from(pass.context_id),0x200,0,0x400] {return Err(EIO);} + for stage in [1,0] { + let state=G18KickQueueState {current_stamp:u64::from(frame.generation), + previous_stamp:if frame.generation==1 {None} else {Some(u64::from(frame.generation-1))}, + base:1,submitted_kicks:(frame.generation-1) as u16}; + let (entry,kick)=g18_stage_kick(stage,pass,state,self.queue_address(stage)).map_err(|_|EINVAL)?; + let config=g18_stage_config(stage,3,(self.addresses.entry_low[stage],self.addresses.entry_high[stage]),pass); + self.entries[stage].with_bytes_mut(|out| { + let slot=out.get_mut(entry.offset as usize..entry.offset as usize+entry.bytes.len()).ok_or(EINVAL)?; + slot.copy_from_slice(&entry.bytes); + if slot!=entry.bytes {return Err(EIO);} Ok(()) + })?; + let a=self.addresses.graph;let at=CONFIGS[stage]+0x300; + self.graph.with_bytes_mut(|out| { + out[at..at+G18_CHANNEL_CONFIG_BYTES].copy_from_slice(&config); + out[KICKS[stage]..KICKS[stage]+G18_KICK_BYTES].copy_from_slice(&kick); + for (i,address) in [pass.command_high[stage],a+at as u64,a+KICKS[stage] as u64].into_iter().enumerate() { + put64(out,ITEMS[stage]+((before as usize+i)%CAPACITY as usize)*8,address); + } + if out[at..at+G18_CHANNEL_CONFIG_BYTES]!=config + || out[KICKS[stage]..KICKS[stage]+G18_KICK_BYTES]!=kick {return Err(EIO);} Ok(()) + })?; + if frame.generation<=3 { + for (name,bytes) in [("sksm",entry.bytes.as_slice()),("kick",kick.as_slice()),("tag15",config.as_slice())] { + dev_info!(dev.as_ref(),"G18_APP_DIGEST frame={} stage={} name={} fnv={:#018x}\n", + frame.generation,stage,name,crate::g18::fw::graphics_payload::fnv64(bytes)); + } + } + } + fence(Ordering::SeqCst);self.step=4;Ok(()) + } + + /// The first firmware-owned precondition of the next frame's + /// [`Self::prepare_application_pass`] that does not hold after `frame` + /// retired, if any. A firmware recovery can reset queue state. + pub(crate) fn render_unready(&mut self, pass: &G18QueuedPass, frame: Frame) -> Result> { + let status=self.firmware_status()?; + let pb=pass.parameter_buffer.ok_or(EINVAL)?; + let (pool,slot)=self.pool_and_pb_status(pass.uma.slot,pb.slot)?; + Ok(if self.counters()? != [[frame.inner_after();3];2] {Some("counters")} + else if self.completion_list()? != [0,self.job_list()] {Some("completion-list")} + else if !Self::installed(&status,&self.addresses) {Some("installed")} + else if !self.scheduler_bound(&status) {Some("scheduler")} + else if pool != [pass.uma.state,pass.uma.generation.ok_or(EINVAL)?,(2u64<<32)|3,0x600,0] {Some("pool")} + else if slot != [pb.state,pb.generation,u64::from(pass.context_id),0x200,0,0x400] {Some("parameter-buffer")} + else {None}) + } + + /// Every firmware word [`Self::render_unready`] reads, for the kernel log. + pub(crate) fn render_state(&mut self, pass: &G18QueuedPass) -> Result<(G18GraphicsFirmwareStatus, [u64;5], [u64;6])> { + let pb=pass.parameter_buffer.ok_or(EINVAL)?; + let (pool,slot)=self.pool_and_pb_status(pass.uma.slot,pb.slot)?; + Ok((self.firmware_status()?,pool,slot)) + } + + pub(crate) fn publish_application_pass(&mut self, frame: Frame) -> Result<[[u8;24];2]> { + if self.step!=4 || self.counters()?!=[[frame.inner_before();3];2] {return Err(EBUSY);} + let mut records=[[0;24];2]; + fence(Ordering::SeqCst); + for stage in [1,0] { + self.graph.write_u32(self.addresses.graph+POINTERS[stage] as u64+0x40,frame.inner_after())?; + records[stage]=encode_g18_outer_record(self.queue_address(stage),G18_GRAPHICS_QUEUE_IDS[stage],frame.inner_after() as u16); + } + fence(Ordering::SeqCst);self.step=5;Ok([records[1],records[0]]) + } +} + +impl G18ParameterBuffer { + /// Keep the PB resident in its hardware slot across VM handoffs. Every + /// alias uses the addresses selected by the PB owner, including row VA. + pub(crate) fn application_aliases(&mut self, vm: &mmu::Vm) -> Result> { + let a=self.addresses;let bias=crate::g18::address_map::COMPACT_BIAS; + let mut aliases=KVec::new(); + for (object,low) in [(&mut self.page_list,a.page_list_low),(&mut self.metrics,a.metrics_low), + (&mut self.scene_scratch,a.scene_scratch_low),(&mut self.discard,a.discard_low)] { + for address in [low,low+bias] {aliases.push(object.map_alias_at(vm,address,mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?;} + } + if a.page_list_row!=a.page_list_low && a.page_list_row!=a.page_list_low+bias { + aliases.push(self.page_list.map_alias_at(vm,a.page_list_row,mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?; + } + for (index,block) in self.blocks.iter_mut().enumerate() { + let low=a.blocks_low+index as u64*G18_PB_BLOCK_BYTES; + for address in [low,low+bias] {aliases.push(block.map_alias_at(vm,address,mmu::PROT_GPU_SHARED_RW)?,GFP_KERNEL)?;} + } + Ok(aliases) + } + + /// The PB words a recovery can leave behind: the firmware-visible state + /// (ring cursors, free list, the +0x1c flag firmware sets when a recovery + /// flushes a render using this PB, b000 0xfd58-0xfd9c), the active scene + /// record, the submitted-render counter and the scene index. + pub(crate) fn recovery_words(&mut self) -> Result<([u32;G18_PB_STATE_FIRMWARE_BYTES/4],[u32;SCENE_BYTES/4],u32,u32)> { + let state: [u8;G18_PB_STATE_FIRMWARE_BYTES]=self.state.with_bytes_mut(|out|out[..G18_PB_STATE_FIRMWARE_BYTES].try_into().map_err(|_|EINVAL))?; + let start=SCENES+self.policy.scene as usize*SCENE_BYTES; + let scene: [u8;SCENE_BYTES]=self.graph.with_bytes_mut(|out|out[start..start+SCENE_BYTES].try_into().map_err(|_|EINVAL))?; + let word=|b:&[u8],i:usize|u32::from_le_bytes(b[i*4..i*4+4].try_into().unwrap_or([0;4])); + let counter=self.graph.read_u32(self.addresses.graph_high+COUNTER as u64)?; + Ok((core::array::from_fn(|i|word(&state,i)),core::array::from_fn(|i|word(&scene,i)),counter,self.policy.scene)) + } + + + pub(crate) fn application_scene(&mut self, template: Option<&[u8;SCENE_BYTES]>) -> Result<[u8;SCENE_BYTES]> { + let start=SCENES+self.policy.scene as usize*SCENE_BYTES; + self.graph.with_bytes_mut(|out| { + if let Some(bytes)=template {out[start..start+SCENE_BYTES].copy_from_slice(bytes);} + out[start..start+SCENE_BYTES].try_into().map_err(|_|EINVAL) + }) + } + + /// Hardware owns the ring cursors and free-list ordering after a render. + /// Do not re-encode them. Check the stable address/capacity and state owner. + pub(crate) fn verify_resident(&mut self, table: &mut MappedObject, context: u32, + generation: u32) -> Result { + if generation==1 {return self.verify_before_kick(table);} + let offset=self.policy.slot as usize*G18_PB_ROW_BYTES; + let row: [u8;G18_PB_ROW_BYTES]=table.with_bytes_mut(|out|out[offset..offset+G18_PB_ROW_BYTES].try_into().map_err(|_|EINVAL))?; + let word=|i:usize|u32::from_le_bytes(row[i..i+4].try_into().unwrap()); + if g18_pb_row_decoded_address(&row)!=Some(self.inputs.page_list_gpu) + || (word(4)&0x3f_ffff)!=(self.row.ok_or(EINVAL)?[4..8].try_into().map(u32::from_le_bytes).map_err(|_|EINVAL)?&0x3f_ffff) + || word(8)&0x3f_ffff>=self.policy.capacity*G18_PB_PAGES_PER_BLOCK + || word(12)&0x3f_ffff>=self.policy.capacity*G18_PB_PAGES_PER_BLOCK {return Err(EIO);} + let state: [u8;G18_PB_STATE_FIRMWARE_BYTES]=self.state.with_bytes_mut(|out|out[..G18_PB_STATE_FIRMWARE_BYTES].try_into().map_err(|_|EINVAL))?; + if u32::from_le_bytes(state[8..12].try_into().map_err(|_|EINVAL)?)!=context {return Err(EIO);} + let counter=self.graph.read_u32(self.addresses.graph_high+COUNTER as u64)?; + if counter!=generation {return Err(EIO);} + Ok(G18PbReadback {row,state,counter}) + } +} diff --git a/drivers/gpu/drm/asahi/g18/queue/mod.rs b/drivers/gpu/drm/asahi/g18/queue/mod.rs new file mode 100644 index 0000000000000..b63c47cc267c5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/queue/mod.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 firmware queues: graphics and compute queue records, scheduler binding. + +#[allow(dead_code)] +pub(crate) mod graphics; +#[allow(dead_code)] +pub(crate) mod probe; +#[allow(dead_code)] +pub(crate) mod scheduler; diff --git a/drivers/gpu/drm/asahi/g18/queue/probe.rs b/drivers/gpu/drm/asahi/g18/queue/probe.rs new file mode 100644 index 0000000000000..ff425b12b424e --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/queue/probe.rs @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! J873/26A428 queue registration and one bounded compute submission. +//! +//! Native kernelcache: AGXChannel::resetChannelState 0xfffffe0008bdd178; +//! AGXCLChannelSKSM::submitBuffer 0xfffffe0008be3ddc..0x8be3fcc. +//! GFX b000: dispatcher 0x1267c, tag15 target 0x12824, table 0xb90b8. +//! M6 adds 12 bytes at config +0x6e; never use M5's 0x86-byte packet here. + +use crate::g18::buffer::object::MappedObject; +use crate::g18::buffer::pool::G18UmaPoolBinding; +use crate::g18::runtime::compute::G18PreparedComputeCommand; +use crate::{driver::AsahiDevice, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{c_str, io, prelude::*}; + +const KERNEL_BASE: u64 = 0xffff_fc20_0000_0000; +const GRAPH_BASE: u64 = KERNEL_BASE + 0xc000_0000; +const ENTRY_HIGH: u64 = KERNEL_BASE + 0x39_0000; +const ENTRY_LOW: u64 = 0x70_005f_0000; +const ENTRY_SIZE: usize = 0x20000; // b000 0x1257c/0x125a0: 256 * 0x200 +pub(crate) const QUEUE_ID: u8 = 11; +const POINTERS: usize = 0x100; +const JOB_LIST: usize = 0x200; +const CONTROL: usize = 0x300; +const ITEMS: usize = 0x800; +const CAPACITY: u32 = 0x100; +const CONFIG: usize = 0x1800; +const DATA_SIZE: usize = 0xbc000; +const SLOT: usize = 0xb90b8 - 0x50000 + QUEUE_ID as usize * 0x28; +const SCHEDULER: usize = 0xb7468 - 0x50000 + QUEUE_ID as usize * 8; +const POOL_SLOT: usize = 0x5f660 - 0x50000; // native slot stride 0x100 +const QUEUE_POOLS: usize = 0x6f660 - 0x50000 + QUEUE_ID as usize * 0x38; + +/// Reserved GFX DATA, never written by the host. No native snapshot pointers +/// are used: the current DT reservation is the only physical address source. +struct FirmwareDataView { + map: io::mem::Mem, + cache_line: usize, +} +// SAFETY: The fixed reserved RAM mapping is observed only through bounded +// volatile reads; no host write or ownership transfer is possible. +unsafe impl Send for FirmwareDataView {} +// SAFETY: Concurrent read-only observations cannot mutate host-owned state. +unsafe impl Sync for FirmwareDataView {} + +impl FirmwareDataView { + fn new(dev: &AsahiDevice) -> Result { + let resource = dev.as_ref().of_node().ok_or(EINVAL)? + .reserved_mem_region_to_resource_byname(c_str!("gfx-data"))?; + if resource.size() != DATA_SIZE as u64 || resource.start() < 0x100_0000_0000 + || resource.start() & 0x3fff != 0 + || resource.start().checked_add(resource.size()).ok_or(EOVERFLOW)? > 0x104_0000_0000 + { + return Err(EINVAL); + } + let ctr: u64; + // SAFETY: Architectural read-only cache geometry register. + unsafe { core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)) }; + let cache_line = 4usize << ((ctr >> 16) & 0xf); + if cache_line > 0x4000 || (SLOT + 40 + cache_line - 1) & !(cache_line - 1) > DATA_SIZE { + return Err(EINVAL); + } + // SAFETY: WB matches reserved System RAM; the host never dirties it. + let map = unsafe { io::mem::Mem::try_new(resource, io::mem::MemFlag::WB.into()) }?; + Ok(Self { map, cache_line }) + } + + fn words(&self, offset: usize) -> [u64; N] { + // The only callers are the fixed slot and scheduler extents. Both + // are below the constructor-checked slot end inside reserved DATA. + let start = offset & !(self.cache_line - 1); + let end = (offset + N * 8 + self.cache_line - 1) & !(self.cache_line - 1); + for offset in (start..end).step_by(self.cache_line) { + // SAFETY: Constructor bounds the whole covering cache lines. + // Invalidate only: cleaning stale data would corrupt firmware. + unsafe { core::arch::asm!("dc ivac, {addr}", + addr = in(reg) self.map.ptr().add(offset), options(nostack, preserves_flags)) }; + } + // SAFETY: Complete cache invalidation before observing firmware stores. + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; + let mut out = [0; N]; + for (index, word) in out.iter_mut().enumerate() { + // SAFETY: Aligned reads wholly inside the constructor-checked slot. + *word = unsafe { core::ptr::read_volatile( + self.map.ptr().add(offset + index * 8).cast::()) }; + } + fence(Ordering::Acquire); + out + } +} + +pub(crate) struct G18QueueProbe { + _entry_alias: mmu::KernelMapping, + _entry: MappedObject, + graph: MappedObject, + firmware: FirmwareDataView, + compute_prepared: bool, + compute_generation: u32, + pool_slot: u32, +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +impl G18QueueProbe { + pub(crate) fn new(dev: &AsahiDevice, uat: &mmu::Uat, bind_scheduler: bool) -> Result { + if !uat.is_t8152_geometry() { return Err(ENODEV); } + let mut entry = MappedObject::new_wc_at(dev, uat.kernel_vm(), ENTRY_HIGH, + ENTRY_SIZE, mmu::PROT_FW_PRIV_RW)?; + let entry_alias = entry.map_alias_at(uat.kernel_lower_vm(), ENTRY_LOW, + mmu::PROT_GPU_SHARED_RW)?; + for offset in (0..ENTRY_SIZE).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(ENTRY_HIGH + offset as u64)? + != uat.kernel_lower_vm().translate_iova(ENTRY_LOW + offset as u64)? { + return Err(EFAULT); + } + } + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), GRAPH_BASE, + 0x4000, mmu::PROT_FW_PRIV_RW)?; + graph.with_bytes_mut(|out| { + // M6 channel state: resetChannelState zeros 0x80 bytes. The + // command processor only needs the pointers and ring capacity + // for this tag15-only experiment; retain native reset sentinels. + put64(out, 0, GRAPH_BASE + POINTERS as u64); + put64(out, 8, GRAPH_BASE + ITEMS as u64); + put64(out, 0x10, GRAPH_BASE + JOB_LIST as u64); + put32(out, 0x24, u32::MAX); + put32(out, 0x28, 4); + put32(out, 0x40, u32::MAX); + put32(out, 0x44, 0); + put64(out, 0x70, GRAPH_BASE + CONTROL as u64); + put32(out, POINTERS + 0x50, u32::MAX); + put32(out, POINTERS + 0x60, CAPACITY); + put64(out, JOB_LIST + 8, GRAPH_BASE + JOB_LIST as u64); + if bind_scheduler { + // 26A428 AGXCommandQueueReclaimableState::init at + // 0xfffffe0008bf9014. Both indices start unallocated. + out[CONTROL..CONTROL + 2].fill(0xff); + out[CONTROL + 5] = 1; + out[CONTROL + 0x33] = 0xff; + // b000 0xb234 sets2 on this fresh process-slot path. + out[CONTROL + 0x26] = 2; + } + + // b000 0x12824 requests 0x98 bytes. Unwritten trailing padding + // is zero. UMA/PB pool bindings and the new +0x6e/+0x72 grow + // pair are disabled. UMA +0x3e is a FList generation identity, + // not an event pointer; +0x46 selects its firmware pool slot. + let config = &mut out[CONFIG..CONFIG + 0x98]; + put32(config, 0, 15); + put64(config, 8, ENTRY_LOW); + put64(config, 0x10, ENTRY_HIGH); + config[0x18..0x1a].copy_from_slice(&u16::from(QUEUE_ID).to_le_bytes()); + for (offset, value) in [(0x1a, 1), (0x1e, 2), (0x22, 2), (0x32, 3), (0x5e, 2)] { + put32(config, offset, value); + } + put64(config, 0x3e, u64::MAX); + put32(config, 0x46, u32::MAX); + put64(config, 0x4a, GRAPH_BASE + CONTROL as u64); + config[0x82..0x92].fill(0xff); + put64(out, ITEMS, GRAPH_BASE + CONFIG as u64); + put32(out, POINTERS + 0x40, 1); + Ok(()) + })?; + fence(Ordering::SeqCst); + Ok(Self { _entry_alias: entry_alias, _entry: entry, graph, + firmware: FirmwareDataView::new(dev)?, compute_prepared: false, + compute_generation: 0, pool_slot: 0 }) + } + + pub(crate) fn queue_address(&self) -> u64 { GRAPH_BASE } + + /// The addition fixture owns context3. In DRM mode the retained graphics + /// owner already holds it, so compute must install the context it actually + /// acquired before its first descriptor is queued. The later context-update + /// command follows that descriptor and cannot repair the initial binding. + pub(crate) fn prepare_initial_context(&mut self, context: u32) -> Result { + if !(3..64).contains(&context) || self.counters()? != [0, 0, 1] + || self.slot() != [0; 5] { return Err(EBUSY); } + self.graph.with_bytes_mut(|out| { + put32(out, CONFIG + 0x32, context); + Ok(()) + })?; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn counters(&mut self) -> Result<[u32; 3]> { + Ok([self.graph.read_u32(GRAPH_BASE + POINTERS as u64)?, + self.graph.read_u32(GRAPH_BASE + POINTERS as u64 + 0x30)?, + self.graph.read_u32(GRAPH_BASE + POINTERS as u64 + 0x40)?]) + } + + pub(crate) fn slot(&self) -> [u64; 5] { self.firmware.words(SLOT) } + + pub(crate) fn scheduler_status(&mut self) -> Result<[u64; 4]> { + Ok([self.firmware.words::<1>(SCHEDULER)[0], + self.graph.read_u64(GRAPH_BASE + CONTROL as u64)?, + self.graph.read_u64(GRAPH_BASE + CONTROL as u64 + 0x12)?, + self.graph.read_u64(GRAPH_BASE + CONTROL as u64 + 0x1a)?]) + } + + pub(crate) fn prepare_scheduler_bind(&mut self) -> Result { + if self.counters()? != [1; 3] || !Self::installed(self.slot()) + || self.scheduler_status()? != [0, 0x1000000ffff, 0, 0] { + return Err(EBUSY); + } + self.graph.with_bytes_mut(|out| { + if out[CONFIG + 0x100..CONFIG + 0x198].iter().any(|x| *x != 0) { + return Err(EBUSY); + } + out.copy_within(CONFIG..CONFIG + 0x98, CONFIG + 0x100); + let config = &mut out[CONFIG + 0x100..CONFIG + 0x198]; + put32(config, 0x1a, 0); // already installed + put32(config, 0x52, 1); // bind scheduler + put64(out, ITEMS + 8, GRAPH_BASE + CONFIG as u64 + 0x100); + Ok(()) + })?; + fence(Ordering::SeqCst); + self.graph.write_u32(GRAPH_BASE + POINTERS as u64 + 0x40, 2)?; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn scheduler_bound([pointer, header, low, high]: [u64; 4]) -> bool { + pointer == GRAPH_BASE + CONTROL as u64 && header & 0xff != 0xff + && (header >> 8) & 0xff != 0xff + && (header >> 16) & 0xffff_ffff == 0x0104_0204 + && low == 1u64 << QUEUE_ID && high == 0 + } + + /// Observe both directions of firmware-owned pool membership, in addition + /// to the state/generation tuple. Retirement alone is not a binding proof. + pub(crate) fn pool_status(&self) -> [u64; 9] { + let slot = POOL_SLOT + self.pool_slot as usize * 0x100; + let header = self.firmware.words::<3>(slot); + let members = self.firmware.words::<2>(slot + 0x78); + let flags = self.firmware.words::<1>(slot + 0xe0); + let growth = self.firmware.words::<2>(slot + 0xf0); + let queues = self.firmware.words::<1>(QUEUE_POOLS); + [header[0], header[1], header[2], members[0], members[1], flags[0], + growth[0], growth[1], queues[0]] + } + + pub(crate) fn prepare_pool_bind(&mut self, binding: G18UmaPoolBinding) -> Result { + if binding.slot > 1 { return Err(EINVAL); } + self.pool_slot = binding.slot; + let before = self.pool_status(); + if self.counters()? != [2; 3] || !Self::installed(self.slot()) + || !Self::scheduler_bound(self.scheduler_status()?) + || binding.slot > 1 || binding.generation != 0 + || binding.state < KERNEL_BASE || binding.state & 0x3fff != 0 + || before[0] != 0 || before[3..5] != [0; 2] + || before[6..9] != [0; 3] { + return Err(EBUSY); + } + self.graph.with_bytes_mut(|out| { + if out[CONFIG + 0x200..CONFIG + 0x298].iter().any(|x| *x != 0) + || out[CONFIG + 0x136..CONFIG + 0x13e].iter().any(|x| *x != 0) + || out[CONFIG + 0x13e..CONFIG + 0x14a].iter().any(|x| *x != 0xff) { + return Err(EBUSY); + } + out.copy_within(CONFIG + 0x100..CONFIG + 0x198, CONFIG + 0x200); + let config = &mut out[CONFIG + 0x200..CONFIG + 0x298]; + put32(config, 0x52, 0); // scheduler is already bound + put64(config, 0x36, binding.state); + put64(config, 0x3e, binding.generation); + put32(config, 0x46, binding.slot); + put64(out, ITEMS + 16, GRAPH_BASE + CONFIG as u64 + 0x200); + Ok(()) + })?; + fence(Ordering::SeqCst); + self.graph.write_u32(GRAPH_BASE + POINTERS as u64 + 0x40, 3)?; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn pool_bound(status: [u64; 9], binding: G18UmaPoolBinding) -> bool { + Self::pool_bound_for_context(status, binding, 3) + } + + pub(crate) fn pool_bound_for_context(status: [u64; 9], binding: G18UmaPoolBinding, + context: u32) -> bool { + (3..64).contains(&context) + && status == [binding.state, binding.generation, (2u64 << 32) | u64::from(context), + 1u64 << QUEUE_ID, 0, 1, 0, 0, 1u64 << binding.slot] + } + + /// Store an unpublished SKSM entry and its three inner commands. The + /// original HAL400 encoder 8c4a290, dependency writer 8c4a5a4 and timestamp + /// converter 8c5620c agree byte-for-byte with the shared M5 serializer in + /// this bounded domain (audit-g18-sksm-native.py). The tag15 remains the + /// M6 0x98-byte form; never substitute the shorter M5 configuration. + pub(crate) fn prepare_compute_job(&mut self, command: G18PreparedComputeCommand, + pool: G18UmaPoolBinding) -> Result { + self.prepare_application_compute(command, pool, 1) + } + + pub(crate) fn prepare_application_compute(&mut self, command: G18PreparedComputeCommand, + pool: G18UmaPoolBinding, generation: u32) -> Result { + self.prepare_application_compute_rebind(command, pool, pool, command.context_id, generation) + } + + pub(crate) fn prepare_application_compute_rebind(&mut self, command: G18PreparedComputeCommand, + pool: G18UmaPoolBinding, previous: G18UmaPoolBinding, previous_context: u32, + generation: u32) -> Result { + use crate::g18::fw::sksm as sksm; + if pool.slot != previous.slot || pool.generation == u64::MAX + || !(pool.generation == previous.generation && pool.state == previous.state + || previous.generation.checked_add(1) == Some(pool.generation) + && pool.state != previous.state) + || pool.state < KERNEL_BASE || pool.state & 0x3fff != 0 { return Err(EINVAL); } + if generation == 0 || generation > 0x00ff_ffff + || generation != self.compute_generation + 1 { return Err(EINVAL); } + let before = (3 + (generation - 1) * 3) % 256; + if self.compute_prepared || self.counters()? != [before; 3] + || !Self::installed(self.slot()) + || !Self::scheduler_usable(self.scheduler_status()?) + || !Self::pool_bound_for_context(self.pool_status(), previous, previous_context) + || !(3..64).contains(&command.context_id) || command.expected_stamp != generation << 8 + || command.firmware_address < KERNEL_BASE + || command.registers.main_count > 127 || command.registers.secondary_count != 4 { + return Err(EINVAL); + } + let entry = sksm::prepare_m5_sksm_entry(sksm::M5SksmQueue { + id: QUEUE_ID, priority: 2, current_stamp: u64::from(generation), submitted: 0, + previous_stamp: if generation == 1 { None } else { Some(u64::from(generation - 1)) }, + }, sksm::M5SksmOperands { + barriers: &[], first_predecessor: 0, implicit_dependency_type: 1, + mcache: None, flag_bit1: false, flag_bit61: false, flag_bit63: false, + payload: [command.firmware_address, GRAPH_BASE], + qos_byte_32: 3, qos_class_40: 8, qos_word_48: 0xffff, + // Native CL event mask uses type 2 * 64 contexts + context ID. + event_mask: [0, 0, 1u64 << command.context_id, 0], rce_kind: 0, + rce: [sksm::M5SksmRce { address: command.registers.main_gpu, + tag: command.registers.main_count as u8 }, + sksm::M5SksmRce { address: command.registers.secondary_gpu, tag: 4 }, + sksm::M5SksmRce { address: 0, tag: 0 }, + sksm::M5SksmRce { address: 0, tag: 0 }], + auxiliary: 0x3f_ffff_ffff_ffff, + }).map_err(|_| EINVAL)?; + // Original 26A428 submitBuffer 8be481c..8be4848 writes this same + // packed 21-byte form: type14, qid, count, priority, stamp, DM2. + let kick = sksm::encode_m5_sksm_add_kicks(QUEUE_ID, 2, u64::from(generation), 1).map_err(|_| EINVAL)?; + self._entry.with_bytes_mut(|out| { + let bytes = &mut out[entry.offset as usize..entry.offset as usize + entry.bytes.len()]; + if generation < 256 && bytes.iter().any(|v| *v != 0) { return Err(EBUSY); } + bytes.copy_from_slice(&entry.bytes); + Ok(()) + })?; + self.graph.with_bytes_mut(|out| { + if generation == 1 && (out[CONFIG + 0x300..CONFIG + 0x500].iter().any(|v| *v != 0) + || out[ITEMS + 24..ITEMS + 48].iter().any(|v| *v != 0)) { + return Err(EBUSY); + } + out.copy_within(CONFIG + 0x200..CONFIG + 0x298, CONFIG + 0x300); + let config = &mut out[CONFIG + 0x300..CONFIG + 0x398]; + put32(config, 0x32, command.context_id); + put64(config, 0x36, pool.state); + put64(config, 0x3e, pool.generation); + put32(config, 0x46, pool.slot); + put32(config, 0x62, 1); // native descriptor+0x190: update context + put32(config, 0x66, 1); // retained process owner identity + out[CONFIG + 0x400..CONFIG + 0x400 + kick.len()].copy_from_slice(&kick); + for (index, address) in [command.firmware_address, + GRAPH_BASE + CONFIG as u64 + 0x300, + GRAPH_BASE + CONFIG as u64 + 0x400].into_iter().enumerate() { + put64(out, ITEMS + ((index + before as usize) % 256) * 8, address); + } + Ok(()) + })?; + fence(Ordering::SeqCst); + self.compute_prepared = true; + self.compute_generation = generation; + Ok(()) + } + + /// Publish only after process roots and submitted-counter are ready. + /// A later failure cannot authorize resetting the shared ring or entry. + pub(crate) fn publish_compute_job(&mut self) -> Result { + if !self.compute_prepared || self.compute_generation == 0 { return Err(EBUSY); } + let before = (3 + (self.compute_generation - 1) * 3) % 256; + if self.counters()? != [before; 3] { return Err(EBUSY); } + fence(Ordering::SeqCst); + self.graph.write_u32(GRAPH_BASE + POINTERS as u64 + 0x40, + (before + 3) % 256)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// The scheduler state a submission needs once bound: the published + /// control, allocated indices, state 4 and QID 11's membership. Unlike + /// [`Self::scheduler_bound`] (the bind gate, checked against the first + /// observed header) it does not pin header bytes 3..7: a recovery clears + /// byte 5, and render's equivalent check never required them. + pub(crate) fn scheduler_usable([pointer, header, low, high]: [u64; 4]) -> bool { + pointer == GRAPH_BASE + CONTROL as u64 && header & 0xff != 0xff + && (header >> 8) & 0xff != 0xff && (header >> 16) & 0xff == 4 + && low == 1u64 << QUEUE_ID && high == 0 + } + + pub(crate) fn retire_application_compute(&mut self, generation: u32) -> Result { + if !self.compute_prepared || self.compute_generation != generation + || self.counters()? != [(3 + generation * 3) % 256; 3] { return Err(EBUSY); } + self.compute_prepared = false; + Ok(()) + } + + /// The first firmware-owned precondition of + /// [`Self::prepare_application_compute`] that does not hold, if any. A + /// firmware recovery can reset queue state the next job requires. + pub(crate) fn compute_unready(&mut self, pool: G18UmaPoolBinding, context: u32, + generation: u32) -> Result> { + let before = (3 + generation.wrapping_sub(1) * 3) % 256; + Ok(if self.compute_prepared { Some("prepared") } + else if self.counters()? != [before; 3] { Some("counters") } + else if !Self::installed(self.slot()) { Some("installed") } + else if !Self::scheduler_usable(self.scheduler_status()?) { Some("scheduler") } + else if !Self::pool_bound_for_context(self.pool_status(), pool, context) { Some("pool") } + else { None }) + } + + /// Every word [`Self::compute_unready`] reads, for the kernel log. + pub(crate) fn compute_state(&mut self) -> Result<([u32; 3], [u64; 5], [u64; 4], [u64; 9])> { + Ok((self.counters()?, self.slot(), self.scheduler_status()?, self.pool_status())) + } + + pub(crate) fn installed(slot: [u64; 5]) -> bool { + slot[0] & 1 != 0 && slot[1] == ENTRY_HIGH && slot[2] == ENTRY_LOW + && slot[3] == 4u64 << 32 && slot[4] as u32 == 2 + } +} diff --git a/drivers/gpu/drm/asahi/g18/queue/scheduler.rs b/drivers/gpu/drm/asahi/g18/queue/scheduler.rs new file mode 100644 index 0000000000000..34b91dd1d2486 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/queue/scheduler.rs @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained command scheduler records and their independent submitted counters. +//! +//! Allocation and preparation do not publish a command. The containing live +//! firmware owner must retain these mappings until BOTH processors are stopped, +//! including after errors. A bounded bring-up session never recycles a record. + +#![allow(dead_code)] + +use crate::g18::buffer::object::MappedObject; +use crate::{driver::AsahiDevice, g18::fw::scheduler::*, mmu}; +use core::ops::Range; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18SchedulerBinding { + pub(crate) record: u64, + pub(crate) submitted_counter: u64, + pub(crate) sequence: u32, +} + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18SchedulerSnapshot { + pub(crate) submitted: u32, + pub(crate) completed: u32, + pub(crate) linked: u32, + pub(crate) batches: [u32; 3], +} + +pub(crate) struct G18SchedulerMemory { + records: MappedObject, + counters: MappedObject, + armed: u64, + sequences: [u32; G18_SCHEDULER_RECORD_COUNT], + submitted: [u32; G18_SCHEDULER_RECORD_COUNT], +} + +impl G18SchedulerMemory { + pub(crate) fn print_render_digests(&mut self, dev: &AsahiDevice) -> Result { + for (name, object) in [("scheduler", &mut self.records), ("submitted-pre-kick", &mut self.counters)] { + object.with_bytes_mut(|bytes| { + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name={} fnv={:#018x}\n",name,crate::g18::fw::graphics_payload::fnv64(bytes)); + Ok(()) + })?; + } + Ok(()) + } + /// Runtime's deterministic reservation, separate from the legacy compute + /// scheduler. Counters and records are independent uncached FW mappings. + pub(crate) fn new_render(dev: &AsahiDevice, uat: &mmu::Uat) -> Result { + use crate::g18::address_map::GRAPHICS_SCHEDULER_START as BASE; + if !uat.is_t8152_geometry() { return Err(ENODEV); } + let counters = MappedObject::new_wc_at(dev, uat.kernel_vm(), BASE, + G18_SCHEDULER_COUNTER_BYTES, mmu::PROT_FW_SHARED_RW)?; + let mut records = MappedObject::new_wc_at(dev, uat.kernel_vm(), BASE + 0x4000, + G18_SCHEDULER_RECORD_BYTES, mmu::PROT_FW_SHARED_RW)?; + records.with_bytes_mut(|out| initialize_g18_scheduler_records( + out.try_into().map_err(|_| EINVAL)?, counters.iova()).ok_or(EINVAL))?; + fence(Ordering::SeqCst); + Ok(Self { records, counters, armed: 0, + sequences: [0; G18_SCHEDULER_RECORD_COUNT], submitted: [0; G18_SCHEDULER_RECORD_COUNT] }) + } + + pub(crate) fn new(dev: &AsahiDevice, uat: &mmu::Uat, high_range: Range) -> Result { + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 { + return Err(ENODEV); + } + let counters = MappedObject::new_wc_in_range( + dev, + uat.kernel_vm(), + high_range.clone(), + G18_SCHEDULER_COUNTER_BYTES, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + )?; + let mut records = MappedObject::new_wc_in_range( + dev, + uat.kernel_vm(), + high_range, + G18_SCHEDULER_RECORD_BYTES, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + )?; + // Both native extents fit in one independently owned 16 KiB page. + // MappedObject zeroes its allocation before installing the mapping. + for object in [&counters, &records] { + uat.kernel_vm().translate_iova(object.iova())?; + } + records.with_bytes_mut(|out| { + let out = out.try_into().map_err(|_| EINVAL)?; + initialize_g18_scheduler_records(out, counters.iova()).ok_or(EINVAL) + })?; + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), + "G18 command scheduler: records={:#x} counters={:#x} count=36 stride=0x100 extent=0xc8 retained, unpublished\n", + records.iova(), counters.iova()); + Ok(Self { + records, + counters, + armed: 0, + sequences: [0; G18_SCHEDULER_RECORD_COUNT], + submitted: [0; G18_SCHEDULER_RECORD_COUNT], + }) + } + + /// Prepare one fresh record. A record can be armed only once during this + /// bounded session; completion alone does not authorize host reset/reuse. + pub(crate) fn prepare(&mut self, index: usize, sequence: u32) -> Result { + if index >= G18_SCHEDULER_RECORD_COUNT { + return Err(EINVAL); + } + if self.armed & (1u64 << index) != 0 { + return Err(EBUSY); + } + let start = index * G18_SCHEDULER_RECORD_STRIDE; + let submitted_counter = self.counters.iova() + index as u64 * 4; + self.counters.write_u32(submitted_counter, 0)?; + self.records.with_bytes_mut(|out| { + let record = out[start..start + G18_SCHEDULER_RECORD_STRIDE] + .as_mut() + .try_into() + .map_err(|_| EINVAL)?; + prepare_g18_scheduler_record( + record, + submitted_counter, + sequence, + G18_SCHEDULER_DEFAULT_THRESHOLD, + 0, + 0, + ) + .ok_or(EINVAL) + })?; + fence(Ordering::SeqCst); + self.sequences[index] = sequence; + self.armed |= 1u64 << index; + self.binding(index) + } + + pub(crate) fn binding(&self, index: usize) -> Result { + if index >= G18_SCHEDULER_RECORD_COUNT || self.armed & (1u64 << index) == 0 { + return Err(EINVAL); + } + Ok(G18SchedulerBinding { + record: self.records.iova() + (index * G18_SCHEDULER_RECORD_STRIDE) as u64, + submitted_counter: self.counters.iova() + index as u64 * 4, + sequence: self.sequences[index], + }) + } + + /// Final counter write before the caller publishes a fully prepared kick. + /// The caller must arrange all fallible preparation before this operation. + /// Once it succeeds, a failed publication requires retaining this session; + /// rolling the target back could race firmware. No firmware-owned record + /// fields are written here, even when submitting another job to the record. + pub(crate) fn publish_kick(&mut self, index: usize) -> Result { + let binding = self.binding(index)?; + let previous = self.submitted[index]; + if self.counters.read_u32(binding.submitted_counter)? != previous { + return Err(EIO); + } + let next = previous.checked_add(1).ok_or(EOVERFLOW)?; + self.counters.write_u32(binding.submitted_counter, next)?; + fence(Ordering::SeqCst); + self.submitted[index] = next; + Ok(next) + } + + /// Diagnostic snapshot only: these separate reads are not an atomic + /// completion fence and do not by themselves prove that a shader ran. + pub(crate) fn snapshot(&mut self, index: usize) -> Result { + let binding = self.binding(index)?; + fence(Ordering::SeqCst); + if self.records.read_u64(binding.record)? != binding.submitted_counter + || self.records.read_u32(binding.record + 8)? != binding.sequence + { + return Err(EIO); + } + Ok(G18SchedulerSnapshot { + submitted: self.counters.read_u32(binding.submitted_counter)?, + completed: self.records.read_u32(binding.record + 0xc)?, + linked: self.records.read_u32(binding.record + 0x94)?, + batches: [ + self.records.read_u32(binding.record + 0x14)?, + self.records.read_u32(binding.record + 0x18)?, + self.records.read_u32(binding.record + 0x1c)?, + ], + }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/regs.rs b/drivers/gpu/drm/asahi/g18/regs.rs new file mode 100644 index 0000000000000..ede40c1b2e16d --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/regs.rs @@ -0,0 +1,451 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! T8152 SGX register window and PSD power-state reads used before firmware start. +//! +//! Items copied verbatim from the J873 module-1134 sources (provenance in +//! the port's extraction ledger); only module paths +//! are rewritten. +use kernel::{ + c_str, + device::Core, + devres::Devres, + io::{mem::IoMem, Io}, + platform, + prelude::*, + sync::aref::ARef, +}; +use crate::g18::{hw, identity}; + +const ID_VERSION: usize = 0xd04000; + +const ID_UNK08: usize = 0xd04008; + +const ID_COUNTS_1: usize = 0xd04010; + +const ID_COUNTS_2: usize = 0xd04014; + +const ID_UNK18: usize = 0xd04018; + +const ID_CLUSTERS: usize = 0xd0401c; + +// The "G14X" register family is shared by G14X (t602x) and every AGX3 part +// (G15/G16/G17). The ID block at 0xd04000 is unchanged, and the per-cluster +// core-enable masks are three u32 words at 0xe01500 (stride 4): AGXG17X +// `readChipInfo` builds the offset (`mov w23,#0x1500; movk w23,#0xe0,lsl#16` +// @ 0xfffffe0008b4b220) and reads [base], [base+4], [base+8]; the identical +// pattern appears in all eight AGX3 kexts of the pinned 26A5378n corpus +// (e.g. AGXG15G @ 0xfffffe0008af22f0). +const CORE_MASKS_G14X: usize = 0xe01500; + +/// Device resources for this GPU instance. +pub(crate) struct Resources { + dev: ARef, + sgx: Pin>>>, + // The G17P SKSM FIFO lock and deferred SKSM writes are not carried: on + // T8152 the firmware owns KSMFE queue setup (hwdata+0x2724 = 0) and the + // host performs no SKSM MMIO. +} + +impl Resources { + + /// Map the required resources given our platform device. + pub(crate) fn new(pdev: &platform::Device) -> Result { + // The pinned host maps accelerator device-memory index 0 and stores + // it at accelerator+0x348 before every SKSM scratch access. That range + // is the J700 GPU block exposed to Linux by the named `sgx` resource + // (the second platform `reg` tuple, after `asc`). + let sgx_req = pdev.io_request_by_name(c_str!("sgx")).ok_or(EINVAL)?; + let sgx_iomem = KBox::pin_init(sgx_req.iomap(), GFP_KERNEL)?; + + Ok(Resources { + // SAFETY: This device does DMA via the UAT IOMMU. + dev: pdev.into(), + sgx: sgx_iomem, + }) + } + + /// Match the measured M5 native AP setup before starting firmware CPUs. + /// + /// The broad 25G76 AP trace writes 0x30001 at physical 0x580e90000; + /// Linux's prior stopped probe reads 0x30000. This diagnostic accepts + /// only those two observed states and changes only bit zero. The bit's + /// precise hardware role remains unqualified; do not generalize it. + pub(crate) fn m5_match_pmgr_boot_control(&self) -> Result { + const OFFSET: usize = 0xe90000; + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let res = sgx.relaxed(); + let before = res.try_read32(OFFSET)?; + if before != 0x30000 && before != 0x30001 { + dev_err!(self.dev.as_ref(), "M5_INIT unexpected PMGR control {:#x}\n", before); + return Err(EIO); + } + res.try_write32(before | 1, OFFSET)?; + let after = res.try_read32(OFFSET)?; + dev_info!(self.dev.as_ref(), "M5_INIT native PMGR control: {:#x} -> {:#x}\n", + before, after); + if after != 0x30001 { + return Err(EIO); + } + Ok(()) + } + + /// Read and decode the T8152 (G18G) GPU identity. + /// + /// Requires the [`T8152SgxPowered`] proof: the SGX PMGR2 PSD service was + /// read and reported active immediately before, so these + /// reads cannot hit a gated block. Only the registers 26A428 + /// `AGX·PI_400·G·A0·Accelerator::readChipInfo` reads are touched: the ID + /// block at 0xd04000..0xd0401c and the single core-enable word at + /// 0xe01500. No register is written. + /// + /// The raw words are always logged. The decode never rejects: an unknown + /// or different family/variant byte is reported and the part is + /// identified by its chip id and DT compatible (`soc`), and implausible + /// counts fall back to the native registry topology (2 x 6 cores). + pub(crate) fn get_gpu_id_t8152( + &self, + powered: &T8152SgxPowered, + soc: &'static hw::agx3::SocConfig, + ) -> Result { + let registry = hw::agx3::T8152_REGISTRY; + let expected = + identity::decode_gpu_identity(soc.hw_family, soc.hw_variant, soc.num_dies as u8) + .ok_or(EINVAL)?; + if soc.chip_id != 0x8152 || expected.gpu_gen != hw::GpuGen::G18 { + return Err(EINVAL); + } + + let (id_version, id_unk08, id_counts_1, id_counts_2, id_unk18, id_clusters, core_mask) = { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + // Ordered (non-relaxed) reads, after the ordered PSD read. + ( + sgx.try_read32(ID_VERSION)?, + sgx.try_read32(ID_UNK08)?, + sgx.try_read32(ID_COUNTS_1)?, + sgx.try_read32(ID_COUNTS_2)?, + sgx.try_read32(ID_UNK18)?, + sgx.try_read32(ID_CLUSTERS)?, + sgx.try_read32(CORE_MASKS_G14X)?, + ) + }; + + dev_info!( + self.dev.as_ref(), + "T8152: PSD sgx state={:#x} ack={:#x}; raw GPU ID {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}; core mask {:#x}\n", + powered.sample().state, + powered.sample().ack, + id_version, + id_unk08, + id_counts_1, + id_counts_2, + id_unk18, + id_clusters, + core_mask + ); + + let family = ((id_version >> 24) & 0xff) as u8; + let variant = ((id_version >> 16) & 0xff) as u8; + let reg_dies = (id_counts_1 >> 16) & 0xf; + match identity::decode_gpu_identity(family, variant, reg_dies as u8) { + Some(decoded) if decoded == expected => dev_info!( + self.dev.as_ref(), + "T8152: ID family {:#x} variant {:#x} decodes to {:?}{:?} as expected\n", + family, + variant, + decoded.gpu_gen, + decoded.gpu_variant + ), + Some(decoded) => dev_warn!( + self.dev.as_ref(), + "T8152: ID family {:#x} variant {:#x} decodes to {:?}{:?}; using the chip-id/compatible identity {:?}{:?}\n", + family, + variant, + decoded.gpu_gen, + decoded.gpu_variant, + expected.gpu_gen, + expected.gpu_variant + ), + None => dev_warn!( + self.dev.as_ref(), + "T8152: unknown ID family {:#x} variant {:#x} dies {}; using the chip-id/compatible identity {:?}{:?}\n", + family, + variant, + reg_dies, + expected.gpu_gen, + expected.gpu_variant + ), + } + + // Same count layout as the PI_400 decode (@ 0xfffffe0008c3d53c..0x554): + // dies [19:16], clusters per die [15:8], cores per cluster [7:0]. + let reg_cores = id_counts_1 & 0xff; + let reg_clusters = ((id_counts_1 >> 8) & 0xff) * reg_dies; + let registry_cores = registry.num_cores / registry.num_mgpus; + let (num_dies, num_clusters, num_cores) = if (1..=32).contains(®_cores) + && reg_clusters != 0 + && reg_cores * reg_clusters <= 32 + { + (reg_dies, reg_clusters, reg_cores) + } else { + dev_warn!( + self.dev.as_ref(), + "T8152: implausible ID counts ({} dies, {} clusters, {} cores); using registry topology {} x {}\n", + reg_dies, + reg_clusters, + reg_cores, + registry.num_mgpus, + registry_cores + ); + (soc.num_dies, registry.num_mgpus, registry_cores) + }; + + let mut core_masks = KVec::new(); + let mut remaining = core_mask as u64; + let cluster_mask = (1u64 << num_cores) - 1; + let mut total_active_cores = 0; + for _ in 0..num_clusters { + let mask = (remaining & cluster_mask) as u32; + core_masks.push(mask, GFP_KERNEL)?; + total_active_cores += mask.count_ones(); + remaining >>= num_cores; + } + if remaining != 0 { + dev_warn!( + self.dev.as_ref(), + "T8152: core mask {:#x} has bits beyond {} x {} cores\n", + core_mask, + num_clusters, + num_cores + ); + } + if total_active_cores != registry.num_cores + || core_masks.as_slice() != registry.core_mask_list.as_slice() + { + dev_warn!( + self.dev.as_ref(), + "T8152: core masks {:#x?} ({} active) differ from the native registry {:#x?} ({})\n", + core_masks, + total_active_cores, + registry.core_mask_list, + registry.num_cores + ); + } + + let mut core_masks_packed = KVec::new(); + core_masks_packed.push(core_mask, GFP_KERNEL)?; + + let (gpu_rev, gpu_rev_id) = match (id_version >> 8) & 0xff { + 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), + 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), + 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), + 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), + 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), + 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), + a => { + dev_warn!( + self.dev.as_ref(), + "T8152: unknown GPU revision byte {:#x}; recorded as an A0 placeholder\n", + a + ); + (hw::GpuRevision::A0, hw::GpuRevisionID::A0) + } + }; + + dev_info!( + self.dev.as_ref(), + "T8152: {:?}{:?} rev {:?}, {} dies, {} clusters x {} cores, {} active, masks {:#x?}\n", + expected.gpu_gen, + expected.gpu_variant, + gpu_rev, + num_dies, + num_clusters, + num_cores, + total_active_cores, + core_masks + ); + + Ok(hw::GpuIdConfig { + gpu_gen: expected.gpu_gen, + gpu_variant: expected.gpu_variant, + usc_generation: expected.usc_generation, + gpu_hal_generation: expected.gpu_hal_generation, + gpu_rev, + gpu_rev_id, + num_dies, + num_clusters, + num_cores, + num_frags: num_cores, + num_gps: (id_counts_2 >> 16) & 0xff, + total_active_cores, + core_masks, + core_masks_packed, + gpc_perf_state_map: 0, + gpc_perf_state_map_low: 0, + gpc_perf_state_control: 0, + }) + } + +} + +const T8152_PSD_STATE: usize = 0x80; + +const T8152_PSD_ACK: usize = 0xbc; + +const T8152_PSD_STATE_MASK: u32 = 0x7; + +/// One of the three PSD services that power the T8152 GPU. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum T8152PsdService { + /// `sgx`, PSD service 3. + Sgx, + /// `gfx-asc` (GFX role ASC), PSD service 81. + GfxAsc, + /// `gfx1-asc` (GFX1 role ASC), PSD service 82. + Gfx1Asc, +} + +impl T8152PsdService { + /// Every GPU PSD service, in ADT order. + pub(crate) const ALL: [Self; 3] = [Self::Sgx, Self::GfxAsc, Self::Gfx1Asc]; + + /// Native 26A428 request/state, reproduced on J873 with PMP running. + pub(crate) const fn active_state(self) -> u32 { + match self { + Self::Sgx => 0, + Self::GfxAsc | Self::Gfx1Asc => 2, + } + } + + const fn resource_name(self) -> &'static CStr { + match self { + Self::Sgx => c_str!("psd-sgx"), + Self::GfxAsc => c_str!("psd-gfx-asc"), + Self::Gfx1Asc => c_str!("psd-gfx1-asc"), + } + } + + /// PMGR2 implicit service id. + pub(crate) const fn service_id(self) -> u32 { + match self { + Self::Sgx => 3, + Self::GfxAsc => 81, + Self::Gfx1Asc => 82, + } + } + + /// CPU-physical base of the service block. + pub(crate) const fn physical_base(self) -> u64 { + 0x3_0300_0000 + 0xa1_0000 + ((self.service_id() as u64) << 8) + } +} + +/// One read-only sample of a PSD service's state and ack words. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct T8152PsdSample { + pub(crate) service: T8152PsdService, + pub(crate) state: u32, + pub(crate) ack: u32, +} + +impl T8152PsdSample { + /// The PSD power state field, state[2:0]. + pub(crate) const fn power_state(&self) -> u32 { + self.state & T8152_PSD_STATE_MASK + } + + /// Point-in-time active-state proof, not a transition-completion test. + /// J873 hardware confirms STATE reads clear ACK. A request must poll + /// ACK first and verify STATE once; later access gates use STATE alone. + pub(crate) const fn powered(&self) -> bool { + self.power_state() == self.service.active_state() + } + + /// Short steady-state verdict for logs. + pub(crate) const fn verdict(&self) -> &'static str { + if self.powered() { + "powered" + } else { + "not powered" + } + } +} + +/// `-EHOSTDOWN`: the block's PSD service is not proven powered, so every +/// MMIO read of it was skipped. The mailbox provider returns the same errno. +pub(crate) fn t8152_not_powered() -> Error { + Error::from_errno(-(kernel::bindings::EHOSTDOWN as i32)) +} + +/// Proof that the SGX PSD service read as powered. +/// +/// Built only by [`t8152_prove_sgx_powered`] from a fresh powered sample. +/// [`Resources::get_gpu_id_t8152`] requires it, so the SGX ID block cannot +/// be read without a preceding PSD read. The proof is a point-in-time fact. +/// It does not keep the block powered, so use it right away. +pub(crate) struct T8152SgxPowered { + sample: T8152PsdSample, +} + +impl T8152SgxPowered { + pub(crate) const fn sample(&self) -> T8152PsdSample { + self.sample + } +} + +/// Read one GPU PSD service's state and ack words. Reads only, no writes. +/// +/// The block is mapped for this call alone and never claimed, so a later +/// power owner can map the same service block. +pub(crate) fn t8152_read_psd( + pdev: &platform::Device, + service: T8152PsdService, +) -> Result { + let resource = pdev + .resource_by_name(service.resource_name()) + .ok_or(ENODEV)?; + if resource.start() != service.physical_base() || resource.size() != 0x100 { + dev_err!( + pdev.as_ref(), + "T8152: PSD service {} resource {:#x}+{:#x} does not match {:#x}+0x100\n", + service.service_id(), + resource.start(), + resource.size(), + service.physical_base() + ); + return Err(EINVAL); + } + let request = pdev + .io_request_by_name(service.resource_name()) + .ok_or(ENODEV)?; + let iomem = KBox::pin_init(request.iomap(), GFP_KERNEL)?; + let io = iomem.try_access().ok_or(ENODEV)?; + // STATE consumes the transition ACK; preserve it in the snapshot first. + let ack = io.try_read32(T8152_PSD_ACK)?; + let state = io.try_read32(T8152_PSD_STATE)?; + Ok(T8152PsdSample { + service, + state, + ack, + }) +} + +/// Read the SGX PSD service and return a power proof only if it is active +/// (state 0). Otherwise log the sample and return [`t8152_not_powered`], so +/// the caller skips every SGX read. +pub(crate) fn t8152_prove_sgx_powered( + pdev: &platform::Device, +) -> Result { + let sample = t8152_read_psd(pdev, T8152PsdService::Sgx)?; + if !sample.powered() { + dev_info!( + pdev.as_ref(), + "T8152: sgx PSD service 3 state={:#x} (power state {}) ack={:#x} is not active ({}); SGX reads skipped\n", + sample.state, + sample.power_state(), + sample.ack, + sample.verdict() + ); + return Err(t8152_not_powered()); + } + Ok(T8152SgxPowered { sample }) +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/compute.rs b/drivers/gpu/drm/asahi/g18/runtime/compute.rs new file mode 100644 index 0000000000000..3874897459a95 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/compute.rs @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained G18 channel working memory, one command, and completion storage. +//! +//! This owner never publishes a ring entry. All mappings remain retained by +//! G18FirmwareMemory until both firmware processors are stopped. Shader and +//! argument buffers, native device policy, and process context installation +//! remain separate obligations of the eventual submission path. + +#![allow(dead_code)] + +use crate::{g18::queue::scheduler::G18SchedulerBinding, g18::buffer::object::MappedObject}; +use crate::{driver::AsahiDevice, g18::fw::compute::*, g18::fw::compute_memory::*, mmu}; +use core::ops::Range; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +// Host-selected range, after the existing UMA directory/block list, before +// the fixed graph. The VM allocator checks overlap with every existing owner. +const LOW_RANGE: Range = 0x70_0800_0000..0x70_1000_0000; +const SUPPORT_BYTES: usize = 0x100; +const STAMP_OFFSETS: [u64; 2] = [0, 0x40]; +const TIME_OFFSETS: [u64; 2] = [0x80, 0xc0]; + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18PreparedComputeCommand { + pub(crate) firmware_address: u64, + pub(crate) gpu_address: u64, + pub(crate) expected_stamp: u32, + pub(crate) context_id: u32, + pub(crate) scheduler_record: u64, + pub(crate) registers: G18ComputeRegisterSummary, +} + +#[derive(Debug, Copy, Clone)] +pub(crate) struct G18ComputeCompletionSnapshot { + pub(crate) stamps: [u32; 2], + pub(crate) timestamps: [u64; 2], +} + +pub(crate) struct G18ComputeMemory { + // Retract the process roots before dropping their aliases and backing. + process: Option, + // Alias owns the same GEM object and is released before the lower mapping. + command_high: mmu::KernelMapping, + command: MappedObject, + channel: MappedObject, + support: MappedObject, + layout: G18ComputeChannelLayout, + pointers: G18ComputeChannelPointers, + prepared: bool, + // Set before attempting publication, so even a partial failure cannot + // authorize the unpublished preparation rollback path. + context_published: bool, +} + +struct G18ComputeProcess { + context: mmu::M5ExecutionContext, + // Same GEM pages and GPU VAs as the firmware-owned command/channel. + aliases: [mmu::KernelMapping; 2], +} + +impl G18ComputeMemory { + /// The application owner calls this only after the complete queue, + /// scheduler and stamp gate, while it still retains the submitted VM. + pub(crate) fn retire_application(&mut self, stamp: u32) -> Result { + let completed = self.completion_snapshot()?; + if completed.stamps[0] != stamp || completed.timestamps[0] == 0 + || completed.timestamps[1] < completed.timestamps[0] { return Err(EBUSY); } + self.release_application() + } + + /// Release a dispatch that a firmware recovery ended after its queue + /// consumed it: the GPU was reset, so no stamp or timestamp will come. + pub(crate) fn retire_recovered(&mut self) -> Result { + if !self.prepared { return Err(EINVAL); } + self.release_application() + } + + fn release_application(&mut self) -> Result { + self.process = None; + self.prepared = false; + self.context_published = false; + self.command.with_bytes_mut(|bytes| { bytes.fill(0); Ok(()) })?; + self.support.with_bytes_mut(|bytes| { bytes.fill(0); Ok(()) })?; + Ok(()) + } + + /// Drop a process lease acquired by a failed preparation. No prepared + /// command or attempted root publication may cross this boundary. + pub(crate) fn abort_unpublished_process(&mut self) -> Result { + if self.prepared || self.context_published { return Err(EBUSY); } + self.command.with_bytes_mut(|bytes| { bytes.fill(0); Ok(()) })?; + self.process = None; + Ok(()) + } + + pub(crate) fn unpublished_idle(&self) -> bool { + !self.prepared && !self.context_published && self.process.is_none() + } + + pub(crate) fn new( + dev: &AsahiDevice, + uat: &mmu::Uat, + high_range: Range, + clusters: u32, + width: u32, + ) -> Result { + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 { + return Err(ENODEV); + } + let layout = G18ComputeChannelLayout::new(clusters, width).ok_or(EINVAL)?; + let size: usize = layout.allocation_bytes.try_into()?; + let channel = MappedObject::new_wc_in_range( + dev, + uat.kernel_lower_vm(), + LOW_RANGE, + size, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + )?; + let pointers = layout.bind(channel.iova()).ok_or(ERANGE)?; + for offset in (0..size).step_by(mmu::UAT_PGSZ) { + uat.kernel_lower_vm() + .translate_iova(channel.iova() + offset as u64)?; + } + let mut command = MappedObject::new_wc_in_range( + dev, + uat.kernel_lower_vm(), + LOW_RANGE, + G18_COMPUTE_COMMAND_BYTES, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + )?; + let command_high = command.map_alias_in_range( + uat.kernel_vm(), + high_range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + )?; + for offset in (0..G18_COMPUTE_COMMAND_BYTES).step_by(mmu::UAT_PGSZ) { + if uat + .kernel_lower_vm() + .translate_iova(command.iova() + offset as u64)? + != uat + .kernel_vm() + .translate_iova(command_high.iova() + offset as u64)? + { + return Err(EFAULT); + } + } + let support = MappedObject::new_wc_in_range( + dev, + uat.kernel_vm(), + high_range, + SUPPORT_BYTES, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + )?; + uat.kernel_vm().translate_iova(support.iova())?; + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), + "G18 compute memory: channel={:#x}+{:#x} operands={:#x}/{:#x} control={:#x} tail={:#x} command={:#x}/{:#x} support={:#x}, retained unpublished\n", + channel.iova(), layout.allocation_bytes, pointers.operands[0], pointers.operands[1], + pointers.control, pointers.tail, command.iova(), command_high.iova(), support.iova()); + Ok(Self { + process: None, + command_high, + command, + channel, + support, + layout, + pointers, + prepared: false, + context_published: false, + }) + } + + /// Bind the payload VM while retaining both aliases and the context lease. + /// A failure leaves the original command/channel and firmware roots intact. + pub(crate) fn bind_process( + &mut self, dev: &AsahiDevice, uat: &mmu::Uat, vm: &mmu::Vm, + ) -> Result { + if !self.unpublished_idle() { return Err(EBUSY); } + let context = uat.prepare_t8152_execution_context(vm)?; + let command = self.command.map_alias_at(vm, self.command.iova(), mmu::PROT_GPU_SHARED_RW)?; + let channel = self.channel.map_alias_at(vm, self.channel.iova(), mmu::PROT_GPU_SHARED_RW)?; + for (address, length) in [(self.command.iova(), G18_COMPUTE_COMMAND_BYTES), + (self.channel.iova(), self.layout.allocation_bytes as usize)] { + for offset in (0..length).step_by(mmu::UAT_PGSZ) { + let address = address + offset as u64; + if vm.translate_iova(address)? != uat.kernel_lower_vm().translate_iova(address)? { + return Err(EFAULT); + } + } + } + let id = u32::from(context.context_id()); + crate::g18_kick_info!(dev.as_ref(), + "G18 compute context: id={} roots={:x?}; command/channel aliases retained, unpublished\n", + id, context.roots()); + self.process = Some(G18ComputeProcess { context, aliases: [command, channel] }); + Ok(id) + } + + /// Encode one unpublished command. Caller must retain shader/argument + /// bindings in the selected process VM and supply native device policy. + pub(crate) fn prepare( + &mut self, + scheduler: G18SchedulerBinding, + inputs: &G18ComputeCommandInputs, + registers: &G18ComputeRegisterInputs, + ) -> Result { + if self.prepared { + return Err(EBUSY); + } + let process = self.process.as_ref().ok_or(EINVAL)?; + if registers.clusters != self.layout.clusters + || registers.cluster_width != self.layout.width + || registers.context_id != u32::from(process.context.context_id()) + || inputs.channel_ordinal >= 4 + || inputs.completion.stamp == 0 + || scheduler.record == 0 + { + return Err(EINVAL); + } + let inputs = G18ComputeCommandInputs { + d198: scheduler.record, + stamp_addresses: STAMP_OFFSETS.map(|offset| self.support.iova() + offset), + d1b0: self.support.iova() + TIME_OFFSETS[0], + d1b8: self.support.iova() + TIME_OFFSETS[1], + ..*inputs + }; + let registers = G18ComputeRegisterInputs { + command_gpu: self.command.iova(), + operands: self.pointers.operands, + support_control: self.pointers.control, + // 26A428 loadCommandBufferData 8be5eac selects this record; + // prepare 8be58b0 then takes its +8 word, independently of the + // UAT context ID. Optional SMAPP resource objects are disabled. + d630: scheduler.sequence, + ..*registers + }; + let summary = self.command.with_bytes_mut(|out| { + let out = out.try_into().map_err(|_| EINVAL)?; + encode_g18_compute_command(out, &inputs, ®isters).map_err(|_| EINVAL) + })?; + fence(Ordering::SeqCst); + self.prepared = true; + Ok(G18PreparedComputeCommand { + firmware_address: self.command_high.iova(), + gpu_address: self.command.iova(), + expected_stamp: inputs.completion.stamp, + context_id: registers.context_id, + scheduler_record: scheduler.record, + registers: summary, + }) + } + + /// Root publication is delayed until every command field is prepared. + /// It does not publish a queue entry or advance a scheduler counter. + pub(crate) fn publish_context(&mut self) -> Result<[u64; 2]> { + if !self.prepared { return Err(EINVAL); } + let process = self.process.as_mut().ok_or(EINVAL)?; + self.context_published = true; + process.context.publish()?; + if !process.context.is_current() { return Err(EIO); } + Ok(process.context.roots()) + } + + /// These independent WC reads are diagnostic, not an atomic completion + /// fence. Shader execution still requires verified output data and a clean + /// firmware/fault observation, even if both stamp words change. + pub(crate) fn completion_snapshot(&mut self) -> Result { + if !self.prepared { + return Err(EINVAL); + } + fence(Ordering::SeqCst); + let base = self.support.iova(); + Ok(G18ComputeCompletionSnapshot { + stamps: [ + self.support.read_u32(base + STAMP_OFFSETS[0])?, + self.support.read_u32(base + STAMP_OFFSETS[1])?, + ], + timestamps: [ + self.support.read_u64(base + TIME_OFFSETS[0])?, + self.support.read_u64(base + TIME_OFFSETS[1])?, + ], + }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/compute_payload.rs b/drivers/gpu/drm/asahi/g18/runtime/compute_payload.rs new file mode 100644 index 0000000000000..eeb9d9a15036d --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/compute_payload.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained private G18 compute fixture in its own, unpublished process VM. +//! Context-root publication, command aliases, device policy and the queue kick +//! are separate requirements. Loading this object never submits work. + +#![allow(dead_code)] + +use crate::g18::buffer::object::MappedObject; +use crate::{driver::AsahiDevice, g18::fw::compute::*, g18::fixture::compute_payload as payload, + g18::fixture::compute_triangle::{TrianglePatch, TriangleResult, PATCH_COUNT}, + g18::fw::compute_state::{G18ComputeState, G18_COMPUTE_STATE_BYTES}, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{firmware::Firmware, prelude::*}; + +/// The verified CDM control stream (register 0x1a420) is a 40-byte direct +/// launch, this barrier and this terminate. Command +0xee0 (`d4e0`, the +/// G17/M5 `cdm_terminator`) is the terminate word's address. Block type is +/// bits 31:29, as Mesa agx3_cmdstream.h decodes this capture. +const CDM_LAUNCH_BYTES: usize = 40; +const CDM_BARRIER: u32 = 0x6000_0160; +const CDM_TERMINATE: u32 = 0x4000_0000; + +fn stream_words(bytes: &[u8]) -> [u32; 12] { + core::array::from_fn(|i| u32::from_le_bytes(bytes[4 * i..4 * i + 4].try_into().unwrap())) +} + +pub(crate) struct G18ComputePayloadMemory { + // Mappings and their GEM storage are released before the VM root owner. + pages: KVec, + vm: mmu::Vm, + state: [u8; G18_COMPUTE_STATE_BYTES], + output: usize, + /// `g18_compute_variant=1` replacement for command +0xee0. + cdm_terminator: Option, +} + +impl G18ComputePayloadMemory { + pub(crate) fn load(dev: &AsahiDevice, uat: &mmu::Uat) -> Result { + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != payload::PAGE_BYTES { + return Err(ENODEV); + } + let firmware = Firmware::request(c"asahi/g18-26a428-compute-add.bin", dev.as_ref())?; + let reference = payload::ComputePayload::parse(firmware.data()).map_err(|_| EINVAL)?; + let vm = uat.new_t8152_compute_fixture_vm()?; + let mut pages = KVec::new(); + let mut output = None; + for index in 0..reference.page_count() { + let input = reference.page(index).ok_or(EINVAL)?; + let mut page = MappedObject::new_wc_at(dev, &vm, input.address, + payload::PAGE_BYTES, mmu::PROT_GPU_SHARED_RW)?; + page.with_bytes_mut(|bytes| { + bytes.copy_from_slice(input.bytes); + if input.address == payload::OUTPUT { + // Defense in depth: never inherit a capture's answer. + for word in bytes.chunks_exact_mut(4) { + word.copy_from_slice(&payload::POISON.to_le_bytes()); + } + } + Ok(()) + })?; + vm.translate_iova(input.address)?; + vm.translate_iova(input.address + payload::PAGE_BYTES as u64 - 1)?; + if input.address == payload::OUTPUT { output = Some(index); } + pages.push(page, GFP_KERNEL)?; + } + let state = reference.state_bytes().try_into().map_err(|_| EINVAL)?; + let mut owner = Self { pages, vm, state, output: output.ok_or(EINVAL)?, cdm_terminator: None }; + let initial = owner.output_snapshot()?; + if initial.checked != 4096 || initial.untouched != 4096 || initial.correct != 0 { + return Err(EIO); + } + match *crate::module_parameters::g18_compute_variant.value() { + 0 => {} + 1 => owner.empty_cdm_stream(dev)?, + 2 => owner.triangle_shader(dev, payload::fnv64(&firmware.data()[payload::HEADER_BYTES..]))?, + _ => return Err(EINVAL), + } + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), + "G18 compute payload: {} pages retained, all 4096 outputs poisoned; private VM roots unpublished\n", + owner.pages.len()); + Ok(owner) + } + + pub(crate) fn vm(&self) -> &mmu::Vm { &self.vm } + + pub(crate) fn scratch_plan(&self) -> Result { + crate::g18::fw::uma::g18_addition_scratch(&self.state).ok_or(EINVAL) + } + + pub(crate) fn apply_state(&self, c: &mut G18ComputeCommandInputs, r: &mut G18ComputeRegisterInputs) { + G18ComputeState::parse(&self.state).unwrap().apply(c, r); + } + + pub(crate) fn addition_inputs(&self) + -> Result<(G18ComputeCommandInputs, G18ComputeRegisterInputs)> { + let (mut command, registers) = payload::addition_inputs(&self.state).map_err(|_| EINVAL)?; + if let Some(terminator) = self.cdm_terminator { command.d4e0 = terminator; } + Ok((command, registers)) + } + + /// Stage8 bisection, `g18_compute_variant=1`: dispatch nothing. Only after + /// the unchanged verified load, rewrite the stream as the capture's own + /// barrier and terminate at +0/+4, zero +8..+0x2f (the old launch, + /// barrier and terminate) and point command +0xee0 at the new terminate. + /// Every other byte, page and register is unchanged; nothing is published. + fn empty_cdm_stream(&mut self, dev: &AsahiDevice) -> Result { + let mut command = G18ComputeCommandInputs::default(); + let mut registers = G18ComputeRegisterInputs::default(); + self.apply_state(&mut command, &mut registers); + // g18_compute.rs emits program[1] as register 0x1a420. + let (stream, terminator) = (registers.program[1], command.d4e0); + let end = CDM_LAUNCH_BYTES + 8; + let page = self.pages.iter().position(|page| page.iova() == stream); + let patched = match page { + Some(index) if terminator == stream + end as u64 - 4 => + self.pages[index].with_bytes_mut(|bytes| { + let before = stream_words(bytes); + if before[0] >> 29 != 0 || before[10] != CDM_BARRIER + || before[11] != CDM_TERMINATE || bytes[end..].iter().any(|b| *b != 0) { + return Ok(None); + } + bytes[..end].fill(0); + bytes[..4].copy_from_slice(&CDM_BARRIER.to_le_bytes()); + bytes[4..8].copy_from_slice(&CDM_TERMINATE.to_le_bytes()); + Ok(Some((before, stream_words(bytes)))) + })?, + _ => None, + }; + let Some((before, after)) = patched else { + dev_err!(dev.as_ref(), + "G18_COMPUTE_VARIANT=1 refused: stream {:#x} terminator {:#x} is not the captured launch, barrier and terminate; nothing changed\n", + stream, terminator); + return Err(EINVAL); + }; + self.cdm_terminator = Some(stream + 4); + dev_info!(dev.as_ref(), + "G18_COMPUTE_VARIANT=1 empty stream {:#x}: words {:08x?} (were {:08x?}); command +0xee0 d4e0 {:#x} -> {:#x}; registers and all other bytes unchanged\n", + stream, after, before, terminator, stream + 4); + Ok(()) + } + + /// Validate every original site before mutating any page. All pages and + /// roots are still private; failure cannot publish a partial program. + fn triangle_shader(&mut self, dev: &AsahiDevice, base_fnv: u64) -> Result { + let firmware = Firmware::request(c"asahi/g18-26a428-compute-triangle.bin", dev.as_ref())?; + let patch = TrianglePatch::parse(firmware.data(), base_fnv).ok_or(EINVAL)?; + let mut indices = [0usize; PATCH_COUNT]; + for (i, index) in indices.iter_mut().enumerate() { + let site = patch.site(i).ok_or(EINVAL)?; + let base = site.address & !(payload::PAGE_BYTES as u64 - 1); + let offset = (site.address - base) as usize; + *index = self.pages.iter().position(|page| page.iova() == base).ok_or(EINVAL)?; + self.pages[*index].with_bytes_mut(|bytes| { + let before = bytes.get(offset..offset + site.bytes.len()).ok_or(EINVAL)?; + if payload::fnv64(before) != site.before_fnv { return Err(EINVAL); } + Ok(()) + })?; + } + for (i, index) in indices.into_iter().enumerate() { + let site = patch.site(i).ok_or(EINVAL)?; + let offset = (site.address & (payload::PAGE_BYTES as u64 - 1)) as usize; + self.pages[index].with_bytes_mut(|bytes| { + let target = bytes.get_mut(offset..offset + site.bytes.len()).ok_or(EINVAL)?; + target.copy_from_slice(site.bytes); + fence(Ordering::SeqCst); + for (j, expected) in site.bytes.iter().enumerate() { + // SAFETY: j is within this initialized, CPU-mapped slice. + if unsafe { core::ptr::read_volatile(target.as_ptr().add(j)) } != *expected { + return Err(EIO); + } + } + Ok(()) + })?; + dev_info!(dev.as_ref(), "G18_COMPUTE_VARIANT=2 patch {} address={:#x} bytes={} before_fnv={:#x} after_fnv={:#x}; VM unpublished\n", + i, site.address, site.bytes.len(), site.before_fnv, payload::fnv64(site.bytes)); + } + Ok(()) + } + + pub(crate) fn triangle_output_snapshot(&mut self) -> Result { + fence(Ordering::SeqCst); + self.pages[self.output].with_bytes_mut(|bytes| TriangleResult::inspect(bytes).ok_or(EINVAL)) + } + + /// Call only while the GPU is quiescent or after an independently verified + /// completion fence. This read does not establish completion itself. + pub(crate) fn output_snapshot(&mut self) -> Result { + fence(Ordering::SeqCst); + self.pages[self.output].with_bytes_mut(|bytes| + payload::ComputeResult::inspect(bytes).ok_or(EINVAL)) + } +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs new file mode 100644 index 0000000000000..cc6a35bd01669 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned H1 graph, typed mapping checks and a single audited publication. +#[path = "graphics_job_application.rs"] +pub(crate) mod application; +use crate::{g18::buffer::object::MappedObject, g18::runtime::graphics_memory::G18GraphicsMemory, + g18::queue::scheduler::{G18SchedulerMemory, G18SchedulerSnapshot}, + g18::queue::graphics::{G18GraphicsQueues, G18GraphicsQueueAddresses, + G18ParameterBuffer, G18PbAddresses, G18PbPolicy, G18QueuedPass, G18PbReadback}}; +use crate::{driver::AsahiDevice, g18::fw::render_params::G17pRenderParameters, + g18::address_map as va, g18::fw::render as regs, g18::fw::render_command as cmd, + g18::fw::render_assembly::{self as assembly, EncodedJob, RenderVariant}, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +pub(crate) struct G18GraphicsJob { + // Payload's context lease retracts before the additional job mappings drop. + program: G18GraphicsMemory, + aliases: KVec, + targets: KVec, + graph: MappedObject, + signals: MappedObject, + scheduler: G18SchedulerMemory, + queues: G18GraphicsQueues, + parameter_buffer: G18ParameterBuffer, + pub parameters: G17pRenderParameters, + pub controls: regs::G18RenderControls, + variant: RenderVariant, + encoded: Option, + audited: bool, +} + +impl G18GraphicsJob { + #[inline(never)] + pub(crate) fn from_payload(dev: &AsahiDevice, uat: &mmu::Uat, + profile: u32, variant: RenderVariant) -> Result> { + if (variant == RenderVariant::GearsThreeFrames) != (profile == 3) { return Err(EINVAL); } + let mut program = G18GraphicsMemory::load_payload(dev, uat, profile)?; + let capture = match program.capture() { + crate::g18::fw::graphics_payload::Capture::ColorA => regs::G18Capture::ColorA, + crate::g18::fw::graphics_payload::Capture::ColorB => regs::G18Capture::ColorB, + crate::g18::fw::graphics_payload::Capture::GearsG1 => regs::G18Capture::GearsG1, + crate::g18::fw::graphics_payload::Capture::FragmentWitness => regs::G18Capture::FragmentWitness, + }; + let (mut p, mut controls) = regs::g18_capture_parameters(capture); + if *crate::module_parameters::g18_vdm_high_alias.value() == 1 { + let source = p.encoder; + p.encoder = program.alias_vdm_for_address_probe(source)?; + dev_info!(dev, "G18_VDM_ADDRESS_CONTROL source={:#x} alias={:#x} compact={:#x} bytes=16384 contents=unchanged\n", + source, va::GRAPHICS_HIGH_VDM, p.encoder); + dev_info!(dev, "G18_RENDER_OBJECT name=vdm-address-control va={:#x} bytes={:#x} context={}\n", + va::GRAPHICS_HIGH_VDM, va::PAGE, program.context_id()); + } + if *crate::module_parameters::g18_ppp_high_alias.value() == 1 { + program.alias_ppp_for_address_probe()?; + dev_info!(dev, "G18_PPP_ADDRESS_CONTROL records=14 pages=4 contents=unchanged vdm_changes=ppp-pointers-only\n"); + for source in va::GRAPHICS_HIGH_PPP_PAGES { + dev_info!(dev, "G18_RENDER_OBJECT name=ppp-address-control-{:x} va={:#x} bytes={:#x} context={}\n", + source, va::GRAPHICS_HIGH_PPP_BASE - 0x58000 + source, va::PAGE, program.context_id()); + } + } + if matches!(variant, RenderVariant::TaOnly | RenderVariant::Clear) { + program.terminate_vdm(p.encoder)?; + } + if variant == RenderVariant::Clear { p.process_empty_tiles = true; } + let plan = assembly::target_plan(&p).ok_or(EINVAL)?; + let mut aliases = KVec::new(); + let mut targets = KVec::new(); + for i in 0..plan.sizes.len() { + let address = va::GRAPHICS_TARGET_OFFSET + plan.offsets[i]; + let mut target = MappedObject::new_wc_at(dev, program.vm(), address, + plan.sizes[i], mmu::PROT_GPU_SHARED_RW)?; + if i == 2 { + target.with_bytes_mut(|bytes| + regs::initialize_g18_deflake(bytes, p.width, p.height).ok_or(EINVAL))?; + } + aliases.push(target.map_alias_at(program.vm(), va::COMPACT_BIAS + address, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + targets.push(target, GFP_KERNEL)?; + } + assembly::apply_target_plan(&mut p, &plan); + if *crate::module_parameters::g18_usc_statistics.value() != 0 { + if variant != RenderVariant::Color || program.context_id() >= 64 { + return Err(EINVAL); + } + // Original G18 setupSMAPPDesc: full GPU VA bits 6..42, context + // in bits 56..61, enable bit 0. These are the separately owned, + // zeroed target pages above, retained for the entire job lifetime. + let encode = |address: u64| -> Result { + if address & 63 != 0 || address & !0x7ff_ffff_ffc0 != 0 { + return Err(EINVAL); + } + Ok(address | (u64::from(program.context_id()) << 56) | 1) + }; + controls.ta.usc_statistics = encode(p.ta_status)?; + controls.fragment.usc_statistics = encode(p.fragment_status)?; + dev_info!(dev, "G18_USC_STATISTICS context={} ta={:#x} fragment={:#x} backing=16384-each\n", + program.context_id(), controls.ta.usc_statistics, controls.fragment.usc_statistics); + } + dev_info!(dev, "G18_RENDER_TARGET kind={} primary={:#x} tpc-allocation={:#x} native={:?}\n", + crate::g18::uapi::render_policy::target_kind(), plan.sizes[0], plan.sizes[1], plan.native); + let mut parameter_buffer = G18ParameterBuffer::new(dev, uat, program.vm(), G18PbAddresses { + state_high: va::GRAPHICS_PB_HIGH, graph_high: va::PB_GRAPH, + page_list_high: va::PB_PAGE_LIST_HIGH, page_list_low: va::PB_PAGE_LIST_LOW, + page_list_row: va::COMPACT_BIAS + va::PB_PAGE_LIST_LOW, + metrics_high: va::PB_METRICS_HIGH, metrics_low: va::PB_METRICS_LOW, + scene_scratch_low: va::PB_SCENE_SCRATCH, blocks_low: va::PB_BLOCKS, + discard_low: va::PB_DISCARD, user_view_delta: 0, + }, G18PbPolicy { blocks: 8, capacity: 1650, pm_154: 550, pm_15c: 1650, + pm_1d4: 0, scene_count: 80, scene: 1, slot: 0, generation: 1 })?; + parameter_buffer.alias_low_window(program.vm(), program.vm(), va::COMPACT_BIAS)?; + p.cycle = parameter_buffer.scene_cycle(); + p.record_index = parameter_buffer.metrics_record() | 1; + p.ta_hardware_buffer_id = 0; + p.usc_flist_hardware_buffer_id = 0; + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), va::GRAPHICS_JOB_HIGH, + va::GRAPHICS_JOB_BYTES, mmu::PROT_FW_PRIV_RW)?; + for vm in [uat.kernel_lower_vm(), program.vm()] { + aliases.push(graph.map_alias_at(vm, va::GRAPHICS_JOB_LOW, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + } + let signals = MappedObject::new_wc_at(dev, uat.kernel_vm(), va::GRAPHICS_SIGNALS, + va::GRAPHICS_SIGNALS_BYTES, mmu::PROT_FW_SHARED_RW)?; + let mut scheduler = G18SchedulerMemory::new_render(dev, uat)?; + scheduler.prepare(0, 0)?; + let queues = G18GraphicsQueues::new(dev, uat, G18GraphicsQueueAddresses { + entry_high: [va::GRAPHICS_ENTRY_HIGH, va::GRAPHICS_ENTRY_HIGH + 0x20000], + entry_low: [va::GRAPHICS_ENTRY_LOW, va::GRAPHICS_ENTRY_LOW + 0x20000], + graph: va::GRAPHICS_QUEUE_HIGH, + })?; + let mut job = KBox::init(try_init!(Self { program, aliases, targets, graph, signals, scheduler, queues, parameter_buffer, + parameters: p, controls, variant, encoded: None, audited: false }), GFP_KERNEL)?; + job.audit_mappings(uat)?; + if job.program.output()?.1 != 65536 { return Err(EIO); } + job.log_objects(dev); + Ok(job) + } + + pub(crate) fn vm(&self) -> &mmu::Vm { self.program.vm() } + pub(crate) fn context_id(&self) -> u8 { self.program.context_id() } + pub(crate) fn scheduler_record(&self) -> Result { Ok(self.scheduler.binding(0)?.record) } + pub(crate) fn queues(&mut self) -> &mut G18GraphicsQueues { &mut self.queues } + pub(crate) fn parameter_buffer(&mut self) -> &mut G18ParameterBuffer { &mut self.parameter_buffer } + #[inline(never)] + pub(crate) fn encode_h1(&mut self, dev: &AsahiDevice) -> Result { + let inputs = self.h1_policy()?; + self.encode(dev, &inputs) + } + + #[inline(never)] + fn h1_policy(&mut self) -> Result> { + let context = u32::from(self.context_id()); + let inputs = crate::g18::uapi::render_policy::h1_inputs(self.program.command(), context, + &mut self.parameters, &mut self.controls); + Ok(KBox::new(inputs, GFP_KERNEL)?) + } + + pub(crate) fn pass(&self) -> Result { + Ok(G18QueuedPass { command_high: [va::GRAPHICS_JOB_HIGH, va::GRAPHICS_JOB_HIGH+0x4000], + context_id: u32::from(self.context_id()), scheduler_control: self.queues.scheduler_control(), + scheduler_index: 0, uma: cmd::G18ConfigUma { state: va::GRAPHICS_UMA_STATE, + generation: Some(0), slot: 0 }, parameter_buffer: Some(self.parameter_buffer.binding(10)), + rce: self.encoded.as_ref().ok_or(EINVAL)?.rce, stamps: [0x100;2], + event_generation: [0;2], ta_barrier_flag: crate::g18::uapi::render_policy::ta_barrier_flag() }) + } + + pub(crate) fn audit_h1(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + pb: &G18PbReadback, uma: &[u64;16]) -> Result { + let pass = self.pass()?; + self.queues.audit_before_pass(uat, &pass)?; + if u32::from_le_bytes(pb.state[8..12].try_into().map_err(|_|EINVAL)?) != pass.context_id + || pb.counter != 1 || uma.iter().all(|&v|v==0) { return Err(EIO); } + let s = self.scheduler.snapshot(0)?; + if s.submitted!=0 || s.completed!=0 || s.linked!=0 || self.program.output()?.1!=65536 { return Err(EBUSY); } + for q in va::QUEUE_IDS { for bank in [0x4000,0x8000] { + if self.signals.read_u32(va::GRAPHICS_SIGNALS+bank+u64::from(q)*4)?!=0 {return Err(EBUSY);} + } } + for off in [0xc000,0xc008,0xc010,0xc018] { + if self.signals.read_u64(va::GRAPHICS_SIGNALS+off)?!=0 {return Err(EBUSY);} + } + let mut pages = self.audit_mappings(uat)?; + pages += self.parameter_buffer.audit_job_mappings(uat,self.vm(),va::COMPACT_BIAS)?; + // Command/RCE/target/PB/UMA graphs are owner-audited page by page. + // Fixed firmware-only objects not covered by those owners: + for (address,bytes) in [(va::GRAPHICS_SIGNALS,va::GRAPHICS_SIGNALS_BYTES), + (va::GRAPHICS_SCHEDULER_START,0x8000),(va::PB_TABLE,0x4000), + (va::UMA_TABLE,0x8000),(va::GRAPHICS_UMA_STATE,0x8000)] { + for off in (0..bytes).step_by(mmu::UAT_PGSZ) { uat.kernel_vm().translate_iova(address+off as u64)?;pages+=1; } + } + // Captured shader/VDM/PPP pages are immutable except terminate/poison. + // Validate all enabled direct register pointer inputs; bind/flag words + // and shader offsets are not mistaken for full addresses. + let p = &self.parameters; + // The register carries a compact VDM offset, not a full GPU VA. + // The low fixture alias used to hide this distinction in the audit. + let vdm_address = va::COMPACT_BIAS.checked_add(p.encoder).ok_or(EINVAL)?; + self.vm().translate_iova(vdm_address)?; + pages += 1; + for address in [p.tilemap,p.heapmeta,p.layermeta,p.tpc,p.deflake_1,p.deflake_2,p.deflake_3, + p.ta_status,p.fragment_status,p.scissor_array,p.depth_bias_array,p.aux_fb, + p.occlusion_query_base,p.depth_buffer,p.stencil_buffer,p.depth_aux_buffer,p.stencil_aux_buffer, + p.sampler_array,p.cycle,p.record_index & !1,p.load_pipeline,p.store_pipeline, + p.partial_load_pipeline,p.partial_store_pipeline] { + if address!=0 { self.vm().translate_iova(address)?; pages+=1; } + } + let inputs=self.h1_policy()?; + let f=&inputs.fragment; + // Native descriptor copies of attachment/resource and shader pointers. + // Other d0518 pairs are disabled in the admitted colour captures; the + // last pair is a packed format word rather than a GPU pointer. + for address in [f.d04e0,f.d04e8,f.d04f0,f.d0518[18][1],f.d0518[21][1],f.d0790[1],f.d07b0[1]] { + if address!=0 {self.vm().translate_iova(address)?;pages+=1;} + } + for (index,pair) in f.d0518.iter().enumerate() { + if ![18,21,22].contains(&index) && *pair!=[0;2] {return Err(ENOTSUPP);} + } + self.parameter_buffer.print_h1_digests(dev,&pb.row)?; + self.scheduler.print_render_digests(dev)?; + for (target,name) in self.targets.iter_mut().zip(assembly::TARGET_NAMES) { + target.with_bytes_mut(|bytes| { + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name={} fnv={:#018x}\n",name,crate::g18::fw::graphics_payload::fnv64(bytes));Ok(()) + })?; + } + if self.variant==RenderVariant::Clear { + let digest=self.program.terminated_digest(self.parameters.encoder)?; + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=terminated-vdm-page fnv={:#018x}\n",digest); + } + let expected=self.encoded.as_ref().ok_or(EINVAL)?.digests; + self.graph.with_bytes_mut(|bytes| { + let ta=&bytes[..cmd::G18_TA_COMMAND_BYTES]; + let fr=&bytes[assembly::FRAGMENT_OFFSET..assembly::FRAGMENT_OFFSET+cmd::G18_3D_COMMAND_BYTES]; + let f=crate::g18::fw::graphics_payload::fnv64; + let actual=[f(ta),f(fr),f(&ta[0x60..0x60+91*12]),f(&fr[0xa0..0xa0+92*12]), + f(&fr[0x7c0..0x7c0+17*12]),f(&fr[0xee0..0xee0+24*12]),f(&fr[0x1600..0x1600+12*12])]; + if actual!=expected {return Err(EIO);} + for (name,digest) in assembly::DIGEST_NAMES.iter().zip(actual) { + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name={} fnv={:#018x}\n",name,digest); + } + Ok(()) + })?; + let mut row=[0u8;128]; for (out,word) in row.chunks_exact_mut(8).zip(uma) {out.copy_from_slice(&word.to_le_bytes());} + dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name=uma-row fnv={:#018x}\n", crate::g18::fw::graphics_payload::fnv64(&row)); + dev_info!(dev.as_ref(), "G18_RENDER_AUDIT pb=match uma=match slots=0,1 scheduler=bound stamps=zero timestamps=zero poison=65536 translations={} policy=captured-M5-H1\n",pages); + self.audited = true; + Ok(()) + } + + pub(crate) fn publish_h1(&mut self) -> Result<[u64;2]> { + if !self.audited {return Err(EINVAL);} + self.program.publish()?; + self.scheduler.publish_kick(0)?; + self.scheduler.publish_kick(0)?; + self.audited=false; + Ok(self.program.roots()) + } + + pub(crate) fn stamps(&mut self) -> Result<[u32;2]> { + Ok([self.signals.read_u32(va::GRAPHICS_SIGNALS+0x4000+9*4)?, + self.signals.read_u32(va::GRAPHICS_SIGNALS+0x4000+10*4)?]) + } + /// A firmware recovery ended this frame after its queues consumed it, so + /// no stage will write its stamps. Publish them from the host; a stage + /// that already wrote its own must have written exactly `stamp`. + pub(crate) fn publish_recovered_stamps(&mut self, previous: u32, stamp: u32) -> Result { + for slot in [9u64,10] { + let address=va::GRAPHICS_SIGNALS+0x4000+slot*4; + match self.signals.read_u32(address)? { + value if value==stamp => {} + value if value==previous => self.signals.write_u32(address,stamp)?, + _ => return Err(EIO), + } + } + fence(Ordering::SeqCst); + Ok(()) + } + pub(crate) fn pixels(&mut self) -> Result<([u32;8],u32,u32)> { + self.program.verify_pixels(self.variant==RenderVariant::Clear) + } + pub(crate) fn capture_fields(&self) -> (cmd::G18RenderCommandInputs, usize) { + let (mut inputs, open) = crate::g18::fw::render_passthrough::capture_fields(self.program.command()); + let pb = self.parameter_buffer.command_fields(); + inputs.ta.pb_slot = pb.pb_slot; inputs.ta.pm_1d4 = pb.pm_1d4; + inputs.ta.pb_state = pb.pb_state; inputs.fragment.pm_state = pb.pb_state; + inputs.ta.pm_scene = pb.pm_scene; inputs.fragment.pm_scene = pb.pm_scene; + inputs.ta.pm_b8 = pb.pm_b8; inputs.fragment.pm_b8 = pb.pm_b8; + (inputs, open) + } + + /// Caller has allocated/retained PB and pool owners and filled all native + /// fields. In particular a default resource word is not an admitted policy. + #[inline(never)] + pub(crate) fn encode(&mut self, dev: &AsahiDevice, inputs: &cmd::G18RenderCommandInputs) -> Result { + if self.encoded.is_some() || self.controls.ta.resource_word == 0 + || self.controls.fragment.resource_word == 0 + || inputs.context_id != u32::from(self.context_id()) + || inputs.scheduler_record != self.scheduler_record()? + || inputs.ta.pb_state == 0 || inputs.ta.pb_state != inputs.fragment.pm_state + || inputs.ta.uma_state == 0 || inputs.ta.uma_state != inputs.fragment.uma_state { + return Err(EINVAL); + } + let mut workspace = KBox::::new_uninit(GFP_KERNEL)?; + // SAFETY: Workspace contains only integer scalars and integer arrays; + // zero is valid for every field. Initialize the allocation in place. + unsafe { core::ptr::write_bytes(workspace.as_mut_ptr(), 0, 1); } + let mut workspace = unsafe { workspace.assume_init() }; + let encoded = self.graph.with_bytes_mut(|bytes| { + let (ta_page, rest) = bytes.split_at_mut(assembly::FRAGMENT_OFFSET); + assembly::encode_into((&mut ta_page[..cmd::G18_TA_COMMAND_BYTES]).try_into().map_err(|_| EINVAL)?, + (&mut rest[..cmd::G18_3D_COMMAND_BYTES]).try_into().map_err(|_| EINVAL)?, + &self.parameters, &self.controls, inputs, &mut workspace).map_err(|_| EINVAL) + })?; + self.encoded = Some(encoded); + fence(Ordering::SeqCst); + Ok(()) + } + + fn audit_mappings(&self, uat: &mmu::Uat) -> Result { + let mut count = self.program.audit()?; + for target in self.targets.iter() { + for offset in (0..target.logical_size).step_by(mmu::UAT_PGSZ) { + let low = target.iova() + offset as u64; + if self.vm().translate_iova(low)? != self.vm().translate_iova(va::COMPACT_BIAS + low)? { + return Err(EFAULT); + } + count += 1; + } + } + for offset in (0..va::GRAPHICS_JOB_BYTES).step_by(mmu::UAT_PGSZ) { + let pa = uat.kernel_vm().translate_iova(va::GRAPHICS_JOB_HIGH + offset as u64)?; + for vm in [uat.kernel_lower_vm(), self.vm()] { + if vm.translate_iova(va::GRAPHICS_JOB_LOW + offset as u64)? != pa { return Err(EFAULT); } + } + count += 1; + } + Ok(count) + } + + pub(crate) fn scheduler_snapshot(&mut self) -> Result { self.scheduler.snapshot(0) } + pub(crate) fn output(&mut self) -> Result<(u64, u32)> { self.program.output() } + + fn log_objects(&self, dev: &AsahiDevice) { + for (i, target) in self.targets.iter().enumerate() { + dev_info!(dev.as_ref(), "G18_RENDER_OBJECT name={} va={:#x} bytes={:#x} context={}\n", + assembly::TARGET_NAMES[i], target.iova(), target.logical_size, self.context_id()); + } + for (name, address, bytes, context) in [ + ("queue-state", va::GRAPHICS_QUEUE_HIGH, 0x8000, 0), + ("ta-entry", va::GRAPHICS_ENTRY_HIGH, 0x20000, 0), + ("3d-entry", va::GRAPHICS_ENTRY_HIGH + 0x20000, 0x20000, 0), + ("ta-command", va::GRAPHICS_JOB_HIGH, cmd::G18_TA_COMMAND_BYTES, 0), + ("3d-command", va::GRAPHICS_JOB_HIGH + 0x4000, cmd::G18_3D_COMMAND_BYTES, 0), + ("signals", va::GRAPHICS_SIGNALS, va::GRAPHICS_SIGNALS_BYTES, 0), + ("scheduler", va::GRAPHICS_SCHEDULER_START + 0x4000, 0x2400, 0), + ("submitted", va::GRAPHICS_SCHEDULER_START, 0x90, 0), + ("pb-table", va::PB_TABLE, 0x4000, 0), ("uma-table", va::UMA_TABLE, 0x8000, 0), + ("pb-state", va::GRAPHICS_PB_HIGH, 0x4000, 0), + ("pb-graph", va::PB_GRAPH, 0x8000, 0), + ("pb-page-list", va::PB_PAGE_LIST_LOW, va::PB_PAGE_LIST_BYTES, self.context_id()), + ("pb-metrics", va::PB_METRICS_LOW, 0x4000, self.context_id()), + ("pb-scene-scratch", va::PB_SCENE_SCRATCH, 0x4000, self.context_id()), + ("pb-discard", va::PB_DISCARD, 0x8000, self.context_id()), + ("pb-blocks", va::PB_BLOCKS, 8*0x80000, self.context_id()), + ("uma-state", va::GRAPHICS_UMA_STATE, 0xa0, 0), + ("uma-counter", va::GRAPHICS_UMA_COUNTER, 8, 0), + ("uma-list", va::GRAPHICS_UMA_LIST, 0x2000, 0), + ("pixels", crate::g18::fw::graphics_payload::OUTPUT, 0x40000, self.context_id()), + ] { + dev_info!(dev.as_ref(), "G18_RENDER_OBJECT name={} va={:#x} bytes={:#x} context={}\n", + name, address, bytes, context); + } + for index in 0..crate::g18::uapi::render_policy::uma_blocks() { + dev_info!(dev.as_ref(), "G18_RENDER_OBJECT name=uma-block-{} va={:#x} bytes=0x100000 context={}\n", + index,crate::g18::uapi::render_policy::block_base()+u64::from(index)*va::POOL_BLOCK_STRIDE,self.context_id()); + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs new file mode 100644 index 0000000000000..fc9cb879637ea --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Application lifetime around the H1 allocation graph. The fixture VM stays +//! unpublished storage; commands are rebuilt entirely from Mesa parameters. +use super::*; +use crate::{g18::uapi::application::G18ApplicationRender,g18::uapi::application_runtime::{self as g18_application_runtime,Frame}}; + +pub(crate) struct ApplicationJob { + pub(crate) job: KBox, + storage: mmu::Vm, + user_aliases: KVec, + pub(crate) pool_aliases: KVec, + guard: Option, + scene: [u8;0x80], + generation: u32, + retired: bool, + /// Frames of the current client VM, counting this one. + client_frames: u32, + /// The current render's adapter-derived private-memory selection. + scratch: Option, +} + +impl ApplicationJob { + pub(crate) fn new(mut job: KBox) -> Result> { + let storage=job.vm().clone(); + let scene=job.parameter_buffer.application_scene(None)?; + // Remove the fixture-only aliases before target growth. The original + // payload and PB backing stay retained but no capture bytes execute. + job.aliases.clear(); + Ok(KBox::new(Self {job,storage,user_aliases:KVec::new(),pool_aliases:KVec::new(), + guard:None,scene,generation:0,retired:true,client_frames:0,scratch:None},GFP_KERNEL)?) + } + pub(crate) fn frame(&self) -> Frame {Frame::new(self.generation).expect("admitted frame")} + /// The last frame kicked to both graphics queues (0 before the first). + pub(crate) fn submitted_generation(&self) -> u32 {self.generation} + pub(crate) fn quarantine(&mut self) { + if let Some(guard)=&self.guard {guard.quarantine();} + if let Some(status)=self.job.vm().status(){status.record(EIO);} + } + pub(crate) fn retired(&mut self) {self.retired=true;self.guard=None;} + /// One of the first frames of its client: logged even when job logging + /// is sparse, so every isolated client leaves a kick/retirement pair. + pub(crate) fn traced(&self) -> bool {self.client_frames<=3} + + #[inline(never)] + pub(crate) fn prepare(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + vm: &mmu::Vm, render: &G18ApplicationRender) -> Result { + if !self.retired {return Err(EBUSY);} + let frame=Frame::new(self.generation.checked_add(1).ok_or(EOVERFLOW)?).ok_or(EOVERFLOW)?; + let plan=assembly::target_plan(&render.parameters).ok_or(EINVAL)?; + // Growth retains each object's original IOVA. Reject a changed target + // policy before touching mappings rather than encode unowned addresses. + if self.job.targets.len()!=plan.offsets.len() + || self.job.targets.iter().zip(plan.offsets) + .any(|(target,offset)|target.iova()!=va::GRAPHICS_TARGET_OFFSET+offset) { + return Err(EINVAL); + } + let guard=vm.retain_t8140_job()?; + let new_client=!self.job.program.matches_vm(vm); + let remap=self.generation==0 || new_client + || self.job.targets.iter().zip(plan.sizes).any(|(target,size)|target.logical_size Result { + let inputs=self.application_policy(frame)?; + self.job.encode(dev,&inputs) + } + + // Return the heap owner before entering the register builders. Otherwise + // LLVM keeps the ~3.6 KiB command-input construction frame beneath them. + #[inline(never)] + fn application_policy(&mut self, frame: Frame) -> Result> { + let context=u32::from(self.job.context_id()); + let scratch=self.scratch; + let job=&mut *self.job; + let mut inputs=KBox::new(g18_application_runtime::inputs(&mut job.parameters,&mut job.controls,context,frame),GFP_KERNEL)?; + // A render whose programs carry a thread-private frame takes both + // launch pairs from its own plan; the TA/3D pool stays the owned one. + // The adapter admitted it only at one sample, so no 4x override exists. + if let Some(selection)=scratch { + if job.controls.fragment_sample_control!=0 {return Err(EINVAL);} + job.controls.ta.resource_word=selection.stages[0].resource_word(); + job.controls.fragment.resource_word=selection.stages[1].resource_word(); + } + let pb=self.job.parameter_buffer.command_fields(); + inputs.ta.pb_slot=pb.pb_slot;inputs.ta.pm_1d4=pb.pm_1d4; + inputs.ta.pb_state=pb.pb_state;inputs.fragment.pm_state=pb.pb_state; + inputs.ta.pm_scene=pb.pm_scene;inputs.fragment.pm_scene=pb.pm_scene; + inputs.ta.pm_b8=pb.pm_b8;inputs.fragment.pm_b8=pb.pm_b8; + inputs.scheduler_record=self.job.scheduler_record()?; + Ok(inputs) + } + + pub(crate) fn pass(&self) -> Result { + let mut pass=self.job.pass()?;pass.stamps=[self.frame().stamp();2];Ok(pass) + } + + pub(crate) fn audit(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + pb: &G18PbReadback, uma: &[u64;16], mut pages: usize) -> Result { + let frame=self.frame();let pass=self.pass()?; + if frame.generation==1 {self.job.queues.audit_before_pass(uat,&pass)?;} + if pb.counter!=frame.generation || uma.iter().all(|v|*v==0) + || u32::from_le_bytes(pb.state[8..12].try_into().map_err(|_|EINVAL)?)!=pass.context_id {return Err(EIO);} + let scheduler=self.job.scheduler.snapshot(0)?; + if scheduler.submitted!=frame.submitted()-2 || scheduler.completed!=frame.submitted()-2 + || scheduler.linked!=0 || self.job.stamps()?!=[frame.previous_stamp();2] {return Err(EBUSY);} + for target in &self.job.targets { + for off in (0..target.logical_size).step_by(mmu::UAT_PGSZ) { + let low=target.iova()+off as u64;let pa=self.storage.translate_iova(low)?; + if self.job.vm().translate_iova(low)?!=pa || self.job.vm().translate_iova(low+va::COMPACT_BIAS)?!=pa {return Err(EFAULT);}pages+=1; + } + } + pages+=self.job.parameter_buffer.audit_job_mappings(uat,self.job.vm(),va::COMPACT_BIAS)?; + for off in (0..va::GRAPHICS_JOB_BYTES).step_by(mmu::UAT_PGSZ) { + let pa=uat.kernel_vm().translate_iova(va::GRAPHICS_JOB_HIGH+off as u64)?; + for vm in [uat.kernel_lower_vm(),self.job.vm()] { + if vm.translate_iova(va::GRAPHICS_JOB_LOW+off as u64)?!=pa {return Err(EFAULT);} + }pages+=1; + } + for (address,bytes) in [(va::GRAPHICS_SIGNALS,va::GRAPHICS_SIGNALS_BYTES), + (va::GRAPHICS_SCHEDULER_START,0x8000),(va::PB_TABLE,0x4000),(va::UMA_TABLE,0x8000)] { + for off in (0..bytes).step_by(mmu::UAT_PGSZ) {uat.kernel_vm().translate_iova(address+off as u64)?;pages+=1;} + } + let expected=self.job.encoded.as_ref().ok_or(EINVAL)?.digests; + self.job.graph.with_bytes_mut(|out| { + if crate::g18::fw::graphics_payload::fnv64(&out[..cmd::G18_TA_COMMAND_BYTES])!=expected[0] + || crate::g18::fw::graphics_payload::fnv64(&out[0x4000..0x4000+cmd::G18_3D_COMMAND_BYTES])!=expected[1] {return Err(EIO);}Ok(()) + })?; + if frame.generation<=3 { + for (name,digest) in assembly::DIGEST_NAMES.iter().zip(expected) { + dev_info!(dev.as_ref(),"G18_APP_DIGEST frame={} name={} fnv={:#018x}\n",frame.generation,name,digest); + } + dev_info!(dev.as_ref(),"G18_APP_AUDIT frame={} pb=resident uma=match translations={} previous_stamp={:#x}\n",frame.generation,pages,frame.previous_stamp()); + } + if frame.generation==1 { + self.job.parameter_buffer.print_h1_digests(dev,&pb.row)?; + self.job.scheduler.print_render_digests(dev)?; + } + self.job.audited=true;Ok(()) + } + + pub(crate) fn publish(&mut self) -> Result<[u64;2]> {self.job.publish_h1()} +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs new file mode 100644 index 0000000000000..222352d1360bd --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained G18GFX01 v2 pages. Construction never publishes roots or work. +use crate::g18::buffer::object::MappedObject; +use crate::{driver::AsahiDevice, g18::address_map as va, g18::fw::graphics_payload as payload, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{firmware::Firmware, prelude::*}; + +pub(crate) struct G18GraphicsMemory { + // Root lease retracts before aliases/backing can drop. The live session + // retains the whole job even after faults; this order also protects errors + // during construction before publication. + context: mmu::M5ExecutionContext, + aliases: KVec, + vdm_alias: Option<(u64, u64)>, + ppp_aliased: bool, + pages: KVec, + capture: payload::Capture, + body_digest: u64, + command_digest: u64, + command: KVec, +} + +impl G18GraphicsMemory { + pub(crate) fn load_payload(dev: &AsahiDevice, uat: &mmu::Uat, profile: u32) -> Result { + let name = match profile { + 1 => c"asahi/g18-26a428-color-a-v2.bin", + 2 => c"asahi/g18-26a428-color-b-v2.bin", + 3 => c"asahi/g18-26a428-gears-g1-v2.bin", + 4 => c"asahi/g18-26a428-fragment-witness-v2.bin", + _ => return Err(EINVAL), + }; + if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != payload::PAGE_BYTES { return Err(ENODEV); } + let firmware = Firmware::request(name, dev.as_ref())?; + let input = payload::GraphicsPayload::parse(firmware.data()).map_err(|_| EINVAL)?; + if input.capture() as u32 != profile { return Err(EINVAL); } + let vm = uat.new_t8152_compute_fixture_vm()?; + let context = uat.prepare_t8152_execution_context(&vm)?; + let mut pages = KVec::new(); + let mut aliases = KVec::new(); + for index in 0..input.page_count() { + let source = input.page(index).ok_or(EINVAL)?; + // Captured usage flags are not PTE permissions. These private + // fixture mappings have an explicit diagnostic RW policy. + let mut page = MappedObject::new_wc_at(dev, &vm, source.address, + payload::PAGE_BYTES, mmu::PROT_GPU_SHARED_RW)?; + page.with_bytes_mut(|out| { + out.copy_from_slice(source.bytes); + if (payload::OUTPUT..payload::OUTPUT + payload::OUTPUT_BYTES).contains(&source.address) { + out.fill(0xa5); + } + Ok(()) + })?; + if source.address < 0x78000 { + aliases.push(page.map_alias_at(&vm, va::COMPACT_BIAS + source.address, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for delta in [0, va::PAGE - 1] { + if vm.translate_iova(source.address + delta)? + != vm.translate_iova(va::COMPACT_BIAS + source.address + delta)? { + return Err(EFAULT); + } + } + } + vm.translate_iova(source.address)?; + vm.translate_iova(source.address + va::PAGE - 1)?; + pages.push(page, GFP_KERNEL)?; + } + let mut command = KVec::new(); + command.extend_from_slice(input.command(), GFP_KERNEL)?; + let owner = Self { context, aliases, vdm_alias: None, ppp_aliased: false, pages, capture: input.capture(), command, + body_digest: input.body_digest(), command_digest: payload::fnv64(input.command()) }; + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), "G18_RENDER_PAYLOAD profile={} pages={} low_aliases={} body_fnv={:#018x} command_fnv={:#018x} context={} roots={:x?}; unpublished\n", + profile, owner.pages.len(), owner.aliases.len(), owner.body_digest, owner.command_digest, + owner.context_id(), owner.roots()); + Ok(owner) + } + + pub(crate) fn vm(&self) -> &mmu::Vm { self.context.vm() } + pub(crate) fn context_id(&self) -> u8 { self.context.context_id() } + pub(crate) fn roots(&self) -> [u64; 2] { self.context.roots() } + pub(crate) fn capture(&self) -> payload::Capture { self.capture } + pub(crate) fn command(&self) -> &[u8] { &self.command } + pub(crate) fn publish(&mut self) -> Result { self.context.publish() } + /// Called only after both graphics queues have independently retired. + pub(crate) fn rebind_retired(&mut self, vm: &mmu::Vm) -> Result { self.context.rebind(vm) } + pub(crate) fn matches_vm(&self, vm: &mmu::Vm) -> bool { self.context.matches(vm) } + pub(crate) fn terminated_digest(&mut self, address: u64) -> Result { + let base=address & !(va::PAGE-1); + let page=self.pages.iter_mut().find(|p|p.iova()==base).ok_or(EFAULT)?; + page.with_bytes_mut(|bytes|Ok(payload::fnv64(bytes))) + } + + pub(crate) fn audit(&self) -> Result { + let mut checked = 0; + for page in self.pages.iter() { + for offset in [0, va::PAGE - 1] { + let address = page.iova() + offset; + self.vm().translate_iova(address)?; + if page.iova() < 0x78000 && self.vm().translate_iova(address)? + != self.vm().translate_iova(address + va::COMPACT_BIAS)? { return Err(EFAULT); } + checked += 1; + } + } + if let Some((source, target)) = self.vdm_alias { + for offset in [0, va::PAGE - 1] { + if self.vm().translate_iova(source + offset)? != + self.vm().translate_iova(target + offset)? { return Err(EFAULT); } + checked += 1; + } + } + if self.ppp_aliased { + for source in va::GRAPHICS_HIGH_PPP_PAGES { + let target = va::GRAPHICS_HIGH_PPP_BASE - 0x58000 + source; + for offset in [0, va::PAGE - 1] { + if self.vm().translate_iova(source + offset)? != + self.vm().translate_iova(target + offset)? { return Err(EFAULT); } + checked += 1; + } + } + } + Ok(checked) + } + + /// Isolate PPP address encoding using the pinned native witness. Only the + /// pointer bits in its 14 PPP records change; the pages themselves alias + /// their original backing. No shader or PPP payload is rewritten. + pub(crate) fn alias_ppp_for_address_probe(&mut self) -> Result { + if self.context.is_current() || self.ppp_aliased { return Err(EBUSY); } + if self.capture != payload::Capture::FragmentWitness || + self.vdm_alias != Some((0x18000, va::GRAPHICS_HIGH_VDM)) { return Err(EINVAL); } + const RECORDS: [(usize, u32, u32); 14] = [ + (0x00, 0x58000, 6), (0x08, 0x58018, 33), (0x10, 0x5809c, 33), + (0x34, 0x40, 5), (0x3c, 0x58120, 7), (0x44, 0x5813c, 6), + (0x4c, 0x58154, 5), (0x54, 0x58168, 7), (0x5c, 0x58184, 5), + (0x64, 0x6ac00, 10), (0x6c, 0x58198, 3), (0x74, 0x581a4, 2), + (0x7c, 0x48000, 5), (0x84, 0x581ac, 3), + ]; + self.pages.iter_mut().find(|p| p.iova() == 0x18000).ok_or(EFAULT)? + .with_bytes_mut(|bytes| { + if payload::fnv64(bytes) != 0x6f21_d211_cfaf_7af1 { return Err(EINVAL); } + for (offset, pointer, words) in RECORDS { + if bytes[offset..offset + 4] != (words << 8).to_le_bytes() || + bytes[offset + 4..offset + 8] != pointer.to_le_bytes() { return Err(EINVAL); } + } + Ok(()) + })?; + let vm = self.context.vm(); + for source in va::GRAPHICS_HIGH_PPP_PAGES { + let target = va::GRAPHICS_HIGH_PPP_BASE - 0x58000 + source; + let page = self.pages.iter_mut().find(|p| p.iova() == source).ok_or(EFAULT)?; + self.aliases.push(page.map_alias_at(vm, target, mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + } + self.ppp_aliased = true; + self.audit()?; + self.pages.iter_mut().find(|p| p.iova() == 0x18000).ok_or(EFAULT)? + .with_bytes_mut(|bytes| { + for (offset, pointer, words) in RECORDS { + let compact = va::GRAPHICS_HIGH_PPP_BASE - 0x58000 - va::COMPACT_BIAS + u64::from(pointer); + bytes[offset..offset + 4].copy_from_slice(&((words << 8) | (compact >> 32) as u32).to_le_bytes()); + bytes[offset + 4..offset + 8].copy_from_slice(&(compact as u32).to_le_bytes()); + } + Ok(()) + })?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Retain a second mapping of the unchanged native VDM page. This isolates + /// the address used for fetching commands from their contents and backing. + pub(crate) fn alias_vdm_for_address_probe(&mut self, address: u64) -> Result { + if self.context.is_current() || self.vdm_alias.is_some() { + return Err(EBUSY); + } + if self.capture != payload::Capture::FragmentWitness || address != 0x18000 || + va::GRAPHICS_HIGH_VDM & (va::PAGE - 1) != 0 { return Err(EINVAL); } + let vm = self.context.vm(); + let page = self.pages.iter_mut().find(|page| page.iova() == address).ok_or(EFAULT)?; + let alias = page.map_alias_at(vm, va::GRAPHICS_HIGH_VDM, + mmu::PROT_GPU_SHARED_RW)?; + self.aliases.push(alias, GFP_KERNEL)?; + self.vdm_alias = Some((address, va::GRAPHICS_HIGH_VDM)); + self.audit()?; + Ok(va::GRAPHICS_HIGH_VDM - va::COMPACT_BIAS) + } + + /// The caller supplies the register builder's decoded VDM stream address. + /// A standalone termination word affects no shaders or other capture data. + pub(crate) fn terminate_vdm(&mut self, address: u64) -> Result { + if self.context.is_current() || address & 3 != 0 { return Err(EBUSY); } + let address = if (va::COMPACT_BIAS..va::COMPACT_BIAS + 0x78000).contains(&address) { + address - va::COMPACT_BIAS + } else { address }; + let base = address & !(va::PAGE - 1); + let page = self.pages.iter_mut().find(|page| page.iova() == base).ok_or(EFAULT)?; + page.write_u32(address, 6 << 29)?; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Output digest is observation only, never a completion predicate. + pub(crate) fn verify_pixels(&mut self, clear: bool) -> Result<([u32;8],u32,u32)> { + let mut sha = crate::g18::fixture::render_pixels::Pixels::new(); + for index in 0..16 { + let address = payload::OUTPUT + index * va::PAGE; + let page = self.pages.iter_mut().find(|p|p.iova()==address).ok_or(EFAULT)?; + page.with_bytes_mut(|bytes| { + if !sha.page(bytes) {return Err(EINVAL);} Ok(()) + })?; + } + let poison=sha.poison; let background=sha.background; + let digest=sha.finish().ok_or(EINVAL)?; + if poison!=0 || (clear && background!=65536) + || (!clear && (background!=47104 || digest!=crate::g18::fixture::render_pixels::COLOR)) {return Err(EIO);} + Ok((digest,background,poison)) + } + + /// Output digest is observation only, never a completion predicate. + pub(crate) fn output(&mut self) -> Result<(u64, u32)> { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + let mut poison = 0; + let mut count = 0; + for page in self.pages.iter_mut().filter(|p| + (payload::OUTPUT..payload::OUTPUT + payload::OUTPUT_BYTES).contains(&p.iova())) { + page.with_bytes_mut(|bytes| { + for word in bytes.chunks_exact(4) { + // SAFETY: aligned four-byte word inside retained WC RAM. + let value = unsafe { core::ptr::read_volatile(word.as_ptr().cast::()) }; + if value == 0xa5a5_a5a5 { poison += 1; } + for b in value.to_le_bytes() { hash = (hash ^ u64::from(b)).wrapping_mul(0x100_0000_01b3); } + count += 1; + } + Ok(()) + })?; + } + if count != 65536 { return Err(EFAULT); } + fence(Ordering::Acquire); + Ok((hash, poison)) + } +} diff --git a/drivers/gpu/drm/asahi/g18/runtime/mod.rs b/drivers/gpu/drm/asahi/g18/runtime/mod.rs new file mode 100644 index 0000000000000..d54b26bb187df --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/runtime/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 job storage: graphics and compute job memory and encoders. + +#[allow(dead_code)] +pub(crate) mod compute; +#[allow(dead_code)] +pub(crate) mod compute_payload; +#[allow(dead_code)] +pub(crate) mod graphics_job; +#[allow(dead_code)] +pub(crate) mod graphics_memory; diff --git a/drivers/gpu/drm/asahi/g18/uapi/application.rs b/drivers/gpu/drm/asahi/g18/uapi/application.rs new file mode 100644 index 0000000000000..f3de19433a52a --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/application.rs @@ -0,0 +1,599 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G18 application path: one decoded Mesa render command -> G18 job inputs. +//! +//! This is the DRM counterpart of `g18_render::g18_capture_parameters`. +//! Where the capture helper inverts Apple's native gears/colour descriptors, +//! this module starts from the UAPI `drm_asahi_cmd_render` that Mesa's G18G +//! (M5-path) Gallium driver submits and produces: +//! +//! - `G17pRenderParameters` with every user field translated by the shared +//! `g17_uapi::translate_render_command` (the M5-proven parser and range +//! checks) and the G18 adaptations below; +//! - `G18RenderControls` from `g18_render::g18_default_controls`; +//! - every job-owned field left at zero (see [`job_owned_fields_clear`]) for +//! the job (`G18RenderJobSink`) to allocate and fill. +//! +//! G18 adaptations, each against a named source: +//! - VDM stream: the UAPI carries the full GART-alias VA; `g18_render` takes +//! the native compact view (the render-register notes, "Kernel-owned inputs"), +//! exactly Mesa's `agx_m5_vdm_compact` (`agx_m5_va.h:34-41`). +//! - aux_fb_flags 0x30c000: indexed multi-draw, the gears-g1 value +//! (`g18_render.rs:490-494`), where M5 used 0x10c000 (`m5_drm.rs:251`). +//! - Main-pass tile flags 0xc00000 (native +8d2/+8d3): present in all three +//! G18 captures (`g18_render.rs:489`, tile_config 0xc10280), dropped by the +//! partial helper (`g18_render.rs:396`). The UAPI cannot express them. +//! - Single-layer ZLS array strides are zeroed, as on M5 (`m5_drm.rs:238-241`). +//! - Uncompressed Z16 uses the shared range-checked ZLS format and preserves +//! ISP bit 18. AGXMetalG18G selects that bit for Depth16Unorm at +//! 0x2196383e8..0x219638400 (the indirect DBIAS/PBE notes, section 2). +//! - msaa163, only with `g18_admit_msaa4=1`: the native 4x render set +//! (`is_msaa4_command`) is admitted with the native sample control +//! 0x40000; every other multisampled command keeps its refusal. +//! - Native682 layered Z32/S8 uses cacheline or page array strides and +//! independently ranged metadata. Single-layer compressed stencil retains +//! its previous inert behavior. +//! `isp_bgobjvals` 0x300 is re-asserted by the register builder; its low +//! byte carries the application's stencil clear value. +//! +//! Pure: no allocation, MMIO, firmware or VM access. The job boundary at the +//! end of the file is the only kernel-typed item (a host parity +//! check compiles it against a stub `kernel` crate). + +use crate::{g18::fw::render_params::G17pRenderParameters, g18::uapi::command as g17_uapi, g18::fw::render as g18_render, g18::fw::vm_layout as m5_vm}; + +/// Largest framebuffer side the target layouts and merge coefficients encode. +/// Whether a target fits is the job's own target plan +/// (`g18_render_assembly::target_plan`), which admission also requires: its +/// TPC slot is 0x20000 bytes, which the conformance FBO surface (64 x +/// GL_MAX_RENDERBUFFER_SIZE = 64x16384, 4x128 macrotiles) fills exactly. +pub(crate) const G18_MAX_RENDER_DIMENSION: u32 = 16384; +/// Indexed multi-draw auxiliary framebuffer flags (gears-g1 capture). +pub(crate) const G18_AUX_FB_FLAGS: u64 = 0x30c000; +/// Native main-pass tile flags (+8d2/+8d3), witnessed in colour-a/b and gears-g1. +pub(crate) const G18_MAIN_TILE_FLAGS: u64 = 0xc0_0000; +/// One-sample PPP multisample control; the only value `g18_render` admits. +pub(crate) const G18_MULTISAMPLE_CONTROL: u64 = 0x88; + +// drm_asahi_render_flags (include/uapi/drm/asahi_drm.h:859-895). +const RENDER_PROCESS_EMPTY_TILES: u32 = 1 << 1; +const RENDER_RSRC_SPEC_HI: u32 = 1 << 4; +const RENDER_DBIAS_IS_INT: u32 = 1 << 18; +const G18_RENDER_FLAGS: u32 = + RENDER_PROCESS_EMPTY_TILES | RENDER_RSRC_SPEC_HI | RENDER_DBIAS_IS_INT; +// ZLS control: stencil load (14) / store (18), stencil compression (4, 8), +// depth load (15) / store (19), depth format (25-26: 0 = Z32F, 2 = Z16). +const ZLS_STENCIL_TRANSFER: u64 = (1 << 14) | (1 << 18); +const ZLS_STENCIL_COMPRESSION: u64 = (1 << 4) | (1 << 8); +const ZLS_DEPTH_TRANSFER: u64 = (1 << 15) | (1 << 19); +const ZLS_DEPTH_FORMAT: u64 = 3 << 25; +/// Upper ISP_BGOBJVALS bits every Mesa backend seeds (`agx_pipe.c:1333`). +const ISP_BGOBJVALS_BASE: u32 = 0x300; +/// Width of the compact VDM view (`agx_m5_va.h:37`, `g17_render.rs:88`). +const VDM_COMPACT_WINDOW: u64 = 1 << 39; + +/// Why a render is outside what the G18 job can do yet. `Invalid` is a +/// malformed or unmapped command (EINVAL); every other variant is a valid +/// command outside the admitted envelope (ENOTSUPP). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G18ApplicationRefusal { + Invalid(g17_uapi::UapiTranslateError), + Flags(u32), + /// Vertex or fragment scratch helper program (spilling shaders). + HelperProgram, + Dimensions { width: u16, height: u16 }, + Layers(u16), + Samples(u8), + Utile(u8, u8), + MultisampleControl(u64), + Samplers { heap: u64, count: u16 }, + StencilTransfer(u64), + StencilClear(u32), + Depth(g17_uapi::UapiTranslateError), + DepthFormat(u64), + VdmCompact(u64), +} + +impl G18ApplicationRefusal { + /// True when the command itself is malformed rather than out of envelope. + pub(crate) fn is_invalid(&self) -> bool { + matches!(self, Self::Invalid(_)) + } +} + +/// Render-target geometry the job must plan and allocate for this command. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18TargetRequest { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) layers: u16, + pub(crate) utile: (u8, u8), + pub(crate) samples: u8, +} + +/// Everything the job receives for one admitted render. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18ApplicationRender { + /// User fields translated; every job-owned field is zero. + pub(crate) parameters: G17pRenderParameters, + /// `g18_default_controls(¶meters)`; job-owned control fields are zero. + pub(crate) controls: g18_render::G18RenderControls, + pub(crate) target: G18TargetRequest, + /// VDM stream as submitted (full VA in the submitting VM). The compact + /// form is `parameters.encoder`; the job needs both views (risk #1). + pub(crate) vdm_base: u64, + /// Attachment ranges Mesa declared (render targets and PBE images). + pub(crate) vertex_attachments: g17_uapi::UapiAttachmentList, + pub(crate) fragment_attachments: g17_uapi::UapiAttachmentList, + /// An inert compressed stencil plane was removed (see module doc). + pub(crate) stencil_stripped: bool, + /// UAPI render flags: only PROCESS_EMPTY_TILES and RSRC_SPEC_HI remain. + pub(crate) flags: u32, + /// TA/3D private-memory selection derived from the render's + /// `drm_asahi_g18_render_resources` suffix by the adapter; `None` keeps + /// the fixed first-use policy (`g18_render_policy::resource_pairs`). + pub(crate) scratch: Option, +} + +/// User pointers the G18 3D command carries as native descriptor copies +/// (render-cmd `G183dCommandFields`), because the G18 fragment program drops +/// their registers 0x15101/0x15109 (`m5_render.rs:306`). The slots are the +/// ones G17P fills from the same parameters (`g17_render.rs:1381-1383`); the +/// gears-g1 native command carries its depth-bias and scissor arrays there. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18DescriptorPointers { + /// d04e0 -> 3D +0x1d20: ISP depth-bias array. + pub(crate) d04e0: u64, + /// d04e8 -> 3D +0x1d30: ISP scissor array. + pub(crate) d04e8: u64, + /// d04f0 -> 3D +0x1d40 / register 0x15311: aligned occlusion query heap. + pub(crate) d04f0: u64, +} + +impl G18ApplicationRender { + /// The job must copy these into its 3D command fields; the register + /// programs built from `parameters` do not contain them. + pub(crate) fn descriptor_pointers(&self) -> G18DescriptorPointers { + G18DescriptorPointers { + d04e0: self.parameters.depth_bias_array, + d04e8: self.parameters.scissor_array, + d04f0: self.parameters.occlusion_query_base, + } + } +} + +/// Whether every job-owned field is still zero. The translation leaves them +/// zero; the job must assign each before building registers: +/// - parameters: context_base, tilemap, layermeta, heapmeta, tpc, +/// deflake_1/2/3, ta_status, fragment_status, aux_fb, cycle, record_index, +/// lifecycle, fragment_lifecycle, ta_hardware_buffer_id, +/// usc_flist_hardware_buffer_id, parameter_buffer and the four timestamps +/// (which `g18_render` requires to stay zero); +/// - controls: resource_word, work_key and usc_statistics of both stages, +/// and the optional TA cluster fields (zero means absent). +/// +/// Sources: the render-register notes, "Kernel-owned inputs", the H1 job +/// (`g18_graphics_job.rs:61-74`, `g18_render_assembly.rs:34-47`) and its +/// policy (`g18_render_policy.rs:25-28`). +pub(crate) fn job_owned_fields_clear( + p: &G17pRenderParameters, + c: &g18_render::G18RenderControls, +) -> bool { + let parameters = [p.context_base, p.tilemap, p.layermeta, p.heapmeta, p.tpc, + p.deflake_1, p.deflake_2, p.deflake_3, p.ta_status, p.fragment_status, + p.aux_fb, p.cycle, p.record_index, p.lifecycle, p.fragment_lifecycle, + p.parameter_buffer, p.ta_timestamp_start, p.ta_timestamp_end, + p.fragment_timestamp_start, p.fragment_timestamp_end]; + let controls = [c.ta.resource_word, c.fragment.resource_word, c.ta.work_key, + c.fragment.work_key, c.ta.usc_statistics, c.fragment.usc_statistics, + c.ta_cluster_tilemap, c.ta_cluster_layermeta]; + parameters.iter().chain(controls.iter()).all(|&v| v == 0) + && p.ta_hardware_buffer_id == 0 && p.usc_flist_hardware_buffer_id == 0 + && c.ta_cluster_meta == [0; 4] && c.ta_cluster_control == [0; 3] +} + +/// Internal state handed to `translate_render_command`: the M5 reference +/// profile's role (`m5_drm.rs:234`) with every job-owned field zeroed and +/// G18 policy applied. `translate_render_command` overwrites all user fields. +pub(crate) fn g18_application_base() -> G17pRenderParameters { + let mut p = G17pRenderParameters::source_rsrc8(); + p.context_base = 0; + p.tilemap = 0; + p.layermeta = 0; + p.heapmeta = 0; + p.tpc = 0; + p.deflake_1 = 0; + p.deflake_2 = 0; + p.deflake_3 = 0; + p.ta_status = 0; + p.fragment_status = 0; + p.aux_fb = 0; + p.cycle = 0; + p.record_index = 0; + p.lifecycle = 0; + p.fragment_lifecycle = 0; + p.ta_hardware_buffer_id = 0; + p.usc_flist_hardware_buffer_id = 0; + p.parameter_buffer = 0; + p.gpc_perf_state_map = 0; + p.gpc_perf_state_map_low = 0; + p.gpc_perf_state_control = 0; + p.ta_timestamp_start = 0; + p.ta_timestamp_end = 0; + p.fragment_timestamp_start = 0; + p.fragment_timestamp_end = 0; + // G18 policy. aux_fb_page_count keeps the value every M5/G18 capture + // carries (0x100000, `g17_render.rs:421`). + p.aux_fb_flags = G18_AUX_FB_FLAGS; + p.samples = 1; + p.absolute_pointers = false; + p.bg_resource_prefix = 0; + p.native_pm_bytes = false; + p.native_ta_registers = 0; + p +} + +/// Mesa's compact VDM view (`agx_m5_vdm_compact`). +pub(crate) fn g18_vdm_compact(vdm_base: u64) -> Option { + let offset = vdm_base.checked_sub(m5_vm::DRIVER_BASE)?; + if vdm_base & 3 != 0 || offset >= VDM_COMPACT_WINDOW { + return None; + } + Some(offset) +} + +/// msaa163: whether `command` is exactly the native 4x render set (see +/// `g18_render::G18_MSAA4_UTILE`): four samples, 32x16 utiles, sample +/// positions 0xeaa26e26. Native331 adds sixteen-byte samples +/// to the eight-byte profile, retaining its geometry. Native315 adds +/// D32/S8 attachments to that render geometry. Gallium currently allocates +/// them uncompressed; compressed multisample metadata remains unadmitted. +fn is_multisample_command(command: &g17_uapi::UapiRenderCommand) -> bool { + let (utile, positions) = match command.samples { + 2 => (g18_render::G18_MSAA2_UTILE, g18_render::G18_MSAA2_MULTISAMPLE_CONTROL), + 4 => (g18_render::G18_MSAA4_UTILE, g18_render::G18_MSAA4_MULTISAMPLE_CONTROL), + _ => return false, + }; + (command.utile_width, command.utile_height) == utile + && command.multisample_control == positions + && matches!(command.sample_size, + g18_render::G18_MSAA4_SAMPLE_SIZE | g18_render::G18_MSAA_WIDE_SAMPLE_SIZE + | g18_render::G18_MSAA_MRT_SAMPLE_SIZE) + && (command.layers == 1 || + ((2..=2048).contains(&command.layers) + && matches!(command.sample_size, + g18_render::G18_MSAA4_SAMPLE_SIZE | g18_render::G18_MSAA_WIDE_SAMPLE_SIZE) + && command.depth.base == 0 && command.stencil.base == 0)) + && command.depth.compression_base == 0 + && command.stencil.compression_base == 0 +} + +/// Complete single-level D16/D32/S8 storage for the admitted one/four-sample +/// render. AIL's tile contains 16 KiB: multiplying bytes per pixel by samples +/// shrinks its logical-pixel dimensions, rather than multiplying the rounded +/// one-sample allocation (which is wrong across a tile boundary). +fn uncompressed_plane_bytes(width: u64, height: u64, bytes_per_sample: u64, + samples: u8) -> u64 { + let block_bytes = bytes_per_sample * u64::from(samples); + let (tile_width, tile_height) = match block_bytes { + 1 => (128, 128), + 2 => (128, 64), + 4 => (64, 64), + 8 => (64, 32), + 16 => (32, 32), + _ => return 0, // Only admitted D16/D32/S8 at 1x/4x reaches this helper. + }; + if width >= tile_width && height >= tile_height { + width.div_ceil(tile_width) * height.div_ceil(tile_height) * 0x4000 + } else { + (width.next_power_of_two() * height.next_power_of_two() * block_bytes) + .div_ceil(128) * 128 + } +} + +/// Native682: bit0 selects 16-KiB units; zero selects 128-byte units. +/// Zero therefore represents a real 128-byte layer, not an absent stride. +/// Validate the complete final layer, not just its first mapped byte. +fn validate_layered_zls_plane( + plane: g17_uapi::UapiZlsBuffer, width: u64, height: u64, + bytes_per_sample: u64, layers: u16, address_space: &A, +) -> Result<(), g17_uapi::UapiTranslateError> { + use g17_uapi::{GpuAccess, UapiTranslateError as E}; + if plane.base == 0 { + return if plane.compression_base == 0 && plane.stride == 0 + && plane.compression_stride == 0 { Ok(()) } else { Err(E::ZlsFields) }; + } + if !matches!(plane.stride & 0x3fff, 0 | 1) || plane.base & 0x7f != 0 { + return Err(E::ZlsFields); + } + let unit = if plane.stride & 1 != 0 { 0x4000 } else { 0x80 }; + let stride = ((u64::from(plane.stride) >> 14) + 1) * unit; + let data_bytes = uncompressed_plane_bytes(width, height, bytes_per_sample, 1); + if data_bytes == 0 || stride < data_bytes { + return Err(E::ZlsFields); + } + let extent = stride.checked_mul(u64::from(layers - 1)) + .and_then(|n| n.checked_add(data_bytes)).ok_or(E::AddressRange)?; + if plane.base.checked_add(extent).is_none() + || !address_space.covers(plane.base, extent, GpuAccess::ReadWrite) { + return Err(E::AddressRange); + } + if plane.compression_base != 0 { + if plane.compression_base & 0x7f != 0 || plane.compression_stride & 0x3fff != 0 { + return Err(E::ZlsFields); + } + let stride = ((u64::from(plane.compression_stride) >> 14) + 1) * 0x80; + let metadata_bytes = (width.next_power_of_two().div_ceil(16) + * height.next_power_of_two().div_ceil(16) * 8).div_ceil(128) * 128; + if stride < metadata_bytes { + return Err(E::ZlsFields); + } + let extent = stride.checked_mul(u64::from(layers - 1)) + .and_then(|n| n.checked_add(metadata_bytes)).ok_or(E::AddressRange)?; + if plane.compression_base.checked_add(extent).is_none() + || !address_space.covers(plane.compression_base, extent, GpuAccess::ReadWrite) { + return Err(E::AddressRange); + } + } else if plane.compression_stride != 0 { + return Err(E::ZlsFields); + } + Ok(()) +} + +/// The envelope checks that need only the raw command. Order matters for +/// diagnostics only; every check is independent. `msaa4` admits exactly the +/// native 4x set (module parameter `g18_admit_msaa4`); refusal classes are +/// unchanged for everything else. +fn admit_envelope(command: &g17_uapi::UapiRenderCommand, msaa4: bool) + -> Result<(), G18ApplicationRefusal> { + use G18ApplicationRefusal as R; + if command.flags & !G18_RENDER_FLAGS != 0 { + return Err(R::Flags(command.flags)); + } + let no_helper = g17_uapi::UapiHelperProgram { binary: 0, config: 0, data: 0 }; + if command.vertex_helper != no_helper || command.fragment_helper != no_helper { + return Err(R::HelperProgram); + } + if !(1..=G18_MAX_RENDER_DIMENSION).contains(&u32::from(command.width)) + || !(1..=G18_MAX_RENDER_DIMENSION).contains(&u32::from(command.height)) + { + return Err(R::Dimensions { width: command.width, height: command.height }); + } + // Native598 proves layered colour target allocation and all five + // register lists. Native618/639 add layered four-sample colour through + // sixteen bytes per sample. Native682 adds one-sample layered Z32/S8; + // each plane and its metadata receive complete array range checks below. + // Native1060 also verifies uncompressed D16 arrays, including48 layers. + // Shared format validation below continues to reject compressed D16. + if !(1..=2048).contains(&command.layers) + || (command.layers > 1 && ((command.samples != 1 + && !(msaa4 && is_multisample_command(command))) + || ((command.depth.base != 0 || command.stencil.base != 0) + && (command.samples != 1 + || !matches!(command.zls_control & ZLS_DEPTH_FORMAT, 0 | 0x0400_0000))))) { + return Err(R::Layers(command.layers)); + } + if !(msaa4 && is_multisample_command(command)) { + if command.samples != 1 { + return Err(R::Samples(command.samples)); + } + if (command.utile_width, command.utile_height) != (32, 32) { + return Err(R::Utile(command.utile_width, command.utile_height)); + } + if command.multisample_control != G18_MULTISAMPLE_CONTROL { + return Err(R::MultisampleControl(command.multisample_control)); + } + } + if command.sampler_heap != 0 || command.sampler_count != 0 { + return Err(R::Samplers { heap: command.sampler_heap, count: command.sampler_count }); + } + if command.occlusion_query_base & 0xf != 0 { + return Err(R::Invalid(g17_uapi::UapiTranslateError::AddressAlignment)); + } + if command.zls_control & ZLS_STENCIL_TRANSFER != 0 + && (command.stencil.base == 0 + || (command.stencil.compression_base != 0 && command.layers == 1)) { + return Err(R::StencilTransfer(command.zls_control)); + } + if command.stencil_clear & !(ISP_BGOBJVALS_BASE | 0xff) != 0 { + return Err(R::StencilClear(command.stencil_clear)); + } + Ok(()) +} + +/// Translate one render command for the G18 job. `address_space` must answer +/// for the submitting VM; `window` is the queue's validated USC window. +/// Multisampled commands are refused (`g18_admit_msaa4=0` behaviour). The +/// adapter calls [`translate_g18_render_admitting`]; this form is kept for +/// the host parity tools (a host parity check). +#[allow(dead_code)] +#[inline(never)] +pub(crate) fn translate_g18_render( + command: &g17_uapi::UapiRenderCommand, + vertex_attachments: g17_uapi::UapiAttachmentList, + fragment_attachments: g17_uapi::UapiAttachmentList, + window: g17_uapi::QueueUscWindow, + address_space: &A, +) -> Result { + translate_g18_render_admitting(command, vertex_attachments, fragment_attachments, + window, address_space, false) +} + +/// [`translate_g18_render`], additionally admitting the native 4x render set +/// when `msaa4` is true (module parameter `g18_admit_msaa4=1`, msaa163). +#[inline(never)] +pub(crate) fn translate_g18_render_admitting( + command: &g17_uapi::UapiRenderCommand, + vertex_attachments: g17_uapi::UapiAttachmentList, + fragment_attachments: g17_uapi::UapiAttachmentList, + window: g17_uapi::QueueUscWindow, + address_space: &A, + msaa4: bool, +) -> Result { + use G18ApplicationRefusal as R; + admit_envelope(command, msaa4)?; + // Native G18 masks the low four base bits. The PPP slot is 15 bits + // and addresses an 8-byte result, matching Gallium's 32768-entry heap. + // Validate the complete read/modify/write range, not only slot zero: + // a mapped first result does not make later indices safe. + if command.occlusion_query_base != 0 { + const QUERY_HEAP_BYTES: u64 = 32768 * 8; + if command.occlusion_query_base.checked_add(QUERY_HEAP_BYTES).is_none() + || !address_space.covers(command.occlusion_query_base, QUERY_HEAP_BYTES, + g17_uapi::GpuAccess::ReadWrite) { + return Err(R::Invalid(g17_uapi::UapiTranslateError::AddressRange)); + } + } + // Validate Z32F and uncompressed Z16, including the complete format-sized + // backing. The shared validator refuses compressed Z16 metadata. + // Retain the existing depth validation while admitting stencil's own + // load/store bits. Its full uncompressed S8 range is checked separately. + // Native321 confirms D16's UNORM16 integer clear at 1x and 4x. + let depth16 = command.zls_control & ZLS_DEPTH_FORMAT == 0x0400_0000; + if command.depth.base != 0 && depth16 && command.depth_clear > 0xffff { + return Err(R::Depth(g17_uapi::UapiTranslateError::ZlsFields)); + } + let mut depth_command = *command; + depth_command.zls_control &= !ZLS_STENCIL_TRANSFER; + // Retain the shared single-plane format/control validation. The G18 + // checks below independently validate every layer's data and metadata. + depth_command.layers = 1; + g17_uapi::validate_m5_reference_depth(&depth_command, address_space).map_err(R::Depth)?; + if command.layers > 1 { + // D16 uses two bytes per sample; Z32 uses four. Applying the Z32 + // extent to D16 rejects its native128-byte8x8 layer stride. + let depth_bytes = if depth16 { 2 } else { 4 }; + for (plane, bytes_per_sample) in [(command.depth, depth_bytes), (command.stencil, 1)] { + validate_layered_zls_plane(plane, u64::from(command.width), + u64::from(command.height), bytes_per_sample, command.layers, + address_space).map_err(R::Depth)?; + } + } + if command.samples > 1 && command.depth.base != 0 { + let bytes = uncompressed_plane_bytes(u64::from(command.width), + u64::from(command.height), if depth16 { 2 } else { 4 }, command.samples); + if bytes == 0 || command.depth.base.checked_add(bytes).is_none() + || !address_space.covers(command.depth.base, bytes, g17_uapi::GpuAccess::ReadWrite) { + return Err(R::Invalid(g17_uapi::UapiTranslateError::AddressRange)); + } + } + if command.stencil.base != 0 && command.stencil.compression_base == 0 { + let bytes = uncompressed_plane_bytes(u64::from(command.width), + u64::from(command.height), 1, command.samples); + if bytes == 0 || command.stencil.base & 0x7f != 0 + || command.stencil.base.checked_add(bytes).is_none() + || !address_space.covers(command.stencil.base, bytes, g17_uapi::GpuAccess::ReadWrite) { + return Err(R::Invalid(g17_uapi::UapiTranslateError::AddressRange)); + } + } + if command.depth.base != 0 + && !matches!(command.zls_control & ZLS_DEPTH_FORMAT, 0 | 0x0400_0000) { + return Err(R::DepthFormat(command.zls_control)); + } + let translated = g17_uapi::translate_g18_render_command(*command, vertex_attachments, + fragment_attachments, window, address_space, + g17_uapi::RenderInternalState { parameters: g18_application_base() }) + .map_err(R::Invalid)?; + let mut p = translated.parameters; + + p.encoder = g18_vdm_compact(command.vdm_base).ok_or(R::VdmCompact(command.vdm_base))?; + // Native single-layer ZLS does not encode an array stride. Layered + // Native682 retains all four already-validated data/metadata strides. + if command.layers == 1 { + p.depth_stride = 0; + p.depth_aux_stride = 0; + p.stencil_stride = 0; + p.stencil_aux_stride = 0; + } + let stencil_stripped = p.stencil_aux_buffer != 0 && command.layers == 1; + if stencil_stripped { + p.stencil_buffer = 0; + p.stencil_aux_buffer = 0; + p.stencil_stride = 0; + p.stencil_aux_stride = 0; + p.depth_flags &= !ZLS_STENCIL_COMPRESSION; + // Same derivation as translate_render_command, now without stencil. + p.partial_depth_flags = p.depth_flags + | if p.depth_buffer != 0 { ZLS_DEPTH_TRANSFER } else { 0 }; + if p.depth_buffer == 0 { + // Neither plane remains: the M5 "no ZLS" envelope. + p.depth_dimensions = 0; + } + } + // ISP_BGOBJVALS: retain the S8 clear value and normalize the fixed bits. + p.stencil_clear_value = command.stencil_clear & 0xff; + // Native G18 passes the integer-depth selector to ISP register 0x15021. + // Do not erase it while replacing the shared translator's base flags. + p.aux_fb_flags = G18_AUX_FB_FLAGS | u64::from(command.flags & RENDER_DBIAS_IS_INT); + p.tile_config |= G18_MAIN_TILE_FLAGS; + + // The job plans every target inside its fixed target window; refuse here + // what it could not plan rather than fail the job. + if crate::g18::fw::render_assembly::target_plan(&p).is_none() { + return Err(R::Dimensions { width: command.width, height: command.height }); + } + + let mut controls = g18_render::g18_default_controls(&p); + // The older UAPI translator's tib_blocks is storage size, whereas G18 + // +770 / 10051 is sample control: native run112 has zero at 1x and + // 0x40000 at 4x. Keep storage accounting in p.tib_blocks for the + // independent +4d0 metadata. msaa163: the admitted 4x render (utile + // 32x16, utile_config 0x6002 and positions from the UAPI; p.samples + // stays 1, see g18_render::G18_MSAA4_UTILE) takes the native 0x40000. + let multisample_render = + msaa4 && is_multisample_command(command) && g18_render::g18_multisample(&p); + // Native464's 48-byte samples occupy two 32-byte partitions; native331's + // 16-byte samples and the original eight-byte profile occupy one. + let partitions = if command.sample_size == g18_render::G18_MSAA_MRT_SAMPLE_SIZE { + 2 + } else { + 1 + }; + controls.fragment_sample_control = + if multisample_render { (u64::from(command.samples) * partitions) << 16 } else { 0 }; + Ok(G18ApplicationRender { + parameters: p, + controls, + target: G18TargetRequest { + width: p.width, + height: p.height, + layers: p.layers, + utile: (p.utile_width, p.utile_height), + samples: if multisample_render { command.samples } else { p.samples }, + }, + vdm_base: command.vdm_base, + vertex_attachments: translated.vertex_attachments, + fragment_attachments: translated.fragment_attachments, + stencil_stripped, + flags: command.flags, + scratch: None, + }) +} + +/// Stage timestamps a retired job can report, in ticks of the architectural +/// counter (`CNTPCT_EL0`, the `DRM_IOCTL_ASAHI_GET_TIME` source): +/// `[vertex start, vertex end, fragment start, fragment end]`. +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct G18RenderRetirement { + pub(crate) stage_ticks: Option<[u64; 4]>, +} + +/// The job boundary. The conductor implements this over the G18 render +/// runtime (`g18_graphics_job.rs` and its queues/PB/UMA owners). +/// +/// Contract for `submit_render`: +/// - calls are serialized by the adapter (one at a time per device); +/// - the job fills every field [`job_owned_fields_clear`] lists, allocates targets +/// for `render.target`, maps them into `vm` (both compact views while risk +/// #1 is open), builds registers from `render.parameters/controls`, copies +/// `render.descriptor_pointers()` into its 3D command fields, encodes, +/// publishes and waits; +/// - `Ok` only after both TA and 3D retired with no fault or recovery; +/// - any error after publication must record it in `vm.status()`, retain +/// every owner and make `G18DrmRuntime::is_crashed` true (M5 contract, +/// `m5_live_boot.rs:639-656`). +#[cfg(not(test))] +pub(crate) trait G18RenderJobSink: Send + Sync { + fn submit_render(&self, vm: &crate::mmu::Vm, render: &G18ApplicationRender) + -> kernel::error::Result; +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/application_runtime.rs b/drivers/gpu/drm/asahi/g18/uapi/application_runtime.rs new file mode 100644 index 0000000000000..d8dc2fff769ec --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/application_runtime.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Pure application frame policy shared with the three-frame host replay. +//! Ring storage wraps; event stamps remain unique until their native u24 limit. +use crate::{g18::fw::render_params::G17pRenderParameters, g18::fw::render::{self as g18_render, G18RenderControls}, + g18::fw::render_command::{G18RenderCommandInputs, G18StageCompletion}, g18::uapi::render_policy as g18_render_policy}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Frame { pub generation: u32 } +impl Frame { + pub(crate) fn new(generation: u32) -> Option { + if generation==0 || generation>0xff_ffff {None} else {Some(Self {generation})} + } + pub(crate) fn stamp(self) -> u32 {self.generation<<8} + pub(crate) fn previous_stamp(self) -> u32 {(self.generation-1)<<8} + pub(crate) fn inner_before(self) -> u32 {(3+(self.generation-1)*3)%256} + pub(crate) fn inner_after(self) -> u32 {(3+self.generation*3)%256} + pub(crate) fn outer_before(self) -> u32 {(6+(self.generation-1)*2)%256} + pub(crate) fn outer_after(self) -> u32 {(6+self.generation*2)%256} + pub(crate) fn submitted(self) -> u32 {self.generation*2} +} + +/// The native G18 tail is populated from application parameters, never a +/// capture template. Optional unsupported features are explicit zero policy. +pub(crate) fn inputs(p: &mut G17pRenderParameters, c: &mut G18RenderControls, + context: u32, frame: Frame) -> G18RenderCommandInputs { + let mut i=g18_render_policy::owned_inputs(G18RenderCommandInputs::default(),context,p,c); + let g=u64::from(frame.generation); + p.lifecycle=g<<32;p.fragment_lifecycle=p.lifecycle; + i.ta.sequence=u64::from(frame.submitted()-1);i.fragment.sequence=u64::from(frame.submitted()-2); + i.fragment.pass=frame.generation as u8; + for (stage,completion) in [&mut i.ta.completion,&mut i.fragment.completion].into_iter().enumerate() { + *completion=G18StageCompletion {stamp:frame.stamp(),completion:(1<<40)|g, + key:u64::from(frame.submitted()-stage as u32),..*completion}; + } + i.ta.linked_3d.stamp=frame.stamp();i.ta.linked_3d.completion=i.fragment.completion.completion; + i.ta.d08a4=frame.generation;i.fragment.d08a4=frame.generation; + i.fragment.d04e0=p.depth_bias_array;i.fragment.d04e8=p.scissor_array;i.fragment.d04f0=p.occlusion_query_base; + i.fragment.d0518[18]=[p.load_pipeline_bind,p.load_pipeline]; + i.fragment.d0518[21]=[p.partial_load_pipeline_bind,p.partial_load_pipeline]; + i.fragment.d0518[22]=[p.depth_flags,0x0404_0404]; + i.fragment.d0690=p.depth_dimensions; + i.fragment.d0698=p.depth_buffer; + i.fragment.d06a0=[p.depth_buffer,p.depth_buffer]; + i.fragment.d0728=p.depth_aux_buffer; + // Captured gears/colour and M5 shared-runtime flag policy. These are + // independent of the application-owned depth/PPP/sample-size fields. + // msaa163: +0x212c (flag 2) is 1 in native 4x (M5: 1 at 2x and 4x). + let msaa4=g18_render::g18_msaa4(p); + i.fragment.d08c5_flags=[1,1,u8::from(msaa4),0]; + i.fragment.d0770=[c.fragment_sample_control,p.aux_fb_flags]; + i.fragment.d0780=[u64::from(p.width)|(u64::from(p.height)<<32),p.aux_fb_page_count]; + i.fragment.d0790=[p.store_pipeline_bind<<32,p.store_pipeline]; + i.fragment.d07b0=[p.partial_store_pipeline_bind<<32,p.partial_store_pipeline]; + i.fragment.d07d8=u64::from(p.depth_clear_value_bits)|(0x300<<32); + // +0x1fb0 is bytes per sample, not storage blocks or sample control. + // Both native2x/4x geometries contain2048 raster samples;1x contains1024. + // All admitted strides are multiples of8, so the2-KiB block count is exact. + let raster_samples=u64::from(p.utile_width)*u64::from(p.utile_height) + *u64::from(g18_render::g18_sample_count(p)); + i.fragment.d04d0=(p.tib_blocks*2048/raster_samples) as u32; + // msaa163: +0x50 samples (native 4) and +0x54 macro size (0x40008). + i.fragment.d0900=g18_render::g18_sample_count(p); + let tiles_x=(p.width+31)/32;let tiles_y=(p.height+31)/32; + i.fragment.d0908=g18_render::g18_macro_size(p); + i.fragment.d0910=[0,u64::from(p.merge_upper_x_bits)|(u64::from(p.merge_upper_y_bits)<<32)]; + i.fragment.d0930=tiles_x*tiles_y; + i +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/command.rs b/drivers/gpu/drm/asahi/g18/uapi/command.rs new file mode 100644 index 0000000000000..4c11e2de8ddb5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/command.rs @@ -0,0 +1,2226 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! Pure modern-Asahi UAPI parsing and G17P command translation. +//! +//! The byte layouts match `drm_asahi_cmd_header`, `drm_asahi_cmd_render`, +//! `drm_asahi_cmd_compute`, and `drm_asahi_attachment`. Translation validates +//! the queue's 4 GiB USC window and every userspace range before constructing +//! G17P render parameters or compute-entry operands. This module allocates no +//! object, publishes no queue state, and performs no firmware or MMIO action. + +#[cfg(not(test))] +use crate::{g18::fw::compute_params as g17_compute, g18::fw::render_params as g17_render, g18::fw::submission as g17_submission}; +#[cfg(test)] +#[path = "../fw/compute_params.rs"] +mod g17_compute; +#[cfg(test)] +#[path = "../fw/render_params.rs"] +mod g17_render; +#[cfg(test)] +#[allow(dead_code)] +#[path = "../fw/submission.rs"] +mod g17_submission; + +use g17_render::{ + G17pRenderParameters, RenderBuildError, RenderDescriptorMetadata, RenderDescriptorObjects, +}; +use g17_submission::{ + G17PClBarrierDependency, G17PClKickEntryOperands, G17PClMcacheAperture, G17PClRceBinding, +}; + +pub(crate) const UAPI_COMMAND_HEADER_SIZE: usize = 8; +pub(crate) const UAPI_RENDER_SIZE: usize = 272; +pub(crate) const UAPI_COMPUTE_SIZE: usize = 88; +pub(crate) const UAPI_G18_COMPUTE_SIZE: usize = 136; +pub(crate) const COMPUTE_G18_RESOURCES: u32 = 1 << 2; +/// `drm_asahi_cmd_render_g18`: the render payload plus its 80-byte suffix. +pub(crate) const UAPI_G18_RENDER_SIZE: usize = UAPI_RENDER_SIZE + 80; +/// `DRM_ASAHI_RENDER_G18_RESOURCES`. Only the G18 adapter may consume and +/// strip it; the shared translator's RENDER_FLAGS keeps refusing it. +pub(crate) const RENDER_G18_RESOURCES: u32 = 1 << 21; +pub(crate) const UAPI_ATTACHMENT_SIZE: usize = 24; +pub(crate) const UAPI_MAX_ATTACHMENTS: usize = 16; +pub(crate) const UAPI_MAX_HARDWARE_COMMANDS: u32 = 64; +// Match the image/framebuffer limit advertised by Honeykrisp. Storage is +// sized from each submission; this is a format limit, not a demo window cap. +pub(crate) const M5_MAX_RENDER_DIMENSION: u32 = 16384; +pub(crate) const UAPI_BARRIER_NONE: u16 = 0xffff; +pub(crate) const USC_WINDOW_SIZE: u64 = 1 << 32; +#[cfg(not(test))] +pub(crate) const COMPUTE_G17P_ADD3_PROOF: u32 = + kernel::uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF as u32; +#[cfg(test)] +pub(crate) const COMPUTE_G17P_ADD3_PROOF: u32 = 1 << 0; +const G17P_ADD3_BUFFER_BYTES: u64 = 64 * 4; + +const COMMAND_RENDER: u16 = 0; +const COMMAND_COMPUTE: u16 = 1; +const SET_VERTEX_ATTACHMENTS: u16 = 2; +const SET_FRAGMENT_ATTACHMENTS: u16 = 3; +const SET_COMPUTE_ATTACHMENTS: u16 = 4; + +const RENDER_VERTEX_SCRATCH: u32 = 1 << 0; +const RENDER_PROCESS_EMPTY_TILES: u32 = 1 << 1; +const RENDER_NO_VERTEX_CLUSTERING: u32 = 1 << 2; +const RENDER_RSRC_SPEC_HI: u32 = 1 << 4; +const RENDER_DBIAS_IS_INT: u32 = 1 << 18; +const RENDER_FLAGS: u32 = RENDER_VERTEX_SCRATCH + | RENDER_PROCESS_EMPTY_TILES + | RENDER_NO_VERTEX_CLUSTERING + | RENDER_RSRC_SPEC_HI + | RENDER_DBIAS_IS_INT + // Private, opt-in matched-workload diagnostic. Ordinary builds reject it. + | if cfg!(g17p_aux_fb_small_probe) { 1 << 31 } else { 0 } + | if cfg!(g17p_mcache_aperture_probe) { 1 << 30 } else { 0 }; + +const SAMPLER_SIZE: u64 = 8; +const MAX_SAMPLERS: u32 = 1024; +const PAGE_SIZE: u64 = 0x4000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum UapiTranslateError { + HeaderTruncated, + PayloadTruncated, + PayloadTail, + CommandType, + AttachmentPayloadSize, + AttachmentCount, + AttachmentFields, + AttachmentBarrier, + HardwareCommandCount, + BarrierFuture, + QueueUscAlignment, + QueueUscRange, + AddressRange, + AddressAlignment, + FlagsUnknownBits, + FlagsUnsupported, + Dimensions, + Layers, + UtileDimensions, + Samples, + TilebufferSize, + SamplerCount, + SamplerPair, + HelperProgram, + Add3ProofAttachments, + ProgramOffset, + ZlsFields, + StencilClearBits, + RenderBuild(RenderBuildError), + ComputeBuild(g17_compute::ComputeBuildError), +} + +impl From for UapiTranslateError { + fn from(value: RenderBuildError) -> Self { + Self::RenderBuild(value) + } +} + +impl From for UapiTranslateError { + fn from(value: g17_compute::ComputeBuildError) -> Self { + Self::ComputeBuild(value) + } +} + +impl UapiTranslateError { + /// Stable short name for one rejection, so a failing submit reports which + /// check refused it instead of a bare EINVAL. Kept as `&'static str` (no + /// `Debug` formatting) to stay cheap on the ioctl path. + pub(crate) fn name(&self) -> &'static str { + match self { + Self::HeaderTruncated => "header-truncated", + Self::PayloadTruncated => "payload-truncated", + Self::PayloadTail => "payload-tail", + Self::CommandType => "command-type", + Self::AttachmentPayloadSize => "attachment-payload-size", + Self::AttachmentCount => "attachment-count", + Self::AttachmentFields => "attachment-fields", + Self::AttachmentBarrier => "attachment-barrier", + Self::HardwareCommandCount => "hardware-command-count", + Self::BarrierFuture => "barrier-future", + Self::QueueUscAlignment => "queue-usc-alignment", + Self::QueueUscRange => "queue-usc-range", + Self::AddressRange => "address-range", + Self::AddressAlignment => "address-alignment", + Self::FlagsUnknownBits => "flags-unknown-bits", + Self::FlagsUnsupported => "flags-unsupported", + Self::Dimensions => "dimensions", + Self::Layers => "layers", + Self::UtileDimensions => "utile-dimensions", + Self::Samples => "samples", + Self::TilebufferSize => "tilebuffer-size", + Self::SamplerCount => "sampler-count", + Self::SamplerPair => "sampler-pair", + Self::HelperProgram => "helper-program", + Self::Add3ProofAttachments => "add3-proof-attachments", + Self::ProgramOffset => "program-offset", + Self::ZlsFields => "zls-fields", + Self::StencilClearBits => "stencil-clear-bits", + Self::RenderBuild(_) => "render-build", + Self::ComputeBuild(_) => "compute-build", + } + } +} + +fn get_u16(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) +} + +fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) +} + +fn padded_payload(raw: &[u8]) -> Result<[u8; N], UapiTranslateError> { + if raw.len() > N && raw[N..].iter().any(|byte| *byte != 0) { + return Err(UapiTranslateError::PayloadTail); + } + let mut out = [0u8; N]; + let count = core::cmp::min(raw.len(), N); + out[..count].copy_from_slice(&raw[..count]); + Ok(out) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiCommandHeader { + pub(crate) command_type: u16, + pub(crate) size: u16, + pub(crate) render_barrier: u16, + pub(crate) compute_barrier: u16, +} + +impl UapiCommandHeader { + fn parse(raw: &[u8]) -> Result { + if raw.len() < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + Ok(Self { + command_type: get_u16(raw, 0), + size: get_u16(raw, 2), + render_barrier: get_u16(raw, 4), + compute_barrier: get_u16(raw, 6), + }) + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct UapiAttachment { + pub(crate) address: u64, + pub(crate) size: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiAttachmentList { + pub(crate) entries: [UapiAttachment; UAPI_MAX_ATTACHMENTS], + pub(crate) count: u8, +} + +impl UapiAttachmentList { + pub(crate) const EMPTY: Self = Self { + entries: [UapiAttachment { + address: 0, + size: 0, + }; UAPI_MAX_ATTACHMENTS], + count: 0, + }; + + pub(crate) fn as_slice(&self) -> &[UapiAttachment] { + &self.entries[..self.count as usize] + } + + fn parse(raw: &[u8]) -> Result { + if raw.len() % UAPI_ATTACHMENT_SIZE != 0 { + return Err(UapiTranslateError::AttachmentPayloadSize); + } + let count = raw.len() / UAPI_ATTACHMENT_SIZE; + if count > UAPI_MAX_ATTACHMENTS { + return Err(UapiTranslateError::AttachmentCount); + } + let mut out = Self::EMPTY; + for index in 0..count { + let offset = index * UAPI_ATTACHMENT_SIZE; + if get_u32(raw, offset + 0x10) != 0 || get_u32(raw, offset + 0x14) != 0 { + return Err(UapiTranslateError::AttachmentFields); + } + out.entries[index] = UapiAttachment { + address: get_u64(raw, offset), + size: get_u64(raw, offset + 8), + }; + } + out.count = count as u8; + Ok(out) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiHelperProgram { + pub(crate) binary: u32, + pub(crate) config: u32, + pub(crate) data: u64, +} + +fn helper(raw: &[u8], offset: usize) -> UapiHelperProgram { + UapiHelperProgram { + binary: get_u32(raw, offset), + config: get_u32(raw, offset + 4), + data: get_u64(raw, offset + 8), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiProgram { + pub(crate) usc: u32, + pub(crate) resource_spec: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamp { + pub(crate) handle: u32, + pub(crate) offset: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamps { + pub(crate) start: UapiTimestamp, + pub(crate) end: UapiTimestamp, +} + +fn timestamps(raw: &[u8], offset: usize) -> UapiTimestamps { + UapiTimestamps { + start: UapiTimestamp { + handle: get_u32(raw, offset), + offset: get_u32(raw, offset + 4), + }, + end: UapiTimestamp { + handle: get_u32(raw, offset + 8), + offset: get_u32(raw, offset + 12), + }, + } +} + +fn program(raw: &[u8], offset: usize, high_offset: usize) -> UapiProgram { + UapiProgram { + usc: get_u32(raw, offset), + resource_spec: get_u32(raw, offset + 4) as u64 + | (get_u32(raw, high_offset) as u64) << 32, + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiZlsBuffer { + pub(crate) base: u64, + pub(crate) compression_base: u64, + pub(crate) stride: u32, + pub(crate) compression_stride: u32, +} + +fn zls(raw: &[u8], offset: usize) -> UapiZlsBuffer { + UapiZlsBuffer { + base: get_u64(raw, offset), + compression_base: get_u64(raw, offset + 8), + stride: get_u32(raw, offset + 0x10), + compression_stride: get_u32(raw, offset + 0x14), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiRenderCommand { + pub(crate) flags: u32, + pub(crate) depth_dimensions: u32, + pub(crate) vdm_base: u64, + pub(crate) vertex_helper: UapiHelperProgram, + pub(crate) fragment_helper: UapiHelperProgram, + pub(crate) scissor_base: u64, + pub(crate) depth_bias_base: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth: UapiZlsBuffer, + pub(crate) stencil: UapiZlsBuffer, + pub(crate) zls_control: u64, + pub(crate) multisample_control: u64, + pub(crate) sampler_heap: u64, + pub(crate) ppp_control: u32, + pub(crate) width: u16, + pub(crate) height: u16, + pub(crate) layers: u16, + pub(crate) sampler_count: u16, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) samples: u8, + pub(crate) sample_size: u8, + pub(crate) merge_upper_x: u32, + pub(crate) merge_upper_y: u32, + pub(crate) background: UapiProgram, + pub(crate) end_of_tile: UapiProgram, + pub(crate) partial_background: UapiProgram, + pub(crate) partial_end_of_tile: UapiProgram, + pub(crate) depth_clear: u32, + pub(crate) stencil_clear: u32, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, + pub(crate) prior_vdm: u64, + pub(crate) prior_cdm: u64, + /// `drm_asahi_g18_render_resources` words; zero unless flagged. + pub(crate) g18_resources: [u32; 20], +} + +impl UapiRenderCommand { + fn parse(raw: &[u8]) -> Result { + // Legacy payloads are zero-extended exactly as before. A flagged G18 + // suffix must be complete; padding could hide a truncated field. + let has_resources = raw.len() >= 4 && get_u32(raw, 0) & RENDER_G18_RESOURCES != 0; + if has_resources && raw.len() < UAPI_G18_RENDER_SIZE { + return Err(UapiTranslateError::PayloadTruncated); + } + if !has_resources && raw.len() > UAPI_RENDER_SIZE + && raw[UAPI_RENDER_SIZE..].iter().any(|byte| *byte != 0) + { + return Err(UapiTranslateError::PayloadTail); + } + let raw = padded_payload::(raw)?; + let mut g18_resources = [0; 20]; + if has_resources { + for (i, word) in g18_resources.iter_mut().enumerate() { + *word = get_u32(&raw, UAPI_RENDER_SIZE + i * 4); + } + } + Ok(Self { + flags: get_u32(&raw, 0x00), + depth_dimensions: get_u32(&raw, 0x04), + vdm_base: get_u64(&raw, 0x08), + vertex_helper: helper(&raw, 0x10), + fragment_helper: helper(&raw, 0x20), + scissor_base: get_u64(&raw, 0x30), + depth_bias_base: get_u64(&raw, 0x38), + occlusion_query_base: get_u64(&raw, 0x40), + depth: zls(&raw, 0x48), + stencil: zls(&raw, 0x60), + zls_control: get_u64(&raw, 0x78), + multisample_control: get_u64(&raw, 0x80), + sampler_heap: get_u64(&raw, 0x88), + ppp_control: get_u32(&raw, 0x90), + width: get_u16(&raw, 0x94), + height: get_u16(&raw, 0x96), + layers: get_u16(&raw, 0x98), + sampler_count: get_u16(&raw, 0x9a), + utile_width: raw[0x9c], + utile_height: raw[0x9d], + samples: raw[0x9e], + sample_size: raw[0x9f], + merge_upper_x: get_u32(&raw, 0xa0), + merge_upper_y: get_u32(&raw, 0xa4), + background: program(&raw, 0xa8, 0xf8), + end_of_tile: program(&raw, 0xb0, 0xf0), + partial_background: program(&raw, 0xb8, 0xfc), + partial_end_of_tile: program(&raw, 0xc0, 0xf4), + depth_clear: get_u32(&raw, 0xc8), + stencil_clear: get_u32(&raw, 0xcc), + vertex_timestamps: timestamps(&raw, 0xd0), + fragment_timestamps: timestamps(&raw, 0xe0), + prior_vdm: get_u64(&raw, 0x100), + prior_cdm: get_u64(&raw, 0x108), + g18_resources, + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiComputeCommand { + pub(crate) flags: u32, + pub(crate) sampler_count: u32, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) sampler_heap: u64, + pub(crate) helper: UapiHelperProgram, + pub(crate) timestamps: UapiTimestamps, + pub(crate) scs_layout: u64, + pub(crate) g18_resources: [u32; 16], + pub(crate) prior_vdm: u64, + pub(crate) prior_cdm: u64, +} + +impl UapiComputeCommand { + fn parse(raw: &[u8]) -> Result { + // Legacy compute payloads are zero-extended as before. A flagged + // metadata payload must be complete; padding it could hide truncation + // of a valid zero-valued field or reserved word. + let has_resources = raw.len() >= 4 && get_u32(raw, 0) & COMPUTE_G18_RESOURCES != 0; + if has_resources && raw.len() < UAPI_G18_COMPUTE_SIZE { + return Err(UapiTranslateError::PayloadTruncated); + } + if !has_resources && raw.len() > UAPI_COMPUTE_SIZE + && raw[UAPI_COMPUTE_SIZE..].iter().any(|byte| *byte != 0) + { + return Err(UapiTranslateError::PayloadTail); + } + let raw = padded_payload::(raw)?; + let mut g18_resources = [0; 16]; + if has_resources { + for (i, word) in g18_resources.iter_mut().enumerate() { + *word = get_u32(&raw, 72 + i * 4); + } + } + Ok(Self { + flags: get_u32(&raw, 0), + sampler_count: get_u32(&raw, 4), + control_stream_base: get_u64(&raw, 8), + control_stream_end: get_u64(&raw, 0x10), + sampler_heap: get_u64(&raw, 0x18), + helper: helper(&raw, 0x20), + timestamps: timestamps(&raw, 0x30), + scs_layout: get_u64(&raw, 0x40), + g18_resources, + prior_vdm: if has_resources { 0 } else { get_u64(&raw, 0x48) }, + prior_cdm: if has_resources { 0 } else { get_u64(&raw, 0x50) }, + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ParsedHardwareCommand { + Render { + index: u32, + header: UapiCommandHeader, + payload: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + }, + Compute { + index: u32, + header: UapiCommandHeader, + payload: UapiComputeCommand, + attachments: UapiAttachmentList, + }, +} + +pub(crate) struct UapiCommandParser<'a> { + raw: &'a [u8], + offset: usize, + command_index: u32, + hardware_count: u32, + render_count: u16, + compute_count: u16, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + compute_attachments: UapiAttachmentList, +} + +impl<'a> UapiCommandParser<'a> { + pub(crate) const fn new(raw: &'a [u8]) -> Self { + Self { + raw, + offset: 0, + command_index: 0, + hardware_count: 0, + render_count: 0, + compute_count: 0, + vertex_attachments: UapiAttachmentList::EMPTY, + fragment_attachments: UapiAttachmentList::EMPTY, + compute_attachments: UapiAttachmentList::EMPTY, + } + } + + pub(crate) fn finish(&self) -> Result { + if self.offset != self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + if self.hardware_count == 0 || self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + Ok(self.hardware_count) + } + + pub(crate) fn next_hardware( + &mut self, + ) -> Result, UapiTranslateError> { + loop { + if self.offset == self.raw.len() { + return Ok(None); + } + if self.raw.len() - self.offset < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + let header = UapiCommandHeader::parse(&self.raw[self.offset..])?; + self.offset += UAPI_COMMAND_HEADER_SIZE; + let end = self + .offset + .checked_add(header.size as usize) + .ok_or(UapiTranslateError::PayloadTruncated)?; + if end > self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + let payload = &self.raw[self.offset..end]; + self.offset = end; + self.command_index += 1; + + match header.command_type { + COMMAND_RENDER | COMMAND_COMPUTE => { + if header.render_barrier != UAPI_BARRIER_NONE + && header.render_barrier > self.render_count + { + return Err(UapiTranslateError::BarrierFuture); + } + if header.compute_barrier != UAPI_BARRIER_NONE + && header.compute_barrier > self.compute_count + { + return Err(UapiTranslateError::BarrierFuture); + } + self.hardware_count += 1; + if self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + if header.command_type == COMMAND_RENDER { + self.render_count += 1; + return Ok(Some(ParsedHardwareCommand::Render { + index: self.command_index, + header, + payload: UapiRenderCommand::parse(payload)?, + vertex_attachments: self.vertex_attachments, + fragment_attachments: self.fragment_attachments, + })); + } + self.compute_count += 1; + return Ok(Some(ParsedHardwareCommand::Compute { + index: self.command_index, + header, + payload: UapiComputeCommand::parse(payload)?, + attachments: self.compute_attachments, + })); + } + SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS | SET_COMPUTE_ATTACHMENTS => { + if header.render_barrier != UAPI_BARRIER_NONE + || header.compute_barrier != UAPI_BARRIER_NONE + { + return Err(UapiTranslateError::AttachmentBarrier); + } + let attachments = UapiAttachmentList::parse(payload)?; + match header.command_type { + SET_VERTEX_ATTACHMENTS => self.vertex_attachments = attachments, + SET_FRAGMENT_ATTACHMENTS => self.fragment_attachments = attachments, + SET_COMPUTE_ATTACHMENTS => self.compute_attachments = attachments, + _ => unreachable!(), + } + } + _ => return Err(UapiTranslateError::CommandType), + } + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum GpuAccess { + Read, + Write, + ReadWrite, +} + +pub(crate) trait GpuAddressSpace { + fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool; +} + +fn require_range( + address_space: &A, + address: u64, + size: u64, + access: GpuAccess, +) -> Result<(), UapiTranslateError> { + let end = address + .checked_add(size) + .ok_or(UapiTranslateError::AddressRange)?; + if size == 0 || end <= address || !address_space.covers(address, size, access) { + return Err(UapiTranslateError::AddressRange); + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueueUscWindow { + pub(crate) base: u64, + pub(crate) user_start: u64, + pub(crate) user_end: u64, +} + +impl QueueUscWindow { + pub(crate) fn validate(self) -> Result<(), UapiTranslateError> { + if self.base & (USC_WINDOW_SIZE - 1) != 0 { + return Err(UapiTranslateError::QueueUscAlignment); + } + let end = self + .base + .checked_add(USC_WINDOW_SIZE) + .ok_or(UapiTranslateError::QueueUscRange)?; + if self.base < self.user_start || end > self.user_end || end <= self.base { + return Err(UapiTranslateError::QueueUscRange); + } + Ok(()) + } + + fn full_program( + self, + address_space: &A, + tagged_offset: u32, + ) -> Result { + self.validate()?; + if tagged_offset == 0 { + return Err(UapiTranslateError::ProgramOffset); + } + let allocation = self + .base + .checked_add((tagged_offset & !7) as u64) + .ok_or(UapiTranslateError::ProgramOffset)?; + require_range(address_space, allocation, 4, GpuAccess::Read)?; + // Honeykrisp carries resource metadata in the low three UAPI bits, + // but G17P's native 3D command strips those bits before programming + // every BG/EOT/partial USC address. Preserving bit 2 launched four + // bytes into the instruction stream and contradicted all complete + // Metal register-list captures. + Ok(allocation) + } +} + +fn validate_sampler( + address_space: &A, + heap: u64, + count: u32, +) -> Result<(), UapiTranslateError> { + if count > MAX_SAMPLERS { + return Err(UapiTranslateError::SamplerCount); + } + if (heap == 0) != (count == 0) { + return Err(UapiTranslateError::SamplerPair); + } + if count != 0 { + if heap & (SAMPLER_SIZE - 1) != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + heap, + count as u64 * SAMPLER_SIZE, + GpuAccess::Read, + )?; + } + Ok(()) +} + +fn validate_attachments( + address_space: &A, + attachments: &UapiAttachmentList, +) -> Result<(), UapiTranslateError> { + for attachment in attachments.as_slice() { + require_range( + address_space, + attachment.address, + attachment.size, + GpuAccess::Write, + )?; + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderInternalState { + pub(crate) parameters: G17pRenderParameters, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedRenderCommand { + /// Exact logical render prefix assigned by the DRM ioctl dispatcher. + pub(crate) render_order: g17_submission::G17PRenderOrder, + /// Original UAPI render index, retained through admission. + pub(crate) render_barrier: u16, + /// Original UAPI compute index, retained through admission. + pub(crate) compute_barrier: u16, + /// Exact compute prefix resolved at ioctl admission. Later independent + /// compute commands must not become parents of this render. + pub(crate) compute_wait_through: Option, + pub(crate) flags: u32, + pub(crate) vdm_base: u64, + pub(crate) parameters: G17pRenderParameters, + pub(crate) vertex_attachments: UapiAttachmentList, + pub(crate) fragment_attachments: UapiAttachmentList, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, +} + +fn validate_zls( + address_space: &A, + value: UapiZlsBuffer, + layers: u16, + g18_subpage: bool, +) -> Result<(), UapiTranslateError> { + if value.base == 0 + && (value.compression_base != 0 || value.stride != 0 || value.compression_stride != 0) + { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_base == 0 && value.compression_stride != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if !g18_subpage && layers > 1 && value.base != 0 && value.stride == 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.stride != 0 && value.stride & 0x3fff != 1 + && !(g18_subpage && value.stride & 0x3fff == 0) { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_stride & 0x3fff != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.base != 0 { + let layer_stride = if g18_subpage && layers > 1 { + let unit = if value.stride & 1 != 0 { PAGE_SIZE } else { 0x80 }; + ((value.stride as u64 >> 14) + 1) * unit + } else if value.stride == 0 { + 0 + } else { + ((value.stride as u64 >> 14) + 1) * PAGE_SIZE + }; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range(address_space, value.base, size, GpuAccess::ReadWrite)?; + } + if value.compression_base != 0 { + let layer_stride = ((value.compression_stride as u64 >> 14) + 1) * 0x80; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range( + address_space, + value.compression_base, + size, + GpuAccess::ReadWrite, + )?; + } + Ok(()) +} + +/// Bounded M5 D32 clear/store envelope, alongside the original color-only +/// profile. Native 25G76 supplies ff00ff dimensions and 80044 with compression; +/// bits2/6 select compression, while bit19 stores the depth plane. Geometry, +/// stencil absence, and application programs are checked by the M5 caller. +pub(crate) fn validate_m5_reference_depth( + command: &UapiRenderCommand, + address_space: &A, +) -> Result<(), UapiTranslateError> { + let depth = command.depth; + if depth.base == 0 { + // No depth plane. The surface may still carry an inert stencil one, + // which contributes nothing but its compression pair and the shared + // pixel dimensions; a render that transfers it is refused with the + // rest of the stencil profile. + let stencil_only = command.stencil.base != 0 + && command.zls_control + == if command.stencil.compression_base != 0 { 0x110 } else { 0 } + && command.depth_dimensions + == ((u32::from(command.width) - 1) + | ((u32::from(command.height) - 1) << 16)); + return if depth.compression_base == 0 && depth.stride == 0 + && depth.compression_stride == 0 + && (stencil_only + || (command.zls_control == 0 && command.depth_dimensions == 0)) { + Ok(()) + } else { + Err(UapiTranslateError::ZlsFields) + }; + } + let compressed = depth.compression_base != 0; + // The control word's load and store bits are the only part a render + // chooses: bit15 loads the depth plane and bit19 stores it, and a render + // that neither tests nor writes depth sets neither, which is how a + // conformance surface with an unused depth buffer arrives. Everything + // else -- the compression pair in bits 2 and 6, the GPU tiling, the 32F + // format -- is the witnessed profile and stays exact. depth_flags reaches + // register 0x15319 unchanged either way, so the admitted envelope is the + // set of load/store combinations over that fixed remainder. + let compression_bits = if compressed { 0x44 } else { 0 }; + // Bits 25-26 select the depth format: 0 is 32-bit float, 2 is 16-bit + // unorm. The narrow plane was captured on 25G76 alongside the wide one + // and moves nothing else in the register program except the clear value + // and one ISP flag, both of which the caller derives. Its compression + // metadata has no witness, so a compressed 16-bit plane stays out. + let z16 = command.zls_control & (3 << 25) == (2 << 25); + if z16 && compressed { + return Err(UapiTranslateError::ZlsFields); + } + let format_bits = if z16 { 2 << 25 } else { 0 }; + // A stencil plane may be attached without being transferred: a surface + // with a depth-stencil buffer carries one for every render, whether or + // not the render uses it. Its compression pair in bits 4 and 8 describes + // the attached layout and is the only stencil state admitted here -- + // loading or storing the plane (bits 14 and 18) needs its own witnesses, + // as does the stencil unit in the graphics record. + let stencil_compression_bits = + if command.stencil.compression_base != 0 { 0x110 } else { 0 }; + let control_transfers = command.zls_control & 0x88000; + let expected_control = + compression_bits | stencil_compression_bits | format_bits | control_transfers; + let width = u32::from(command.width); + let height = u32::from(command.height); + if !(1..=M5_MAX_RENDER_DIMENSION).contains(&width) + || !(1..=M5_MAX_RENDER_DIMENSION).contains(&height) + || command.layers != 1 || (compressed && (width < 16 || height < 16)) { + return Err(UapiTranslateError::ZlsFields); + } + // AIL tiles are 16 KiB whatever the format: 64x64 at four bytes a pixel, + // 128x64 at two. While both axes fit one tile the level is a whole + // number of them; if either axis is smaller, the level uses a + // power-of-two miptail aligned to a 128-byte cacheline. Single-layer M5 + // surfaces are not page-padded. + let (tile_width, bytes_per_pixel) = if z16 { (128u32, 2u64) } else { (64u32, 4u64) }; + let data_bytes = if width >= tile_width && height >= 64 { + u64::from(width.div_ceil(tile_width)) * u64::from(height.div_ceil(64)) * 0x4000 + } else { + (u64::from(width.next_power_of_two()) + * u64::from(height.next_power_of_two()) * bytes_per_pixel) + .div_ceil(128) * 128 + }; + let metadata_bytes = (u64::from(width.next_power_of_two().div_ceil(16)) + * u64::from(height.next_power_of_two().div_ceil(16)) * 8).div_ceil(128) * 128; + let dimensions = (width - 1) | ((height - 1) << 16); + if command.depth_dimensions != dimensions + || command.zls_control != expected_control + // The clear value is whatever the application asked for; it reaches + // register 0x15301 verbatim and is only consumed when the render + // actually clears the plane. + || (command.depth_clear & 0x7fffffff) > 0x7f800000 + // Image suballocations and small miptails use cacheline alignment; + // the backing VM pages remain independently validated below. + || depth.base & 0x7f != 0 || depth.compression_base & 0x7f != 0 + // Both strides are array strides, and the M5 caller zeroes them + // because native single-layer ZLS does not encode one. Every render + // admitted here is single-layer -- layers != 1 is refused above -- so + // the value that arrives is whatever the userspace packer computed + // and the hardware never sees it: for a surface smaller than a page + // Mesa's (pages - 1) << 14 | 1 underflows to 0xffffc001. Comparing it + // would refuse a valid render over a field that is discarded. The + // backing store is still validated in full below, from the + // dimensions rather than from either stride. + || (!compressed && depth.compression_stride != 0) { + return Err(UapiTranslateError::ZlsFields); + } + // Validate the entire resized surface and metadata before the + // M5 caller removes the unused single-layer array stride fields. + require_range(address_space, depth.base, data_bytes, GpuAccess::ReadWrite)?; + if compressed { + require_range(address_space, depth.compression_base, metadata_bytes, + GpuAccess::ReadWrite)?; + } + Ok(()) +} + +pub(crate) fn translate_render_command( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, +) -> Result { + translate_render_command_with_capacity(command, vertex_attachments, + fragment_attachments, queue, address_space, internal, 32_768, false) +} + +/// G18 has a separately admitted native 96-KiB resident multisample profile. +/// Its caller validates sample count, utile geometry, stride and layer count. +pub(crate) fn translate_g18_render_command( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, +) -> Result { + // Native1044's 1x RGBA32 MRT footprint reaches128 bytes per sample at 32x32 + // (128KiB); native1121 adds 40/56/72. Layered, geometry and MSAA stay separate. + let wide_single = command.samples == 1 && command.layers == 1 + && (command.utile_width, command.utile_height) == (32, 32) + && (((48..=128).contains(&command.sample_size) && command.sample_size & 15 == 0) + || matches!(command.sample_size, 40 | 56 | 72)); + let capacity = if wide_single { 131_072 } + else if matches!(command.samples, 2 | 4) { 98_304 } + else { 32_768 }; + translate_render_command_with_capacity(command, vertex_attachments, + fragment_attachments, queue, address_space, internal, capacity, true) +} + +fn translate_render_command_with_capacity( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, + tile_capacity: u64, + g18_subpage: bool, +) -> Result { + if command.flags & !RENDER_FLAGS != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + // Genuinely unimplemented on G17P rather than merely unvalidated: + // `G17pRenderParameters` carries no helper-program fields at all, so + // VERTEX_SCRATCH has nowhere to put the scratch binding, and there is no + // cluster-count control for NO_VERTEX_CLUSTERING. A clean EINVAL beats + // rendering with the request silently dropped. + if command.flags & (RENDER_VERTEX_SCRATCH | RENDER_NO_VERTEX_CLUSTERING) != 0 { + return Err(UapiTranslateError::FlagsUnsupported); + } + // RSRC_SPEC_HI is *declarative*, and honouring it must not mean rejecting. + // The UAPI defines it as "the appended resource-specifier high dwords are + // present", and states its purpose: "This flag lets old kernels report an + // unsupported render payload instead of silently truncating 64-bit G17 + // values." It exists for kernels that do NOT understand the dwords. + // + // This kernel does understand them -- `program()` ORs the high dword in + // unconditionally, with no reference to the flag -- so the flag cannot + // change what we do with a payload that sets it. The old check therefore + // refused payloads it would otherwise have handled identically, protecting + // nothing; it is the same bring-up leftover as the stencil-clear mask. + // + // Honour the documented meaning instead: when the flag is clear the high + // dwords are "not present", so mask them off rather than refuse the + // submission. A userspace built against the 240-byte upstream struct (whose + // payload `padded_payload` zero-fills) and one that clears the flag now + // behave identically, which is what "not present" has to mean. + let resource_spec_mask = if command.flags & RENDER_RSRC_SPEC_HI != 0 { + u64::MAX + } else { + u32::MAX as u64 + }; + if !(1..=16_384).contains(&command.width) || !(1..=16_384).contains(&command.height) { + return Err(UapiTranslateError::Dimensions); + } + if !(1..=2048).contains(&command.layers) { + return Err(UapiTranslateError::Layers); + } + if !matches!( + (command.utile_width, command.utile_height), + (32, 32) | (32, 16) | (16, 16) + ) { + return Err(UapiTranslateError::UtileDimensions); + } + let samples_log2 = match command.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err(UapiTranslateError::Samples), + }; + let utile_bytes = command.sample_size as u64 + * command.utile_width as u64 + * command.utile_height as u64 + * command.samples as u64; + if utile_bytes > tile_capacity { + return Err(UapiTranslateError::TilebufferSize); + } + let blocks_per_utile = utile_bytes.div_ceil(2048); + + if command.vdm_base == 0 || command.vdm_base & 3 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.vdm_base, 4, GpuAccess::Read)?; + + if command.scissor_base == 0 || command.scissor_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.scissor_base, 8, GpuAccess::Read)?; + if command.depth_bias_base != 0 { + if command.depth_bias_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.depth_bias_base, 8, GpuAccess::Read)?; + } + if command.occlusion_query_base != 0 { + if command.occlusion_query_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.occlusion_query_base, + 8, + GpuAccess::Write, + )?; + } + validate_zls(address_space, command.depth, command.layers, g18_subpage)?; + validate_zls(address_space, command.stencil, command.layers, g18_subpage)?; + validate_sampler( + address_space, + command.sampler_heap, + command.sampler_count as u32, + )?; + if command.vertex_helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + || command.fragment_helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + // `isp_bgobjvals` is the *whole* ISP_BGOBJVALS register value, not just + // the stencil clear: the UAPI documents "the bottom 8-bits contain the + // stencil buffer clear value", and every Mesa backend unconditionally + // seeds the upper bits with 0x300 before OR-ing the clear in + // (hk_cmd_draw.c `render->cr.isp_bgobjvals = 0x300;`, + // agx_pipe.c `c->isp_bgobjvals = 0x300;`, d12_queue.c `0x300 | stencil`). + // The legacy queue path passes the field straight through to the firmware + // (queue/render.rs `let load_bgobjvals = cmdbuf.isp_bgobjvals as u64`), so + // rejecting anything above 0xff refused every real render pass while the + // in-tree render selftest -- which leaves the field zero -- was accepted. + // Accept the register value; `build_*_descriptor` re-asserts the 0x300 + // bits with `stencil_clear_value | 0x300`, which is idempotent for the + // value userspace sends and still supplies 0x300 when it sends zero. + + let background = queue.full_program(address_space, command.background.usc)?; + let end_of_tile = queue.full_program(address_space, command.end_of_tile.usc)?; + let partial_background = queue.full_program(address_space, command.partial_background.usc)?; + let partial_end_of_tile = queue.full_program(address_space, command.partial_end_of_tile.usc)?; + validate_attachments(address_space, &vertex_attachments)?; + validate_attachments(address_space, &fragment_attachments)?; + + let utile_config = ((command.utile_width as u64 / 16) << 12) + | ((command.utile_height as u64 / 16) << 14) + | samples_log2; + let mut tile_config = 0x280; + if command.layers > 1 { + tile_config |= 1; + } + if command.flags & RENDER_PROCESS_EMPTY_TILES != 0 { + tile_config |= 0x1_0000; + } + + let mut parameters = internal.parameters; + parameters.width = command.width as u32; + parameters.height = command.height as u32; + parameters.encoder = command.vdm_base; + parameters.layers = command.layers; + parameters.utile_width = command.utile_width; + parameters.utile_height = command.utile_height; + parameters.utile_config = utile_config; + parameters.multisample_control = command.multisample_control; + parameters.ppp_control = command.ppp_control as u64; + parameters.tib_blocks = blocks_per_utile; + parameters.tile_config = tile_config; + parameters.depth_dimensions = command.depth_dimensions as u64; + parameters.depth_buffer = command.depth.base; + parameters.depth_aux_buffer = command.depth.compression_base; + parameters.depth_stride = command.depth.stride as u64; + parameters.depth_aux_stride = command.depth.compression_stride as u64; + parameters.stencil_buffer = command.stencil.base; + parameters.stencil_aux_buffer = command.stencil.compression_base; + parameters.stencil_stride = command.stencil.stride as u64; + parameters.stencil_aux_stride = command.stencil.compression_stride as u64; + parameters.depth_flags = command.zls_control; + // The firmware's internal partial-render phases preserve attached Z/S + // surfaces even when the render-pass boundary is store-only. Mesa already + // computes this distinction for its partial BG/EOT programs, but the + // current 256-byte render UAPI carries only the boundary ZLS word. Derive + // the exact uncompressed enable bits here until the append-only partial + // ZLS UAPI field is plumbed; tiling/compression/format bits remain those + // packed into the original word. + parameters.partial_depth_flags = command.zls_control + | if command.depth.base != 0 { (1 << 15) | (1 << 19) } else { 0 } + | if command.stencil.base != 0 { (1 << 14) | (1 << 18) } else { 0 }; + parameters.occlusion_query_base = command.occlusion_query_base; + parameters.scissor_array = command.scissor_base; + parameters.depth_bias_array = command.depth_bias_base; + parameters.sampler_array = command.sampler_heap; + parameters.sampler_count = command.sampler_count as u32; + parameters.process_empty_tiles = command.flags & RENDER_PROCESS_EMPTY_TILES != 0; + parameters.merge_upper_x_bits = command.merge_upper_x; + parameters.merge_upper_y_bits = command.merge_upper_y; + parameters.depth_clear_value_bits = command.depth_clear; + parameters.stencil_clear_value = command.stencil_clear; + parameters.load_pipeline_bind = g17p_bg_resource_spec( + command.background.resource_spec, + resource_spec_mask, + internal.parameters.bg_resource_prefix, + ); + parameters.load_pipeline = background; + parameters.store_pipeline_bind = command.end_of_tile.resource_spec & resource_spec_mask; + parameters.store_pipeline = end_of_tile; + // These high 17 bits are BG shader pass control, not an address class. + // Native derives main and partial masks from their respective shader + // outputs; they need not match. The default diagnostic prefix is zero, + // preserving both full RT0 output masks and no-output masks from userspace. + parameters.partial_load_pipeline_bind = g17p_bg_resource_spec( + command.partial_background.resource_spec, + resource_spec_mask, + internal.parameters.bg_resource_prefix, + ); + parameters.partial_load_pipeline = partial_background; + parameters.partial_store_pipeline_bind = + command.partial_end_of_tile.resource_spec & resource_spec_mask; + parameters.partial_store_pipeline = partial_end_of_tile; + + Ok(TranslatedRenderCommand { + render_order: g17_submission::G17PRenderOrder::default(), + render_barrier: UAPI_BARRIER_NONE, + compute_barrier: UAPI_BARRIER_NONE, + compute_wait_through: None, + flags: command.flags, + vdm_base: command.vdm_base, + parameters, + vertex_attachments, + fragment_attachments, + vertex_timestamps: command.vertex_timestamps, + fragment_timestamps: command.fragment_timestamps, + }) +} + +fn g17p_bg_resource_spec(resource_spec: u64, present_mask: u64, diagnostic_prefix: u64) -> u64 { + diagnostic_prefix | (resource_spec & present_mask) +} + +pub(crate) fn build_render_descriptors( + command: &TranslatedRenderCommand, + objects: RenderDescriptorObjects, + tiling_metadata: RenderDescriptorMetadata, + fragment_metadata: RenderDescriptorMetadata, + tiling: &mut [u8], + fragment: &mut [u8], +) -> Result<(), UapiTranslateError> { + g17_render::build_ta_descriptor(&command.parameters, objects, tiling_metadata, tiling)?; + g17_render::build_fragment_descriptor( + &command.parameters, + objects, + fragment_metadata, + fragment, + )?; + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedComputeCommand { + /// Monotonic logical index, independent of the hardware timestamp. + pub(crate) compute_sequence: u64, + /// Explicit same-engine prefix; only host-published timestamps need host waits. + pub(crate) compute_wait_through: Option, + pub(crate) render_wait_through: Option, + pub(crate) render_wait_last: Option, + pub(crate) usc_exec_base: u64, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) control_stream_terminator: u64, + pub(crate) sampler_heap: u64, + pub(crate) sampler_count: u32, + pub(crate) attachments: UapiAttachmentList, + pub(crate) g17p_add3_buffers: Option<[u64; 3]>, + pub(crate) timestamps: UapiTimestamps, + pub(crate) scs_layout: u64, + /// G18 only: the command carried `DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE`. + /// [`translate_compute_command`] refuses that bit like any other unknown + /// one and leaves this false; the G18 adapter admits the bit, strips it + /// before translation and sets this afterwards. + pub(crate) g18_texture_state: bool, + /// Validated native descriptor inputs; resource/launch integration is + /// separate, so this field does not by itself authorize scratch use. + pub(crate) g18_resource_descriptor: Option<[u32; 23]>, +} + +pub(crate) fn translate_compute_command( + command: UapiComputeCommand, + attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, +) -> Result { + queue.validate()?; + if command.flags & !COMPUTE_G17P_ADD3_PROOF != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + if command.g18_resources != [0; 16] { + return Err(UapiTranslateError::FlagsUnsupported); + } + if command.control_stream_base & 3 != 0 + || command.control_stream_end & 3 != 0 + || command.control_stream_end <= command.control_stream_base + { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.control_stream_base, + command.control_stream_end - command.control_stream_base, + GpuAccess::Read, + )?; + validate_sampler(address_space, command.sampler_heap, command.sampler_count)?; + if command.helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + let g17p_add3_buffers = if command.flags & COMPUTE_G17P_ADD3_PROOF != 0 { + let entries = attachments.as_slice(); + if entries.len() != 3 + || entries + .iter() + .any(|entry| entry.size < G17P_ADD3_BUFFER_BYTES) + { + return Err(UapiTranslateError::Add3ProofAttachments); + } + require_range( + address_space, + entries[0].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Read, + )?; + require_range( + address_space, + entries[1].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Read, + )?; + require_range( + address_space, + entries[2].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Write, + )?; + Some([entries[0].address, entries[1].address, entries[2].address]) + } else { + validate_attachments(address_space, &attachments)?; + None + }; + Ok(TranslatedComputeCommand { + compute_sequence: 0, + compute_wait_through: None, + render_wait_through: None, + render_wait_last: None, + usc_exec_base: queue.base, + control_stream_base: command.control_stream_base, + control_stream_end: command.control_stream_end, + control_stream_terminator: command.control_stream_end - 4, + sampler_heap: command.sampler_heap, + sampler_count: command.sampler_count, + attachments, + g17p_add3_buffers, + timestamps: command.timestamps, + scs_layout: command.scs_layout, + g18_texture_state: false, + g18_resource_descriptor: None, + }) +} + +/// Queue-owned inputs that are independent of one userspace compute command. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeInternalState { + pub(crate) preempt_base: u64, + pub(crate) dispatch_identity: u64, + pub(crate) context_id: u32, + pub(crate) work_ordinal: u32, + pub(crate) robustness: u64, + pub(crate) operand_state_base: u64, + pub(crate) usc_flist_id: u32, +} + +/// GPU addresses resolved from the UAPI timestamp handles while the object +/// table is locked. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ComputeTimestampAddresses { + pub(crate) start: u64, + pub(crate) end: u64, +} + +/// A compute command owns exactly one start/end pair. Both words must be +/// visible before the queue entry and its userspace fence can retire. +pub(crate) const fn completed_compute_timestamps(start: u64, end: u64) -> Option<[u64; 2]> { + if start == 0 || end == 0 { + None + } else { + Some([start, end]) + } +} + +/// Derive the descriptor/RCE checkpoint quartet from the exact command's +/// register program without publishing either object. +pub(crate) fn compute_resume_checkpoints( + command: &TranslatedComputeCommand, + internal: ComputeInternalState, +) -> Result<[u16; 4], UapiTranslateError> { + let registers = g17_compute::build_compute_registers_for_layout( + g17_compute::ComputeRegisterParameters { + preempt_base: internal.preempt_base, + cdm_base: command.control_stream_base, + usc_exec_base: command.usc_exec_base, + helper_binary: 0, + helper_data: 0, + helper_config: 0, + dispatch_identity: internal.dispatch_identity, + context_id: internal.context_id, + work_ordinal: internal.work_ordinal, + robustness: internal.robustness, + operand_state_base: internal.operand_state_base, + usc_flist_id: internal.usc_flist_id, + }, + command.scs_layout, + )?; + Ok(g17_compute::compute_resume_checkpoints(registers.as_slice())?) +} + +/// Build the complete selector-3 descriptor for a translated command. +/// +/// This is the boundary that consumes the command's CDM range, sampler state, +/// USC window, and timestamp destinations. The translated attachment list +/// remains in `command` for the queue's reservation and lifetime tracking. +pub(crate) fn build_compute_descriptor( + command: &TranslatedComputeCommand, + internal: ComputeInternalState, + objects: g17_compute::ComputeDescriptorObjects, + metadata: g17_compute::ComputeDescriptorMetadata, + timestamps: ComputeTimestampAddresses, + raw: &mut [u8], +) -> Result<[u16; 4], UapiTranslateError> { + let registers = g17_compute::build_compute_registers_for_layout( + g17_compute::ComputeRegisterParameters { + preempt_base: internal.preempt_base, + cdm_base: command.control_stream_base, + usc_exec_base: command.usc_exec_base, + helper_binary: 0, + helper_data: 0, + helper_config: 0, + dispatch_identity: internal.dispatch_identity, + context_id: internal.context_id, + work_ordinal: internal.work_ordinal, + robustness: internal.robustness, + operand_state_base: internal.operand_state_base, + usc_flist_id: internal.usc_flist_id, + }, + command.scs_layout, + )?; + let resume_checkpoints = g17_compute::build_compute_descriptor( + ®isters, + objects, + metadata, + g17_compute::ComputeDescriptorCommand { + cdm_terminator: command.control_stream_terminator, + sampler_array: command.sampler_heap, + sampler_count: command.sampler_count, + user_timestamp_start: timestamps.start, + user_timestamp_end: timestamps.end, + }, + raw, + )?; + Ok(resume_checkpoints) +} + +#[derive(Debug, Copy, Clone)] +pub(crate) struct ComputeEntryContext<'a> { + pub(crate) descriptor_flag_4c: bool, + pub(crate) converted_command_timestamp: u64, + pub(crate) barriers: &'a [G17PClBarrierDependency], + pub(crate) mcache: Option, + pub(crate) payload: [u64; 2], + pub(crate) event_mask: [u64; 4], + pub(crate) rce_kind: u8, + pub(crate) rce_bindings: [G17PClRceBinding; 4], + pub(crate) auxiliary: u64, +} + +impl TranslatedComputeCommand { + /// Native `processComputeSetup` writes descriptor +0x5c8 from whether the + /// derived SCS page count is nonzero. `scs_pages_for_layout` has already + /// validated the same layout before an entry can be published; its native + /// zero-page gate is exactly both payload fields being zero. + pub(crate) const fn scs_enabled(&self) -> bool { + self.scs_layout >> 32 != 0 + } + + pub(crate) const fn entry_operands<'a>( + &self, + context: ComputeEntryContext<'a>, + ) -> G17PClKickEntryOperands<'a> { + G17PClKickEntryOperands { + descriptor_flag_4c: context.descriptor_flag_4c, + // `generateCLRegisterListG16Common` uses the same +0x5c8 bit to + // prepend 0x17d9, and encodeCLKickEntry copies it to header bit61. + // Derive both from the command so they cannot diverge. + descriptor_flag_5c8: self.scs_enabled(), + converted_command_timestamp: context.converted_command_timestamp, + barriers: context.barriers, + mcache: context.mcache, + payload: context.payload, + event_mask: context.event_mask, + rce_kind: context.rce_kind, + rce_bindings: context.rce_bindings, + auxiliary: context.auxiliary, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const fn nibble(value: u8) -> u8 { + match value { + b'0'..=b'9' => value - b'0', + b'a'..=b'f' => value - b'a' + 10, + _ => 0xff, + } + } + + const fn hex(source: &[u8; M]) -> [u8; N] { + let mut out = [0u8; N]; + let mut index = 0; + while index < N { + out[index] = (nibble(source[index * 2]) << 4) | nibble(source[index * 2 + 1]); + index += 1; + } + out + } + + const RENDER_FIXTURE: [u8; 312] = hex::<312, 624>(b"02001800ffffffff00000a00000000000001000000000000000000000000000003001800ffffffff00000b0000000000000200000000000000000000000000000000f000ffffffff020000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000088000000000000000000000000000000020200004000300001000000202001040000000000000000040001004000000004000200080000000400010000000000040002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"); + const COMPUTE_FIXTURE: [u8; 104] = hex::<104, 208>(b"04001800ffffffff00000300000000000001000000000000000000000000000001004000ffffffff00000000000000000000010000000000400001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"); + + fn fnv1a64(raw: &[u8]) -> u64 { + let mut value = 0xcbf2_9ce4_8422_2325u64; + for byte in raw { + value ^= *byte as u64; + value = value.wrapping_mul(0x0000_0100_0000_01b3); + } + value + } + + struct TestAddressSpace; + + impl GpuAddressSpace for TestAddressSpace { + fn covers(&self, address: u64, size: u64, _access: GpuAccess) -> bool { + let end = match address.checked_add(size) { + Some(value) => value, + None => return false, + }; + (0x1_0000..=0x20_0000).contains(&address) && end <= 0x20_0000 + || (0x11_0000_0000..0x12_0000_0000).contains(&address) && end <= 0x12_0000_0000 + } + } + + fn window() -> QueueUscWindow { + QueueUscWindow { + base: 0x11_0000_0000, + user_start: 0x4000, + user_end: (1 << 39) - 0x8000, + } + } + + fn objects() -> RenderDescriptorObjects { + RenderDescriptorObjects { + record_a: 0xffff_fc20_0150_0000, + shared: 0xffff_fc20_0160_0000, + record_b: 0xffff_fc20_0158_0000, + zero: 0xffff_fc20_0168_0000, + } + } + + #[test] + fn m5_d32_envelope_rejects_wrong_layout_and_incomplete_backing() { + let mut parser = UapiCommandParser::new(&RENDER_FIXTURE); + let ParsedHardwareCommand::Render { mut payload, .. } = + parser.next_hardware().unwrap().unwrap() else { panic!("not render") }; + payload.width = 256; + payload.height = 256; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + payload.depth_dimensions = 0x00ff00ff; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload.depth.base = 0x10000; + payload.depth.stride = 0x3c001; + payload.zls_control = 0x80000; + payload.depth_clear = 0x3f800000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + payload.depth.compression_base = 0x60000; + payload.zls_control = 0x80044; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + let valid = payload; + // The real gears command formerly used G17P's 15-bit dimensions. + payload.depth_dimensions = 0x007f80ff; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload = valid; + payload.depth.compression_base = 0; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload = valid; + payload.depth.base = 0x1fc000; // First byte mapped, remainder absent. + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::AddressRange)); + payload = valid; + payload.depth.compression_base = 0x200000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::AddressRange)); + payload = valid; + // Every combination of the load and store bits is a render the + // application may ask for, including neither: a bound depth buffer + // that this render does not test is not written back. + for transfers in [0u64, 0x8000, 0x80000, 0x88000] { + payload.zls_control = 0x44 | transfers; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Ok(())); + } + // Nothing outside those two bits may move: stencil transfers, the + // twiddled tilings, the 16-bit format and the resolve bits all still + // need their own witnesses. + for stray in [0x2u64, 0x4000, 0x40000, 1 << 25, 1 << 56] { + payload.zls_control = 0x80044 | stray; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + } + payload = valid; + payload.layers = 2; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload = valid; + // The clear value is the application's float and reaches register + // 0x15301 unchanged; only a NaN payload is refused. + for clear in [0u32, 0x3f000000, 0xbf800000, 0x7f800000] { + payload.depth_clear = clear; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Ok(())); + } + payload.depth_clear = 0x7fc00000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + // An attached stencil plane contributes only its compression pair. + payload = valid; + payload.stencil.base = 0x10000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + payload.stencil.compression_base = 0x60000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload.zls_control = 0x80154; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + // Loading or storing it is outside the profile. + for transfer in [0x4000u64, 0x40000] { + payload.zls_control = 0x80154 | transfer; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + } + } + + #[test] + fn m5_resized_depth_validates_tile_padding_and_complete_backing() { + let mut parser = UapiCommandParser::new(&RENDER_FIXTURE); + let ParsedHardwareCommand::Render { mut payload, .. } = + parser.next_hardware().unwrap().unwrap() else { panic!("not render") }; + payload.width = 319; + payload.height = 191; + payload.depth_dimensions = 318 | (190 << 16); + payload.depth.base = 0x10000; + payload.depth.stride = 0x38001; // 5x3 pages, not width*height*4. + payload.depth.compression_base = 0x60000; + payload.depth.compression_stride = 0x7c000; // 512x256 / (16x16) *8. + payload.zls_control = 0x80044; + payload.depth_clear = 0x3f800000; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + let valid = payload; + // The array strides are discarded for a single-layer render, so any + // value in them is admitted -- including the underflow a surface + // smaller than one page produces. The backing store below is what + // actually bounds the render. + for stride in [0u32, 0x3c001, 0xffffc001] { + payload.depth.stride = stride; + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Ok(())); + } + payload = valid; + payload.depth.compression_stride = 0x3c000; // Old metadata stride. + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), Ok(())); + payload = valid; + payload.depth.compression_base = 0; // Metadata stride without metadata. + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::ZlsFields)); + payload = valid; + payload.depth.base = 0x1c8000; // Only 14 pages remain, but 15 are needed. + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::AddressRange)); + payload = valid; + payload.width = 1024; + payload.height = 1024; + payload.depth_dimensions = 1023 | (1023 << 16); + assert_eq!(validate_m5_reference_depth(&payload, &TestAddressSpace), + Err(UapiTranslateError::AddressRange)); // Four MiB exceeds this test VM. + } + + #[test] + fn render_fixture_translates_to_full_g17p_programs_and_descriptors() { + assert_eq!(fnv1a64(&RENDER_FIXTURE), 0xf056_ea1d_ef06_32c5); + let mut parser = UapiCommandParser::new(&RENDER_FIXTURE); + let command = parser.next_hardware().unwrap().unwrap(); + let (payload, vertex, fragment) = match command { + ParsedHardwareCommand::Render { + index, + payload, + vertex_attachments, + fragment_attachments, + .. + } => { + assert_eq!(index, 3); + (payload, vertex_attachments, fragment_attachments) + } + _ => panic!("wrong command kind"), + }; + assert!(parser.next_hardware().unwrap().is_none()); + assert_eq!(parser.finish(), Ok(1)); + + let mut internal = G17pRenderParameters::source_rsrc8(); + internal.context_base = 0; + // This frozen descriptor fixture predates userspace high-dword masks; + // retain its explicit legacy diagnostic mask, not a default injection. + internal.bg_resource_prefix = 0x0007_8000_0000_0000; + let translated = translate_render_command( + payload, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: internal, + }, + ) + .unwrap(); + assert_eq!(translated.parameters.load_pipeline, 0x11_0001_0000); + assert_eq!(translated.parameters.store_pipeline, 0x11_0002_0000); + assert_eq!( + translated.parameters.load_pipeline_bind, + 0x0007_8000_0000_0040 + ); + assert_eq!(translated.parameters.store_pipeline_bind, 0x8); + assert_eq!(translated.parameters.partial_load_pipeline, 0x11_0001_0000); + assert_eq!(translated.parameters.partial_store_pipeline, 0x11_0002_0000); + assert_eq!(translated.parameters.utile_config, 0xa000); + assert_eq!(translated.parameters.tib_blocks, 2); + assert_eq!(translated.parameters.tile_config, 0x1_0280); + assert_eq!(translated.parameters.encoder, 0x1_0000); + assert_eq!( + translated.vertex_attachments.as_slice(), + &[UapiAttachment { + address: 0xa_0000, + size: 0x100, + }] + ); + assert_eq!( + translated.fragment_attachments.as_slice(), + &[UapiAttachment { + address: 0xb_0000, + size: 0x200, + }] + ); + + let mut tiling = [0u8; g17_render::TA_DESCRIPTOR_SIZE]; + let mut fragment_raw = [0u8; g17_render::FRAGMENT_DESCRIPTOR_SIZE]; + build_render_descriptors( + &translated, + objects(), + RenderDescriptorMetadata::source(g17_render::RenderDescriptorKind::Tiling), + RenderDescriptorMetadata::source(g17_render::RenderDescriptorKind::Fragment), + &mut tiling, + &mut fragment_raw, + ) + .unwrap(); + assert_eq!(fnv1a64(&tiling), 0x8581_5830_2d5b_e227); + assert_eq!(fnv1a64(&fragment_raw), 0x6688_e9f8_d061_d776); + } + + #[test] + fn background_pass_masks_preserve_userspace_and_explicit_diagnostics() { + let default_prefix = G17pRenderParameters::source_rsrc8().bg_resource_prefix; + assert_eq!(default_prefix, 0); + assert_eq!(g17p_bg_resource_spec(0xffff_8000_0000_0040, u64::MAX, default_prefix), + 0xffff_8000_0000_0040); + assert_eq!(g17p_bg_resource_spec(0x40, u64::MAX, default_prefix), 0x40); + assert_eq!(g17p_bg_resource_spec(0xffff_8000_0000_0040, u32::MAX as u64, default_prefix), + 0x40); + assert_eq!(g17p_bg_resource_spec(0x40, u64::MAX, 0xffff_8000_0000_0000), + 0xffff_8000_0000_0040); + } + + #[test] + fn partial_zls_preserves_depth_and_stencil_across_internal_splits() { + let mut parser = UapiCommandParser::new(&RENDER_FIXTURE); + let (mut payload, vertex, fragment) = match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Render { + payload, + vertex_attachments, + fragment_attachments, + .. + } => (payload, vertex_attachments, fragment_attachments), + _ => panic!("wrong command kind"), + }; + payload.depth.base = 0x1_8000; + payload.stencil.base = 0x1_c000; + payload.zls_control = 0x000c_0000; + + let translated = translate_render_command( + payload, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: G17pRenderParameters::source_rsrc8(), + }, + ) + .unwrap(); + assert_eq!(translated.parameters.depth_flags, 0x000c_0000); + assert_eq!(translated.parameters.partial_depth_flags, 0x000c_c000); + } + + #[test] + fn render_resource_spec_high_dwords_reach_g17_register_programs() { + const RENDER_HEADER: usize = 64; + const PAYLOAD: usize = RENDER_HEADER + UAPI_COMMAND_HEADER_SIZE; + let mut extended = [0u8; RENDER_FIXTURE.len() + 16]; + extended[..RENDER_FIXTURE.len()].copy_from_slice(&RENDER_FIXTURE); + extended[RENDER_HEADER + 2..RENDER_HEADER + 4] + .copy_from_slice(&(UAPI_RENDER_SIZE as u16).to_le_bytes()); + extended[PAYLOAD..PAYLOAD + 4] + .copy_from_slice(&(RENDER_PROCESS_EMPTY_TILES | RENDER_RSRC_SPEC_HI).to_le_bytes()); + + for (offset, value) in [(0xac, 0x42), (0xb4, 0x40), (0xbc, 0x43), (0xc4, 0x41)] { + extended[PAYLOAD + offset..PAYLOAD + offset + 4] + .copy_from_slice(&(value as u32).to_le_bytes()); + } + for (offset, value) in [ + (0xf0, 0x0007_8000), + (0xf4, 0x0007_8001), + (0xf8, 0xffff_8002), + (0xfc, 0), + ] { + extended[PAYLOAD + offset..PAYLOAD + offset + 4] + .copy_from_slice(&(value as u32).to_le_bytes()); + } + + let mut parser = UapiCommandParser::new(&extended); + let (payload, vertex, fragment) = match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Render { + payload, + vertex_attachments, + fragment_attachments, + .. + } => (payload, vertex_attachments, fragment_attachments), + _ => panic!("wrong command kind"), + }; + assert_eq!(parser.finish(), Ok(1)); + let translated = translate_render_command( + payload, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: G17pRenderParameters::source_rsrc8(), + }, + ) + .unwrap(); + assert_eq!( + translated.parameters.store_pipeline_bind, + 0x0007_8000_0000_0040 + ); + assert_eq!( + translated.parameters.partial_store_pipeline_bind, + 0x0007_8001_0000_0041 + ); + assert_eq!( + translated.parameters.load_pipeline_bind, + 0xffff_8002_0000_0042 + ); + assert_eq!( + translated.parameters.partial_load_pipeline_bind, + 0x43 + ); + let registers = g17_render::build_fragment_registers(&translated.parameters).unwrap(); + assert_eq!( + registers + .iter() + .find(|register| register.number == 0x15369) + .unwrap() + .value, + 0xffff_8002_0000_0042 + ); + + // Clearing RSRC_SPEC_HI means "the high dwords are not present", so + // they are masked off -- not grounds to refuse the submission. + let mut no_flag = payload; + no_flag.flags &= !RENDER_RSRC_SPEC_HI; + let masked = translate_render_command( + no_flag, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: G17pRenderParameters::source_rsrc8(), + }, + ) + .unwrap(); + assert_eq!(masked.parameters.load_pipeline_bind, 0x42); + assert_eq!( + masked.parameters.partial_load_pipeline_bind, + 0x43 + ); + + // An explicitly selected diagnostic OR-mask still applies; it is no + // longer silently imposed on ordinary no-output background programs. + let mut diagnostic = G17pRenderParameters::source_rsrc8(); + diagnostic.bg_resource_prefix = 0xffff_8000_0000_0000; + let overridden = translate_render_command( + no_flag, vertex, fragment, window(), &TestAddressSpace, + RenderInternalState { parameters: diagnostic }, + ) + .unwrap(); + assert_eq!(overridden.parameters.load_pipeline_bind, 0xffff_8000_0000_0042); + assert_eq!( + overridden.parameters.partial_load_pipeline_bind, + 0xffff_8000_0000_0043 + ); + } + + #[test] + fn compute_fixture_translates_to_existing_cl_operands() { + assert_eq!(fnv1a64(&COMPUTE_FIXTURE), 0xeb42_2636_cc6a_c274); + let mut parser = UapiCommandParser::new(&COMPUTE_FIXTURE); + let (payload, attachments) = match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Compute { + index, + payload, + attachments, + .. + } => { + assert_eq!(index, 2); + (payload, attachments) + } + _ => panic!("wrong command kind"), + }; + assert!(parser.next_hardware().unwrap().is_none()); + assert_eq!(parser.finish(), Ok(1)); + let translated = + translate_compute_command(payload, attachments, window(), &TestAddressSpace).unwrap(); + assert_eq!(translated.control_stream_base, 0x1_0000); + assert_eq!(translated.control_stream_end, 0x1_0040); + assert_eq!(translated.control_stream_terminator, 0x1_003c); + assert_eq!(translated.usc_exec_base, 0x11_0000_0000); + assert_eq!(translated.g17p_add3_buffers, None); + assert_eq!( + translated.attachments.as_slice(), + &[UapiAttachment { + address: 0x3_0000, + size: 0x100, + }] + ); + + let empty = G17PClRceBinding { address: 0, tag: 0 }; + let operands = translated.entry_operands(ComputeEntryContext { + descriptor_flag_4c: false, + converted_command_timestamp: 1, + barriers: &[], + mcache: None, + payload: [0xffff_fc20_c001_a000, 0xffff_fc20_c000_0480], + event_mask: [1, 2, 3, 4], + rce_kind: 0, + rce_bindings: [empty; 4], + auxiliary: 0x003f_ffff_ffff_ffff, + }); + assert_eq!( + operands.payload, + [0xffff_fc20_c001_a000, 0xffff_fc20_c000_0480] + ); + assert_eq!(operands.event_mask, [1, 2, 3, 4]); + assert_eq!(operands.auxiliary, 0x003f_ffff_ffff_ffff); + assert!(!operands.descriptor_flag_5c8); + + let mut scs_translated = translated; + scs_translated.scs_layout = 0x48u64 << 48; + let scs_operands = scs_translated.entry_operands(ComputeEntryContext { + descriptor_flag_4c: false, + converted_command_timestamp: 1, + barriers: &[], + mcache: None, + payload: [0xffff_fc20_c001_a000, 0xffff_fc20_c000_0480], + event_mask: [1, 2, 3, 4], + rce_kind: 0, + rce_bindings: [empty; 4], + auxiliary: 0x003f_ffff_ffff_ffff, + }); + assert!(scs_operands.descriptor_flag_5c8); + let entry = g17_submission::prepare_g17p_cl_kick_entry( + g17_submission::G17PClKickProducerState { + queue_id: 4, + num_stamps: 4, + entry_stride: 0x200, + completion_selector: 2, + qos: g17_submission::G17PClQosConfig { + queue_byte_32: 0, + class_40: 0, + word_48: 0, + }, + submitted: 0, + current_timestamp: 0, + last_add_kicks_timestamp: 0, + base_timestamp: 0, + previous_valid: false, + previous_timestamp: 0, + }, + operands, + ) + .unwrap(); + assert_eq!(fnv1a64(&entry.entry), 0xbcc4_342d_108e_f9d0); + + let mut descriptor = [0u8; g17_compute::COMPUTE_DESCRIPTOR_SIZE]; + build_compute_descriptor( + &translated, + ComputeInternalState { + preempt_base: 0x100_0404_0000, + dispatch_identity: 0x1234_5678_0000_0001, + context_id: 3, + work_ordinal: 7, + robustness: 0x100_0430_0000, + operand_state_base: 0x70_0022_8000, + usc_flist_id: 2, + }, + g17_compute::ComputeDescriptorObjects { + scheduler_record: 0xffff_fc20_c099_8100, + descriptor_low_alias: 0x70_0034_0000, + dispatch_a: 0xffff_fc20_001c_8028, + dispatch_b: 0xffff_fc20_c07c_0028, + status_a: 0xffff_fc20_0002_4c68, + status_b: 0xffff_fc20_0002_4c70, + shared_control: 0xffff_fc20_c095_0000, + usage_record: 0xffff_fc20_0171_0000, + }, + g17_compute::ComputeDescriptorMetadata { + submit_sequence: 5, + context_id: 3, + grid_index: 2, + work_ordinal: 7, + queue_submission: 8, + queue_ordinal: 7, + submission_index: 9, + support_control: 0xe0a0_0001, + support_flags: 0, + }, + ComputeTimestampAddresses { + start: 0x200_0000_4000, + end: 0x200_0000_4008, + }, + &mut descriptor, + ) + .unwrap(); + assert_eq!( + u64::from_le_bytes(descriptor[0x0ee0..0x0ee8].try_into().unwrap()), + translated.control_stream_terminator + ); + assert_eq!( + u64::from_le_bytes(descriptor[0x0f8c..0x0f94].try_into().unwrap()), + 0x200_0000_4000 + ); + assert_eq!( + u64::from_le_bytes(descriptor[0x8c..0x94].try_into().unwrap()), + translated.usc_exec_base + ); + assert_eq!(completed_compute_timestamps(0, 9), None); + assert_eq!(completed_compute_timestamps(7, 0), None); + assert_eq!(completed_compute_timestamps(7, 9), Some([7, 9])); + } + + #[test] + fn add3_proof_maps_exactly_three_buffers_into_the_retained_table() { + let mut parser = UapiCommandParser::new(&COMPUTE_FIXTURE); + let (mut payload, _) = match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Compute { + payload, + attachments, + .. + } => (payload, attachments), + _ => panic!("wrong command kind"), + }; + payload.flags = COMPUTE_G17P_ADD3_PROOF; + let mut attachments = UapiAttachmentList::EMPTY; + attachments.entries[..3].copy_from_slice(&[ + UapiAttachment { + address: 0x3_0000, + size: G17P_ADD3_BUFFER_BYTES, + }, + UapiAttachment { + address: 0x3_8000, + size: G17P_ADD3_BUFFER_BYTES, + }, + UapiAttachment { + address: 0x4_0000, + size: G17P_ADD3_BUFFER_BYTES, + }, + ]); + attachments.count = 3; + + let translated = + translate_compute_command(payload, attachments, window(), &TestAddressSpace).unwrap(); + assert_eq!( + translated.g17p_add3_buffers, + Some([0x3_0000, 0x3_8000, 0x4_0000]) + ); + + let mut short = attachments; + short.entries[2].size = G17P_ADD3_BUFFER_BYTES - 1; + assert_eq!( + translate_compute_command(payload, short, window(), &TestAddressSpace), + Err(UapiTranslateError::Add3ProofAttachments) + ); + } + + #[test] + fn parser_zero_extends_old_payloads_and_checks_new_tail_bytes() { + let mut command = [0u8; UAPI_COMMAND_HEADER_SIZE + 8]; + command[0..2].copy_from_slice(&COMMAND_COMPUTE.to_le_bytes()); + command[2..4].copy_from_slice(&8u16.to_le_bytes()); + command[4..6].copy_from_slice(&UAPI_BARRIER_NONE.to_le_bytes()); + command[6..8].copy_from_slice(&UAPI_BARRIER_NONE.to_le_bytes()); + let mut parser = UapiCommandParser::new(&command); + match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Compute { payload, .. } => { + assert_eq!( + payload, + UapiComputeCommand { + flags: 0, + sampler_count: 0, + control_stream_base: 0, + control_stream_end: 0, + sampler_heap: 0, + helper: UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }, + timestamps: UapiTimestamps { + start: UapiTimestamp { + handle: 0, + offset: 0, + }, + end: UapiTimestamp { + handle: 0, + offset: 0, + }, + }, + scs_layout: 0, + g18_resources: [0; 16], + prior_vdm: 0, + prior_cdm: 0, + } + ); + } + _ => panic!("wrong command kind"), + } + + let mut extended = [0u8; UAPI_COMPUTE_SIZE + 1]; + extended[UAPI_COMPUTE_SIZE] = 1; + assert_eq!( + UapiComputeCommand::parse(&extended), + Err(UapiTranslateError::PayloadTail) + ); + } + + #[test] + fn translation_checks_sampler_program_and_attachment_ranges() { + let mut parser = UapiCommandParser::new(&RENDER_FIXTURE); + let (mut payload, vertex, fragment) = match parser.next_hardware().unwrap().unwrap() { + ParsedHardwareCommand::Render { + payload, + vertex_attachments, + fragment_attachments, + .. + } => (payload, vertex_attachments, fragment_attachments), + _ => panic!("wrong command kind"), + }; + payload.sampler_count = 1; + assert_eq!( + translate_render_command( + payload, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: G17pRenderParameters::source_rsrc8(), + }, + ), + Err(UapiTranslateError::SamplerPair) + ); + payload.sampler_count = 0; + payload.end_of_tile.usc = 0; + assert_eq!( + translate_render_command( + payload, + vertex, + fragment, + window(), + &TestAddressSpace, + RenderInternalState { + parameters: G17pRenderParameters::source_rsrc8(), + }, + ), + Err(UapiTranslateError::ProgramOffset) + ); + } +} + + +#[cfg(test)] +mod exact_prior_payload_tests { + use super::*; + + #[test] + fn historical_wire_offsets_preserve_scs_and_render_high_words() { + let mut render = [0u8; 272]; + render[0xf0..0xf4].copy_from_slice(&0x12345678u32.to_le_bytes()); + render[256..264].copy_from_slice(&0x123456789abcdef0u64.to_le_bytes()); + render[264..272].copy_from_slice(&0xfedcba9876543210u64.to_le_bytes()); + let p = UapiRenderCommand::parse(&render).unwrap(); + assert_eq!(p.prior_vdm, 0x123456789abcdef0); + assert_eq!(p.prior_cdm, 0xfedcba9876543210); + let old = UapiRenderCommand::parse(&render[..256]).unwrap(); + assert_eq!(old.end_of_tile, p.end_of_tile); + assert_eq!((old.prior_vdm, old.prior_cdm), (0, 0)); + let mut compute = [0u8; 88]; + compute[64..72].copy_from_slice(&0x8877665544332211u64.to_le_bytes()); + compute[72..80].copy_from_slice(&0x0123456789abcdefu64.to_le_bytes()); + compute[80..88].copy_from_slice(&0xfedcba9876543210u64.to_le_bytes()); + let c = UapiComputeCommand::parse(&compute).unwrap(); + assert_eq!(c.scs_layout, 0x8877665544332211); + assert_eq!((c.prior_vdm, c.prior_cdm), (0x0123456789abcdef, 0xfedcba9876543210)); + let old72 = UapiComputeCommand::parse(&compute[..72]).unwrap(); + assert_eq!(old72.scs_layout, c.scs_layout); + assert_eq!((old72.prior_vdm, old72.prior_cdm), (0, 0)); + let old64 = UapiComputeCommand::parse(&compute[..64]).unwrap(); + assert_eq!((old64.scs_layout, old64.prior_vdm, old64.prior_cdm), (0, 0, 0)); + } + + #[test] + fn truncated_extensions_zero_fill_and_unknown_nonzero_tails_reject() { + let mut render = [0u8; 280]; + render[256] = 7; + assert_eq!(UapiRenderCommand::parse(&render[..257]).unwrap().prior_vdm, 7); + assert!(UapiRenderCommand::parse(&render).is_ok()); + render[272] = 1; + assert_eq!(UapiRenderCommand::parse(&render), Err(UapiTranslateError::PayloadTail)); + let mut compute = [0u8; 96]; + compute[80] = 9; + assert_eq!(UapiComputeCommand::parse(&compute[..81]).unwrap().prior_cdm, 9); + assert!(UapiComputeCommand::parse(&compute).is_ok()); + compute[88] = 1; + assert_eq!(UapiComputeCommand::parse(&compute), Err(UapiTranslateError::PayloadTail)); + } +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/compute_inputs.rs b/drivers/gpu/drm/asahi/g18/uapi/compute_inputs.rs new file mode 100644 index 0000000000000..2f6ad2b0cf46f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/compute_inputs.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded G18 compute resource demand, before pool sizing or allocation. +//! +//! Derived from 26A428 HAL400 SpillInfoGen5::checkSpillParamsForCompute and +//! allocateUSCSpillBuffer. It was compared +//! with the original code. This module is not yet +//! connected to submission; it does not admit scratch or allocate GPU memory. + +/// Original Device constructor default (0x1ace020b8 in the exact BaseSystem). +/// Capacity is 64 KiB plus this many 4 KiB units; 63 disables the limit. +pub(crate) const G18_TGMEM_POLICY: u32 = 12; + +/// Inputs in native units. Main and constant-program storage are independent. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ComputeDemand { + pub(crate) scratch_bytes: u32, + pub(crate) constant_scratch_bytes: u32, + pub(crate) registers_32: u32, + pub(crate) ipr_bytes: u32, + pub(crate) constant_ipr_bytes: u32, + pub(crate) threadgroup_bytes: u32, + pub(crate) imageblock_bytes: u32, + /// Zero means unknown local geometry, not an empty global grid. + pub(crate) threads: u32, +} + +/// Effective compiler metadata, including any linked prolog/epilog demand. +/// Counts use Mesa's units. This is an internal type, not a userspace ABI. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct G18ComputeShader { + pub(crate) registers_16: u32, + pub(crate) scratch_bytes: u32, + pub(crate) spectpr_scratch_bytes: u32, + pub(crate) staging_16: u32, + pub(crate) staging_known: bool, + pub(crate) constant_present: bool, + pub(crate) constant_registers_16: u32, + pub(crate) constant_scratch_bytes: u32, + pub(crate) constant_staging_16: u32, + pub(crate) constant_staging_known: bool, + pub(crate) local_bytes: u32, + pub(crate) imageblock_stride: u32, +} + +impl G18ComputeDemand { + /// Legacy metadata v1 and precompiled G17G kernels use byte offsets in + /// element-index fields, requiring a fourfold SPECLM footprint. + pub(crate) fn from_shader(shader: G18ComputeShader, local: Option<[u32; 3]>, + variable_shared_bytes: u32) -> Option { + Self::from_shader_scaled(shader, local, variable_shared_bytes, 4) + } + + /// Metadata v2 describes shaders with correctly lowered element indices. + /// Their static and variable shared storage both count actual bytes. + pub(crate) fn from_shader_element_indices(shader: G18ComputeShader, + local: Option<[u32; 3]>, variable_shared_bytes: u32) -> Option { + Self::from_shader_scaled(shader, local, variable_shared_bytes, 1) + } + + fn from_shader_scaled(shader: G18ComputeShader, local: Option<[u32; 3]>, + variable_shared_bytes: u32, shared_scale: u32) -> Option { + if shader.registers_16 == 0 || shader.registers_16 > 256 + || !shader.staging_known || shader.staging_16 > 0xffff + || shader.imageblock_stride != 0 + { + return None; + } + if shader.constant_present { + if shader.constant_registers_16 == 0 || shader.constant_registers_16 > 256 + || !shader.constant_staging_known || shader.constant_staging_16 > 0xffff + { + return None; + } + } else if shader.constant_registers_16 != 0 || shader.constant_scratch_bytes != 0 + || shader.constant_staging_16 != 0 || shader.constant_staging_known + { + return None; + } + let threads = if let Some(dims) = local { + if dims.contains(&0) { return None; } + dims[0].checked_mul(dims[1])?.checked_mul(dims[2])? + } else { 0 }; + if threads > 1024 { return None; } + let scratch = shader.scratch_bytes.max(shader.spectpr_scratch_bytes); + // Match the qualified static AGX3 TPS frame's 16-byte alignment. + let scratch = scratch.checked_add(15)? & !15; + if scratch > 0xffff * 4 || shader.constant_scratch_bytes > 0xffff * 4 { + return None; + } + let shared = shader.local_bytes.checked_add(variable_shared_bytes)?.checked_mul(shared_scale)?; + let shared = if shared == 0 { 0 } else { (shared.checked_add(63)? & !63).max(256) }; + if shared > 32768 { return None; } + Some(Self { + scratch_bytes: scratch, + constant_scratch_bytes: shader.constant_scratch_bytes, + registers_32: (shader.registers_16 + 1) / 2, + ipr_bytes: shader.staging_16 * 2, + constant_ipr_bytes: shader.constant_staging_16 * 2, + threadgroup_bytes: shared, + imageblock_bytes: 0, + threads, + }) + } +} + +/// Native compute-only accumulator. Keep component maxima until serialization. +/// The two pointer words stay zero; no host or GPU pointer is encoded here. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct G18ComputeInputs { + words: [u32; 58], +} + +impl G18ComputeInputs { + pub(crate) fn new() -> Self { + let mut result = Self { words: [0; 58] }; + result.words[0xc4 / 4] = 4; + result.words[0xc8 / 4] = 1; + result.words[0xcc / 4] = 128; + result + } + + fn maximum(&mut self, offset: usize, value: u64) -> Option<()> { + let value = u32::try_from(value).ok()?; + self.words[offset / 4] = self.words[offset / 4].max(value); + Some(()) + } + + /// Arithmetic domain covers 16-bit SPECTPR word counts, 128 32-bit GPRs, + /// current GL threadgroup memory, and at most 1024 local invocations. + /// Validation is independent of later pool/backing limits. A rejected + /// merge never modifies the caller's accumulated state. + pub(crate) fn merge(&mut self, d: G18ComputeDemand, tgmem_policy: u32) -> Option<()> { + if d.scratch_bytes > 0xffff * 4 + || d.constant_scratch_bytes > 0xffff * 4 + || d.registers_32 > 128 + || d.ipr_bytes > 0xffff * 2 + || d.constant_ipr_bytes > 0xffff * 2 + || d.threadgroup_bytes > 32768 + || d.imageblock_bytes > 32768 + || d.threads > 1024 + || tgmem_policy > 63 + { + return None; + } + + // Calculate in u64, retaining native rounding order. The input bounds + // make every intermediate fit the original unsigned 32-bit domain. + let align = |v: u64, a: u64| (v + a - 1) & !(a - 1); + let pages = |v: u64| (v + 8191) / 8192; + let warps = if d.threads == 0 { 32 } else { (u64::from(d.threads) + 31) / 32 }; + let mut groups = 96 / if d.threads == 0 { 1 } else { warps }; + if d.threadgroup_bytes != 0 && tgmem_policy < 63 { + let capacity = 65536 + u64::from(tgmem_policy) * 4096; + groups = groups.min(capacity / align(u64::from(d.threadgroup_bytes), 64)); + } + let ideal_warps = (groups * warps).min(96); + let minimum_page_warps = warps.min(8); + let ideal_page_warps = (groups * warps).min(8); + + let scratch_warp = align(u64::from(d.scratch_bytes) * 32 + 896, 1024); + let register_warp = align(u64::from(d.registers_32) * 128, 1024); + let ipr_warp = align(u64::from(d.ipr_bytes) * 32, 1024); + let constant_scratch = align(u64::from(d.constant_scratch_bytes) * 32 + 896, 1024); + let constant_ipr = align(u64::from(d.constant_ipr_bytes) * 32, 1024); + let constant_floor = (constant_scratch + constant_ipr) / 1024 + 16; + let warp_kib = (scratch_warp + register_warp + ipr_warp) / 1024; + let local_kib = align(u64::from(d.threadgroup_bytes), 1024) / 1024; + let image_kib = align(u64::from(d.imageblock_bytes), 1024) / 1024; + let scratch_pages = pages(scratch_warp); + let register_pages = pages(register_warp); + let ipr_pages = pages(ipr_warp); + let constant_scratch_pages = pages(constant_scratch); + let constant_ipr_pages = pages(constant_ipr); + let local_pages = pages(u64::from(d.threadgroup_bytes)); + let image_pages = pages(u64::from(d.imageblock_bytes)); + + let mut next = *self; + next.words[0x08 / 4] = 3; + next.maximum(0x44, (warps * warp_kib).max(constant_floor) + local_kib + image_kib)?; + next.maximum(0x84, (ideal_warps * warp_kib).max(constant_floor) + groups * (local_kib + image_kib))?; + next.maximum(0x0c, (minimum_page_warps * register_pages).max(2))?; + next.maximum(0x10, (minimum_page_warps * scratch_pages).max(constant_scratch_pages))?; + next.maximum(0x14, (minimum_page_warps * ipr_pages).max(constant_ipr_pages))?; + next.maximum(0x18, register_pages.max(2))?; + next.maximum(0x1c, scratch_pages.max(constant_scratch_pages))?; + next.maximum(0x20, ipr_pages.max(constant_ipr_pages))?; + next.maximum(0x24, local_pages)?; + next.maximum(0x28, image_pages)?; + next.maximum(0x30, local_pages)?; + next.maximum(0x34, image_pages)?; + next.maximum(0x4c, (ideal_page_warps * register_pages).max(2))?; + next.maximum(0x50, (ideal_page_warps * scratch_pages).max(constant_scratch_pages))?; + next.maximum(0x54, (ideal_page_warps * ipr_pages).max(constant_ipr_pages))?; + next.maximum(0x58, register_pages.max(2))?; + next.maximum(0x5c, scratch_pages.max(constant_scratch_pages))?; + next.maximum(0x60, ipr_pages.max(constant_ipr_pages))?; + next.maximum(0x64, local_pages * groups.min(6))?; + next.maximum(0x68, image_pages * groups.min(6))?; + next.maximum(0x70, local_pages)?; + next.maximum(0x74, image_pages)?; + *self = next; + Some(()) + } + + /// Original endComputePass contributes an all-zero, unknown-geometry job. + pub(crate) fn finish(&mut self) -> Option<()> { + self.merge(G18ComputeDemand::default(), G18_TGMEM_POLICY) + } + + /// Compute-only AGXSMADescRec. Render/ray/protected-memory contributors + /// remain zero and cannot be admitted through this interface. + pub(crate) fn descriptor(&self) -> [u32; 23] { + let w = |off: usize| self.words[off / 4]; + [w(0x44), w(0x0c) + w(0x10) + w(0x14) + w(0x24) + w(0x28), 0, 0, + w(0x84), w(0x4c) + w(0x50) + w(0x54) + w(0x64) + w(0x68), 0, 0, + w(0x58), w(0x5c), w(0x60), w(0x70), w(0x74), 0, 0, 0, + w(0xc8), 0, 0, w(0xcc), w(0xc4), 0, 0] + } + + /// Expose component state for the retained original-instruction audit. + pub(crate) fn component_words(&self) -> &[u32; 58] { + &self.words + } +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/compute_metadata.rs b/drivers/gpu/drm/asahi/g18/uapi/compute_metadata.rs new file mode 100644 index 0000000000000..71eaa91bb705f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/compute_metadata.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Versioned per-dispatch compiler metadata, before any GPU allocation. + +use crate::g18::uapi::compute_inputs::{G18ComputeDemand, G18ComputeInputs, G18ComputeShader, + G18_TGMEM_POLICY}; + +pub(crate) struct G18ComputeMetadata { + pub(crate) demand: G18ComputeDemand, + pub(crate) descriptor: [u32; 23], +} + +impl G18ComputeMetadata { + /// `drm_asahi_g18_compute_resources`, sixteen little-endian u32 words. + /// Unknown local geometry has zero dimensions and its own flag; it must + /// not accidentally become a one-thread group in the native accumulator. + pub(crate) fn decode(words: [u32; 16]) -> Option { + let flags = words[1]; + if !matches!(words[0], 1 | 2) || flags & !0xf != 0 || words[15] != 0 { + return None; + } + let local = if flags & 8 != 0 { + if words[12..15] != [0; 3] { return None; } + None + } else { + Some([words[12], words[13], words[14]]) + }; + let shader = G18ComputeShader { + registers_16: words[2], + scratch_bytes: words[3], + spectpr_scratch_bytes: words[4], + staging_16: words[5], + staging_known: flags & 1 != 0, + constant_present: flags & 2 != 0, + constant_registers_16: words[6], + constant_scratch_bytes: words[7], + constant_staging_16: words[8], + constant_staging_known: flags & 4 != 0, + local_bytes: words[9], + imageblock_stride: words[10], + }; + let demand = if words[0] == 2 { + G18ComputeDemand::from_shader_element_indices(shader, local, words[11])? + } else { + G18ComputeDemand::from_shader(shader, local, words[11])? + }; + let mut inputs = G18ComputeInputs::new(); + inputs.merge(demand, G18_TGMEM_POLICY)?; + inputs.finish()?; + Some(Self { demand, descriptor: inputs.descriptor() }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/compute_plan.rs b/drivers/gpu/drm/asahi/g18/uapi/compute_plan.rs new file mode 100644 index 0000000000000..39222ca5aba43 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/compute_plan.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G18 shared compute pool sizing for the measured J873 profile. +//! +//! Native 26A428 prepareLockedBegin, compute sizing and allocatePagesEnd. +//! This is a pure plan for one active compute job in a fresh/retired pool. +//! It does not allocate memory or enable scratch submission. Compared with +//! the original code. + +pub(crate) const G18_COMPUTE_BLOCK_BYTES: u64 = 0x20_0000; +pub(crate) const G18_COMPUTE_MAX_BYTES: u64 = 0x8000_0000; +const USCS: u64 = 12; +const CHUNKS_PER_USC: u64 = (G18_COMPUTE_MAX_BYTES / 0x1_0000 / USCS) * 64; +pub(crate) const G18_COMPUTE_DIRECTORY_STRIDE: u64 = 0x15_8000; +pub(crate) const G18_COMPUTE_DIRECTORY_PAGE: u64 = 0x4000; + +/// Sparse directory commitment for the measured flags3 / 16-KiB profile. +/// Initial anchor pages remain mapped. Additional pages form the tail of +/// each USC stride; native growth rounds to quarter/half/full stride tiers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18ComputeDirectory { + pub(crate) tail_pages_per_usc: u32, +} + +impl G18ComputeDirectory { + pub(crate) fn for_blocks(blocks: u32) -> Option { + if blocks == 0 || u64::from(blocks) > G18_COMPUTE_MAX_BYTES / G18_COMPUTE_BLOCK_BYTES { + return None; + } + let units = u64::from(blocks) * (G18_COMPUTE_BLOCK_BYTES >> 16); + let per_usc = (units + USCS - 1) / USCS; + let initial = ((G18_COMPUTE_DIRECTORY_PAGE - 0x400) >> 9) + 8; + let excess = per_usc.saturating_sub(initial); + let needed = (excess * 512 + G18_COMPUTE_DIRECTORY_PAGE - 1) + / G18_COMPUTE_DIRECTORY_PAGE; + let stride_pages = G18_COMPUTE_DIRECTORY_STRIDE / G18_COMPUTE_DIRECTORY_PAGE; + let pages = if needed == 0 { 0 } + else if needed <= stride_pages / 4 { stride_pages / 4 } + else if needed <= stride_pages / 2 { stride_pages / 2 } + else { stride_pages - 1 }; + if needed > pages { return None; } + Some(Self { tail_pages_per_usc: pages as u32 }) + } + + pub(crate) fn reservation_bytes(&self) -> u64 { + USCS * G18_COMPUTE_DIRECTORY_STRIDE + G18_COMPUTE_DIRECTORY_PAGE + } + + pub(crate) fn committed_pages(&self) -> u32 { + USCS as u32 + 1 + USCS as u32 * self.tail_pages_per_usc + } + + /// Enumerate anchors first, followed by each USC's contiguous tail. + pub(crate) fn page_offset(&self, index: u32) -> Option { + if index >= self.committed_pages() { return None; } + if index <= USCS as u32 { + return Some(u64::from(index) * G18_COMPUTE_DIRECTORY_STRIDE); + } + let tail = u64::from(self.tail_pages_per_usc); + let index = u64::from(index) - USCS - 1; + let usc = index / tail; + let page = index % tail; + let offset = (usc + 1) * G18_COMPUTE_DIRECTORY_STRIDE + - (tail - page) * G18_COMPUTE_DIRECTORY_PAGE; + if offset + G18_COMPUTE_DIRECTORY_PAGE > self.reservation_bytes() { return None; } + Some(offset) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18ComputeAllocation { + pub(crate) bytes: u64, + pub(crate) block_count: u32, + pub(crate) register_pair: [u32; 2], + pub(crate) chunk_counters: [u32; 2], + pub(crate) chunks_per_usc: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct G18ComputePlan { + pub(crate) ideal: G18ComputeAllocation, + /// Native minimum is retained for comparison, not an allocation fallback. + pub(crate) minimum: G18ComputeAllocation, + /// Native AGXUMADataRec+0x100, packed above the slot in register0x10241. + pub(crate) sma_units: u32, +} + +impl G18ComputePlan { + /// Serialize the ideal plan only after its backing and aliases are owned. + /// Ownership, address-space validity and retirement are enforced by the + /// pool owner, not this arithmetic helper. The initial hardware slot is + /// unbound; load_hardware_descriptor publishes it in a separate step. + pub(crate) fn pool_state(&self, statistics_id: u64, directory_base: u64, + block_list_gpu: u64, counter_gpu: u64, counter: u64) + -> Option<[u8; crate::g18::fw::uma::G18_POOL_STATE_BYTES]> + { + let directory = G18ComputeDirectory::for_blocks(self.ideal.block_count)?; + if directory_base == 0 || directory_base & 0x3fff != 0 + || block_list_gpu == 0 || block_list_gpu & 0x3fff != 0 + || counter_gpu == 0 || counter_gpu & 7 != 0 + || directory_base.checked_add(directory.reservation_bytes())? > (1 << 48) + || block_list_gpu.checked_add(0x4000)? > (1 << 48) + { + return None; + } + Some(crate::g18::fw::uma::encode_g18_pool_state(&crate::g18::fw::uma::G18UmaPoolState { + statistics_id, grow_engine: 2, priority: 2, + directory_gpu: directory_base + G18_COMPUTE_DIRECTORY_PAGE - 0x400, + directory_stride: G18_COMPUTE_DIRECTORY_STRIDE, + chunk_counters: self.ideal.chunk_counters, + block_list_gpu, populated_blocks: self.ideal.block_count, + block_units_64k: (G18_COMPUTE_BLOCK_BYTES >> 16) as u32, + sma_mode: 0, preempt_counter_gpu: counter_gpu, preempt_counter: counter, + compute: true, policy_value: 2, + })) + } + + /// Bounded compute-only descriptor from G18ComputeInputs::descriptor(). + /// The caller must finish its accumulator and preserve effective linked + /// metadata. Render, protected, alternate-policy and concurrent-job + /// contributions need their own contract and are rejected here. + pub(crate) fn from_descriptor(words: &[u32; 23]) -> Option { + for index in [2, 3, 6, 7, 13, 14, 15, 17, 18, 21, 22] { + if words[index] != 0 { return None; } + } + if words[16] != 1 || words[19] != 128 || words[20] != 4 + || words[0] < 32 || words[1] < 10 + || words[0] > words[4] || words[1] > words[5] + { + return None; + } + let d = words.map(u64::from); + // Original SIMD sequence at 8c916c0..8c91724. Each header + // component rounds separately; summing before rounding is wrong. + let h = &d[8..16]; + let header = h.iter().sum::() + d[16] + + ((h[1] + 131) >> 7) + + ((h[2] + h[3] + 127) >> 7) + + ((h[4] + h[5] + 127) >> 7) + + ((h[6] + h[2] + h[7] + d[16] + 127) >> 7); + // Shared compute reserves its minimum in a separate contribution. + let reserved = d[0] + d[1] + d[19] + d[20]; + let size = |ideal: bool| -> Option { + let registers = if ideal { d[5] } else { d[1] }; + let scratch = if ideal { d[4] } else { d[0] }; + // One active job, up to two native shared-compute slots; native + // helper 8c91ae4 includes the two-slot minimum only for ideal. + let demand = scratch.saturating_sub(d[0] * 2) + + (registers - d[1]) + if ideal { d[1] * 2 } else { 0 }; + let chunks = (demand + reserved + header + 3 + 63) & !63; + if chunks > CHUNKS_PER_USC { return None; } + let bytes = (chunks * USCS * 1024 + G18_COMPUTE_BLOCK_BYTES - 1) + & !(G18_COMPUTE_BLOCK_BYTES - 1); + if bytes == 0 || bytes > G18_COMPUTE_MAX_BYTES { return None; } + // Both launch fields occupy sixteen bits in register 0xa5e1. + if registers > 0xffff || header > 0xffff { return None; } + Some(G18ComputeAllocation { + bytes, + block_count: u32::try_from(bytes / G18_COMPUTE_BLOCK_BYTES).ok()?, + register_pair: [registers as u32, header as u32], + chunk_counters: [0, u32::try_from(demand).ok()?], + chunks_per_usc: u32::try_from(chunks).ok()?, + }) + }; + Some(Self { ideal: size(true)?, minimum: size(false)?, + sma_units: u32::try_from(reserved).ok()? }) + } +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/graphics_scratch.rs b/drivers/gpu/drm/asahi/g18/uapi/graphics_scratch.rs new file mode 100644 index 0000000000000..0991e6bcdb5f2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/graphics_scratch.rs @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G18 render (TA/3D) thread-private scratch plan from per-render metadata. +//! +//! Pure arithmetic: no allocation, MMIO, firmware or VM access. Evidence and +//! differential checks: the graphics-scratch design notes. +//! +//! - Per-program accumulator: the native-verified compute arithmetic +//! ([`G18ComputeInputs::merge`]) for one 32-thread warp reproduces all eight +//! native 26A428 graphics shader-only descriptors (closed-form-check-v3). +//! - Fragment pass term: the native RenderContext framebuffer-resource +//! constructor (AGXMetalG18G 1ad040548..1ad040614) and fragment pass merge +//! (1ad027c8c..1ad027e68) for one sample, no user storage and no MSAA +//! extra: 20/20 native comparisons (pass-closed-form-check). +//! - Render sizing: the TA kick, then the 3D kick on the same pool/FList. +//! Native allocatePagesEnd stores the selected per-index parameters in +//! FList+0xbc..0x19c and the next prepareLockedBegin max-merges them +//! (8c91168..8c913f0), so the 3D requirement includes the TA requirement. +//! Component maxima of descriptor words 0..18 then the ideal/minimum pair, +//! chunk counter and bytes below match the original instructions for every +//! in-capacity ideal case and all minimum cases of a 1,220-case grid at +//! 1 and 2 MiB blocks, and all four captured TA/3D register pairs +//! (sequential-kicks, closed-form-check-v3). +//! +//! Not covered (callers must refuse): multisample or layered renders, user +//! (FramebufferDriverConfig+0x904) storage, tile storage above one 8 KiB +//! page, scratch in background/end-of-tile programs, and renders without any +//! scratch (those keep the existing fixed policy unchanged). The native +//! tile-program KiB terms are not fully reproduced (pass-terms-v2); they +//! affect only words 0/4, i.e. chunk counters and bytes, never the pair. + +use crate::g18::uapi::compute_inputs::{G18ComputeDemand, G18ComputeInputs, G18_TGMEM_POLICY}; + +/// `drm_asahi_g18_render_resources.version`. +pub(crate) const G18_RENDER_RESOURCES_VERSION: u32 = 1; +/// Little-endian u32 words in `drm_asahi_g18_render_resources`. +pub(crate) const G18_RENDER_RESOURCE_WORDS: usize = 20; +const FLAG_VERTEX: u32 = 1 << 0; +const FLAG_FRAGMENT: u32 = 1 << 1; +const TILE_PROGRAMS: usize = 4; +const USCS: u64 = 12; +const POOL_MAX_BYTES: u64 = 0x8000_0000; +/// Native per-USC chunk capacity of a 2 GiB, twelve-USC pool +/// (g18_compute_plan.rs uses the same measured geometry). +const CHUNKS_PER_USC: u64 = (POOL_MAX_BYTES / 0x1_0000 / USCS) * 64; +/// Largest tile storage whose pass term is covered by native evidence. +const MAX_TILE_BYTES: u32 = 8192; + +/// One program's demand in Mesa units (16-bit registers, IPR halfwords). +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct G18GraphicsProgram { + pub(crate) registers_16: u32, + pub(crate) scratch_bytes: u32, + pub(crate) staging_16: u32, +} + +impl G18GraphicsProgram { + fn valid(&self) -> bool { + (1..=256).contains(&self.registers_16) + && self.scratch_bytes & 15 == 0 + && self.scratch_bytes <= 0xffff * 4 + && self.staging_16 <= 0xffff + } + + /// Accumulator components of this program alone (one 32-thread warp). + fn components(&self) -> Option<[u32; 58]> { + let mut inputs = G18ComputeInputs::new(); + inputs.merge(G18ComputeDemand { + scratch_bytes: self.scratch_bytes, + constant_scratch_bytes: 0, + registers_32: (self.registers_16 + 1) / 2, + ipr_bytes: self.staging_16 * 2, + constant_ipr_bytes: 0, + threadgroup_bytes: 0, + imageblock_bytes: 0, + threads: 32, + }, G18_TGMEM_POLICY)?; + Some(*inputs.component_words()) + } +} + +/// Decoded `drm_asahi_g18_render_resources` plus the command's tile storage. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderMetadata { + pub(crate) vertex: G18GraphicsProgram, + pub(crate) fragment: G18GraphicsProgram, + /// Background, end-of-tile, partial background, partial end-of-tile. + pub(crate) tiles: [Option; TILE_PROGRAMS], + pub(crate) tile_bytes: u32, +} + +impl G18RenderMetadata { + /// `tile_bytes` comes from the validated render command (utile area x + /// bytes per sample x samples), never from userspace metadata. + pub(crate) fn decode(words: &[u32; G18_RENDER_RESOURCE_WORDS], tile_bytes: u32) -> Option { + if words[0] != G18_RENDER_RESOURCES_VERSION + || words[1] != (FLAG_VERTEX | FLAG_FRAGMENT) + || words[16..].iter().any(|&w| w != 0) + || tile_bytes == 0 || tile_bytes > MAX_TILE_BYTES + { + return None; + } + let stage = |at: usize| G18GraphicsProgram { + registers_16: words[at], scratch_bytes: words[at + 1], staging_16: words[at + 2], + }; + let (vertex, fragment) = (stage(2), stage(5)); + if !vertex.valid() || !fragment.valid() + || (vertex.scratch_bytes == 0 && fragment.scratch_bytes == 0) + { + return None; + } + let mut tiles = [None; TILE_PROGRAMS]; + for (index, slot) in tiles.iter_mut().enumerate() { + let program = G18GraphicsProgram { + registers_16: words[8 + index], scratch_bytes: 0, staging_16: words[12 + index], + }; + if program.registers_16 == 0 { + if program.staging_16 != 0 { return None; } + } else if program.valid() { + *slot = Some(program); + } else { + return None; + } + } + Some(Self { vertex, fragment, tiles, tile_bytes }) + } + + /// TA (vertex stage) descriptor: the vertex program alone. + pub(crate) fn ta_descriptor(&self) -> Option<[u32; 23]> { + Some(serialize(&self.vertex.components()?)) + } + + /// 3D (fragment stage) descriptor: component maxima of the fragment and + /// tile programs, then the native pass merge of (fragment + pass terms). + pub(crate) fn fragment_descriptor(&self) -> Option<[u32; 23]> { + let main = self.fragment.components()?; + let mut acc = main; + for program in self.tiles.iter().flatten() { + let words = program.components()?; + for (a, b) in acc.iter_mut().zip(words.iter()) { *a = (*a).max(*b); } + } + let kib = (self.tile_bytes + 1023) / 1024; + let pages = (self.tile_bytes + 8191) / 8192; + let mut passed = main; + passed[0x44 / 4] = passed[0x44 / 4].checked_add(kib)?; + passed[0x84 / 4] = passed[0x84 / 4].checked_add(kib.checked_mul(11)?)?; + for (offset, value) in [(0x28, pages), (0x68, pages * 6), (0x74, pages)] { + passed[offset / 4] = passed[offset / 4].max(value); + } + for (a, b) in acc.iter_mut().zip(passed.iter()) { *a = (*a).max(*b); } + Some(serialize(&acc)) + } + + /// Per-render plan for a pool of `block_bytes` blocks. + pub(crate) fn plan(&self, block_bytes: u64) -> Option { + if !matches!(block_bytes, 0x10_0000 | 0x20_0000) { return None; } + let ta = self.ta_descriptor()?; + let fragment = self.fragment_descriptor()?; + // The 3D kick follows the TA kick on the same FList: its requirement + // is the component maximum of both descriptors (words 0..18). + let mut d3 = fragment; + for (a, b) in d3[..19].iter_mut().zip(ta[..19].iter()) { *a = (*a).max(*b); } + Some(G18RenderScratchPlan { + ideal: [size(&ta, false, block_bytes)?, size(&d3, false, block_bytes)?], + minimum: [size(&ta, true, block_bytes)?, size(&d3, true, block_bytes)?], + }) + } +} + +/// Native AGXSMADescRec serialization of the 58-word accumulator +/// (identical to `G18ComputeInputs::descriptor`). +fn serialize(c: &[u32; 58]) -> [u32; 23] { + let w = |off: usize| c[off / 4]; + [w(0x44), w(0x0c) + w(0x10) + w(0x14) + w(0x24) + w(0x28), 0, 0, + w(0x84), w(0x4c) + w(0x50) + w(0x54) + w(0x64) + w(0x68), 0, 0, + w(0x58), w(0x5c), w(0x60), w(0x70), w(0x74), 0, 0, 0, + w(0xc8), 0, 0, w(0xcc), w(0xc4), 0, 0] +} + +/// One kick's selected native result. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderStagePlan { + /// Launch pair, packed `[0] | [1] << 16` in 0xa5e9 (TA) / 0xa5f1 (3D). + pub(crate) register_pair: [u32; 2], + /// Native FList chunk counter (+0xb0) for this kick. + pub(crate) chunk_counter: u32, + /// Native per-USC chunk reservation (64-aligned). + pub(crate) chunks_per_usc: u32, + pub(crate) bytes: u64, +} + +impl G18RenderStagePlan { + pub(crate) fn resource_word(&self) -> u64 { + u64::from(self.register_pair[0]) | u64::from(self.register_pair[1]) << 16 + } +} + +fn size(d: &[u32; 23], minimum: bool, block_bytes: u64) -> Option { + let d = d.map(u64::from); + // Original SIMD sequence at 8c916c0..8c91724 (see g18_compute_plan.rs): + // each header component rounds separately. + let h = &d[8..16]; + let header = h.iter().sum::() + d[16] + + ((h[1] + 131) >> 7) + + ((h[2] + h[3] + 127) >> 7) + + ((h[4] + h[5] + 127) >> 7) + + ((h[6] + h[2] + h[7] + d[16] + 127) >> 7); + let (first, counter) = if minimum { (d[1], d[0] + d[19]) } else { (d[5], d[4] + d[1] + d[19]) }; + let chunks = (counter + header + 3 + 63) & !63; + if chunks > CHUNKS_PER_USC || first > 0xffff || header > 0xffff { return None; } + let bytes = (chunks * USCS * 1024 + block_bytes - 1) & !(block_bytes - 1); + if bytes == 0 || bytes > POOL_MAX_BYTES { return None; } + Some(G18RenderStagePlan { + register_pair: [u32::try_from(first).ok()?, u32::try_from(header).ok()?], + chunk_counter: u32::try_from(counter).ok()?, + chunks_per_usc: u32::try_from(chunks).ok()?, + bytes, + }) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum G18RenderScratchMode { + Ideal, + /// Native's below-ideal selection (allocatePagesEnd minimum flag). + Minimum, +} + +/// Both native results for `[TA, 3D]`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderScratchPlan { + pub(crate) ideal: [G18RenderStagePlan; 2], + pub(crate) minimum: [G18RenderStagePlan; 2], +} + +/// The selection the job applies to its TA/3D resource words. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RenderScratchSelection { + pub(crate) mode: G18RenderScratchMode, + pub(crate) stages: [G18RenderStagePlan; 2], +} + +impl G18RenderScratchPlan { + /// Ideal when both kicks fit the owned pool, else the native minimum + /// when both fit, else none. `owned_chunks_per_usc` is the chunk counter + /// already published in the owned pool state; neither the pool nor its + /// state is resized by this selection. + pub(crate) fn select(&self, owned_bytes: u64, owned_chunks_per_usc: u32) + -> Option + { + let fits = |stages: &[G18RenderStagePlan; 2]| stages.iter() + .all(|s| s.bytes <= owned_bytes && s.chunks_per_usc <= owned_chunks_per_usc); + if fits(&self.ideal) { + Some(G18RenderScratchSelection { mode: G18RenderScratchMode::Ideal, stages: self.ideal }) + } else if fits(&self.minimum) { + Some(G18RenderScratchSelection { mode: G18RenderScratchMode::Minimum, stages: self.minimum }) + } else { + None + } + } +} diff --git a/drivers/gpu/drm/asahi/g18/uapi/mod.rs b/drivers/gpu/drm/asahi/g18/uapi/mod.rs new file mode 100644 index 0000000000000..4a1095e31ca92 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/mod.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! User command parsing, admission and per-command planning for G18 (T8152). + +#[allow(dead_code)] +pub(crate) mod application; +#[allow(dead_code)] +pub(crate) mod application_runtime; +#[allow(dead_code)] +pub(crate) mod command; +#[allow(dead_code)] +pub(crate) mod compute_inputs; +#[allow(dead_code)] +pub(crate) mod compute_metadata; +#[allow(dead_code)] +pub(crate) mod compute_plan; +#[allow(dead_code)] +pub(crate) mod graphics_scratch; +#[allow(dead_code)] +pub(crate) mod render_policy; diff --git a/drivers/gpu/drm/asahi/g18/uapi/render_policy.rs b/drivers/gpu/drm/asahi/g18/uapi/render_policy.rs new file mode 100644 index 0000000000000..c6bb9b4f6bf42 --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/uapi/render_policy.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded H1/application policy shared by the kernel and host. +//! Target kinds 1/2 use native geometry; mode 0 retains the stage33d policy. +use crate::{g18::fw::render_params::G17pRenderParameters, g18::address_map as va, + g18::fw::render::G18RenderControls, g18::fw::render_command::*, g18::fw::render_assembly as g18_render_assembly}; + +pub(crate) const PB_BLOCKS: u32 = 8; // plan: overprovision >=8; no PB growth. +pub(crate) const PB_CAPACITY: u32 = 1650; +pub(crate) const UMA_BLOCKS: u32 = 24; // 24 MiB > sum of both M5 budgets. +pub(crate) const USC_REQUIRED: [u64; 2] = [0x7d4000, 0x934000]; // M5 DRM hardware path. +// G18 native SMAV1 first-use unit pair, proven by the compute fixture. The +// graphics byte budgets above remain M5 policy; no claim of derived sizing. +pub(crate) const RESOURCE_PAIR: [u64; 2] = [16, 6]; +pub(crate) const UMA_CHUNKS: [u32; 2] = [0, + ((USC_REQUIRED[0] + USC_REQUIRED[1] + 12 * 1024 - 1) / (12 * 1024)) as u32]; + +// Native SMAV1 minimum-result serialization for the captured witness. Each +// stage needs 3 MiB, below the unchanged H1 backing/accounting. This probes the +// two resource words only; it does not claim native ideal allocation policy. +pub(crate) const NATIVE_MINIMUM_PAIRS: [[u64; 2]; 2] = [[4, 8], [9, 11]]; +#[cfg(not(CONFIG_RUST))] +pub(crate) static HOST_SMA_MINIMUM: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false); + +// Fixed first-draw witness policy, from the original M6 SMAV1 methods. This +// experiment is admitted only for stage9/variant0/profile4 before MMIO. +pub(crate) const NATIVE_IDEAL_PAIRS: [[u64; 2]; 2] = [[24, 8], [42, 11]]; +pub(crate) const IDEAL_GROWTH_BYTES: u64 = 21 * va::PAGE; +pub(crate) const IDEAL_GROWTH_OFFSET: u64 = 0x158000 - IDEAL_GROWTH_BYTES; +#[cfg(not(CONFIG_RUST))] +pub(crate) static HOST_SMA_IDEAL: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false); + +pub(crate) fn ideal_enabled() -> bool { + #[cfg(CONFIG_RUST)] { *crate::module_parameters::g18_sma_ideal.value() == 1 } + #[cfg(not(CONFIG_RUST))] { HOST_SMA_IDEAL.load(core::sync::atomic::Ordering::Relaxed) } +} +pub(crate) fn ta_barrier_flag() -> u32 { + #[cfg(CONFIG_RUST)] { *crate::module_parameters::g18_native_barrier.value() } + #[cfg(not(CONFIG_RUST))] { 0 } +} +pub(crate) fn uma_blocks() -> u32 { if ideal_enabled() { 40 } else { UMA_BLOCKS } } +/// Block size of the owned render pool (prepare_render_backing). +pub(crate) const RENDER_POOL_BLOCK_BYTES: u64 = 0x10_0000; +/// Capacity of the owned render pool as published in its state: backing +/// bytes and the per-USC chunk counter. Graphics scratch selections must fit +/// both; they never resize the pool (g18_graphics_scratch.rs). +pub(crate) fn owned_scratch_capacity() -> (u64, u32) { + (u64::from(uma_blocks()) * RENDER_POOL_BLOCK_BYTES, uma_chunks()[1]) +} +pub(crate) fn uma_chunks() -> [u32; 2] { if ideal_enabled() { [0, 708 + 2305] } else { UMA_CHUNKS } } +pub(crate) fn block_base() -> u64 { if ideal_enabled() { va::GRAPHICS_IDEAL_BLOCKS } else { va::POOL_BLOCKS } } +pub(crate) fn usc_required() -> [u64; 2] { if ideal_enabled() { [9 * 0x100000, 28 * 0x100000] } else { USC_REQUIRED } } + +pub(crate) fn resource_pairs() -> [[u64; 2]; 2] { + #[cfg(CONFIG_RUST)] + let minimum = *crate::module_parameters::g18_sma_minimum.value() == 1; + #[cfg(not(CONFIG_RUST))] + let minimum = HOST_SMA_MINIMUM.load(core::sync::atomic::Ordering::Relaxed); + if ideal_enabled() { NATIVE_IDEAL_PAIRS } else if minimum { NATIVE_MINIMUM_PAIRS } else { [RESOURCE_PAIR; 2] } +} + +// Keep registration in this owned file. This is the same KernelParam ABI as +// kernel::module!, with load-time-only access (perm=0), so a live job cannot +// change layout under its mappings. Default 1 matches the parsed colour/gears +// force-kind-1 flag. Invalid selectors fail target_plan before publication. +#[cfg(CONFIG_RUST)] +mod parameter { + use kernel::{bindings, module_param::{KernelParam, ModuleParamAccess, PARAM_OPS_U32}}; + pub(super) static VALUE: ModuleParamAccess = ModuleParamAccess::new(1); + #[link_section = "__param"] + #[used(compiler)] + static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g18_render_target_kind" + } else { c"asahi.g18_render_target_kind" }), + // SAFETY: module lifetime and parameter serialization are managed by + // the kernel, identically to the kernel::module! registration. + #[cfg(MODULE)] + mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&PARAM_OPS_U32), perm: 0, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { arg: VALUE.as_void_ptr() }, + }); + #[link_section = ".modinfo"] + #[used(compiler)] + static TYPE: [u8; b"parmtype=g18_render_target_kind:uint\0".len()] = *b"parmtype=g18_render_target_kind:uint\0"; + #[link_section = ".modinfo"] + #[used(compiler)] + static DESCRIPTION: [u8; b"parm=g18_render_target_kind:0 legacy, 1 native single target (default), 2 native clustered target\0".len()] = + *b"parm=g18_render_target_kind:0 legacy, 1 native single target (default), 2 native clustered target\0"; +} + +#[cfg(not(CONFIG_RUST))] +pub(crate) static HOST_TARGET_KIND: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(1); + +// Fragment resource word 0xa5f1 for the admitted 4x render. Apple's run112 +// renders use 0xb0022 (34,11) at 1x and 0xd002e (46,13) at 4x: the size class +// grows by log2(samples). With the 1x minimum word (9,11) the 4x draw faulted +// into firmware recovery (run164). Default (9,13): the minimum count at the 4x +// size class. 0 keeps the 1x word. 0644 (C uint ops, like the g18_app_* +// knobs), read at every 4x render. +#[cfg(CONFIG_RUST)] +mod msaa4_fragment { + use kernel::{bindings, module_param::KernelParam}; + pub(super) static VALUE: core::sync::atomic::AtomicU32 = + core::sync::atomic::AtomicU32::new(0xd_0009); + #[link_section = "__param"] + #[used(compiler)] + static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + c"g18_msaa4_fragment_word" + } else { c"asahi.g18_msaa4_fragment_word" }), + #[cfg(MODULE)] + mod_: &raw mut bindings::__this_module, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + // SAFETY: exported vmlinux ops; they access `arg` as an unsigned int + // under the parameter lock, which AtomicU32 represents exactly. + ops: unsafe { core::ptr::from_ref(&bindings::param_ops_uint) }, + perm: 0o644, level: -1, flags: 0, + __bindgen_anon_1: bindings::kernel_param__bindgen_ty_1 { + arg: core::ptr::from_ref(&VALUE).cast_mut().cast(), + }, + }); + #[link_section = ".modinfo"] + #[used(compiler)] + static TYPE: [u8; b"parmtype=g18_msaa4_fragment_word:uint\0".len()] = + *b"parmtype=g18_msaa4_fragment_word:uint\0"; + #[link_section = ".modinfo"] + #[used(compiler)] + static DESCRIPTION: [u8; b"parm=g18_msaa4_fragment_word:0xa5f1 for the admitted 4x render, count|log2<<16 (default 0xd0009; native 0xd002e; 0 = 1x word)\0".len()] = + *b"parm=g18_msaa4_fragment_word:0xa5f1 for the admitted 4x render, count|log2<<16 (default 0xd0009; native 0xd002e; 0 = 1x word)\0"; +} + +pub(crate) fn msaa4_fragment_word() -> u64 { + #[cfg(CONFIG_RUST)] + { u64::from(msaa4_fragment::VALUE.load(core::sync::atomic::Ordering::Relaxed)) } + #[cfg(not(CONFIG_RUST))] + { 0xd_0009 } +} + +pub(crate) fn target_kind() -> u32 { + #[cfg(CONFIG_RUST)] { *parameter::VALUE.value() } + #[cfg(not(CONFIG_RUST))] { HOST_TARGET_KIND.load(core::sync::atomic::Ordering::Relaxed) } +} + +/// All non-capture fields are either assigned here or disabled explicitly by +/// the default zero optional-feature policy. No value is taken from post-run +/// firmware memory or a captured kernel pointer. +pub(crate) fn h1_inputs(command: &[u8], context: u32, + p: &mut G17pRenderParameters, c: &mut G18RenderControls, +) -> G18RenderCommandInputs { + let (i, _) = crate::g18::fw::render_passthrough::capture_fields(command); + owned_inputs(i, context, p, c) +} + +/// Shared first-use policy; application callers supply their own descriptor +/// fields and then replace frame identity and PB addresses via owner accessors. +pub(crate) fn owned_inputs(mut i: G18RenderCommandInputs, context: u32, + p: &mut G17pRenderParameters, c: &mut G18RenderControls) -> G18RenderCommandInputs { + // Linux first-pass lifecycle and work identity, shared with command copies. + p.lifecycle = 1u64 << 32; p.fragment_lifecycle = p.lifecycle; + c.ta.work_key = u64::from(context) << 8; c.fragment.work_key = c.ta.work_key; + let pairs = resource_pairs(); + c.ta.resource_word = pairs[0][0] | pairs[0][1] << 16; + c.fragment.resource_word = pairs[1][0] | pairs[1][1] << 16; + // Only the admitted 4x render carries a sample control (msaa163). + let msaa4_word = msaa4_fragment_word(); + if c.fragment_sample_control != 0 && msaa4_word != 0 { + c.fragment.resource_word = msaa4_word; + } + i.context_id = context; i.scheduler_record = va::GRAPHICS_SCHEDULER_START + 0x4000; + i.fragment.sequence = 0; i.ta.sequence = 1; i.fragment.pass = 1; + for stage in 0..2 { + let qid = va::QUEUE_IDS[stage]; + let completion = G18StageCompletion { + stamp_addresses: [va::GRAPHICS_SIGNALS + 0x4000 + u64::from(qid)*4, + va::GRAPHICS_SIGNALS + 0x8000 + u64::from(qid)*4], + stamp: 0x100, queue_index: u32::from(qid), ordinal: 0, + has_wait: stage == 1, completion: (1u64 << 40) | 1, + key: if stage == 0 {2} else {1}, + }; + if stage == 0 { i.ta.completion = completion; } else { i.fragment.completion = completion; } + } + i.ta.linked_3d = G18TaLinked3d { queue_index: 10, stamp: 0x100, flag: 0, + completion: i.fragment.completion.completion }; + i.ta.pb_slot = 0; i.ta.pm_1d4 = 0; // M5 first-pass PM policy. + i.ta.pb_state = va::GRAPHICS_PB_HIGH; i.fragment.pm_state = i.ta.pb_state; + i.ta.pm_scene = va::PB_GRAPH + 0x1080; i.fragment.pm_scene = i.ta.pm_scene; + i.ta.pm_b8 = va::PB_GRAPH + 0x3800; i.fragment.pm_b8 = i.ta.pm_b8; + i.ta.uma_state = va::GRAPHICS_UMA_STATE; i.fragment.uma_state = i.ta.uma_state; + // Native render-target-loader fields with the M5 geometry of a 256x256, + // 32x32, single-layer target, sized with the register worker's 2x policy. + let plan = g18_render_assembly::target_plan(p).expect("admitted H1 geometry"); + g18_render_assembly::apply_target_controls(p, c, &plan); + i.ta.d11b8 = p.tpc; + i.ta.d11d8 = plan.native.map_or(plan.layout.tpc_bytes, |n| n.slice_sizes[0]) as u64; + i.ta.d10f8 = p.deflake_1; + i.fragment.d08d8 = p.tilemap; + i.fragment.d08f0 = p.multisample_control; + i.fragment.d08f8 = u32::from(p.samples); + i.fragment.d0900 = 1; + i.fragment.d0908 = 0x40004; + i.fragment.d0910 = [0, 0]; + i.fragment.d0920 = 0; + i.fragment.d0930 = 64; + if plan.native.is_some() { + let tx = (p.width + 31) / 32; + let ty = (p.height + 31) / 32; + i.fragment.d08f8 = 0; // target +0x20: native single-sample value + // msaa163: same values as g18_application_runtime (4 / 0x40008 for + // the admitted 4x render; unchanged otherwise). + i.fragment.d0900 = crate::g18::fw::render::g18_sample_count(p); + i.fragment.d0908 = crate::g18::fw::render::g18_macro_size(p); + i.fragment.d0910 = [0, u64::from(p.merge_upper_x_bits) | (u64::from(p.merge_upper_y_bits) << 32)]; + i.fragment.d0930 = tx * ty; + } + i.ta.d08a4 = 1; i.ta.d08a8 = 0; i.ta.d08ac = 0; + i.fragment.d08a4 = 1; i.fragment.d08a8 = 0; i.fragment.d08ac = 0; + // Passes per scene, as native (by the H1 parameter-buffer audit): + // dc98 = pass count, dc9c = 1 for a single pass. The encoder copies dc98 + // to 3D +0x21f9; firmware adds it to PB state +0x00 per 3D completion and + // adds 1 to +0x14 per TA completion (b000 0xcc1c, 0xd214). The PB is + // idle, and one a recovery flagged can be reset, only while the two + // balance (0xcca8, 0x11c94). Zero here left every flushed scene's PB + // blocking the render queues for the rest of the boot. + i.fragment.dc98 = 1; i.fragment.dc9c = 1; + // M5 TA activation bit; no optional policy/event/history payloads. + i.ta.d11a4 = 1; + i.ta.d08b0 = va::GRAPHICS_SIGNALS + 0xc000; + i.fragment.d08b0 = [va::GRAPHICS_SIGNALS + 0xc010, va::GRAPHICS_SIGNALS + 0xc018]; + // Native UMA data-record tail: required/allocated byte accounting. + let required = usc_required(); + i.ta.d1418 = [0, 0]; i.ta.d1428 = required[0]; i.ta.d1430 = 0; + i.fragment.dad0 = [0, 0]; i.fragment.dae0 = required[1]; i.fragment.dae8 = 0; + i +} diff --git a/drivers/gpu/drm/asahi/mmu/t8152.rs b/drivers/gpu/drm/asahi/mmu/t8152.rs new file mode 100644 index 0000000000000..55c481b33d6d9 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/t8152.rs @@ -0,0 +1,651 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! T8152 (G18G; Mac mini M6, J873) UAT support on the Direct backend. +//! +//! Module 1134 ran T8152 on its own MMU (`Uat::new_t8152`, handoff mode +//! `NoPeerT8152`, `m5_context.rs`). This file carries those pieces onto this +//! driver's UAT with [`UatFirmware::NoPeerRoots`]: +//! - the constructor and its firmware-reservation checks; +//! - the preparation, confirmation and mirroring of the firmware-published +//! high roots, with the shared-L2 topology proof, the DMB barrier and the +//! log lines of module 1134; +//! - the M5 execution-context leases (contexts 3..63, private empty upper +//! root, explicit publication), used instead of [`ExecutionContext`]; +//! - guard-page mappings and guard-tail aliases; +//! - the module-1134 names G18 calls (`retain_t8140_job`, `id`, `kernel_range`). +//! +//! Bodies are module 1134's (mmu.rs, m5_context.rs) unless a comment says otherwise. + +use super::*; +use kernel::drm::gem::BaseObject; +use kernel::page::Page; + +/// J873 (T8152) boot-local firmware reservations from the ADT /arm-io/sgx +/// node, as a loader copies them into /reserved-memory. +const T8152_DRAM_BASE: u64 = 0x100_0000_0000; +/// J873 DRAM end: /chosen dram-base 0x100_0000_0000 + dram-size 24 GiB. +const T8152_DRAM_END: u64 = 0x106_0000_0000; +/// (memory-region name, exact size): 26A428 GFX/GFX1 __DATA, then the +/// __DATA_SHARED_RO segments (firmware VA +0x10c000 / +0x108000). +const T8152_FIRMWARE_DATA: [(&CStr, u64); 2] = [ + (c_str!("gfx-data"), 0xbc000), + (c_str!("gfx1-data"), 0xb4000), +]; +const T8152_FIRMWARE_SHARED_RO: [(&CStr, u64); 2] = [ + (c_str!("gfx-data-shared-ro"), 0x8000), + (c_str!("gfx1-data-shared-ro"), 0x8000), +]; + +/// The GFX1 root follows the GFX root in the `pagetables` region (module 1134 +/// `t8140_secondary_ttb1 = pagetables + 0x40000`). +const T8152_SECONDARY_ROOT_OFFSET: u64 = 0x40000; +/// 42-bit input and output address space (module 1134 `UatConfig::T8152`). +const T8152_GEOMETRY: UatGeometry = UatGeometry { ias: 42, oas: 42 }; + +/// Entries 0 and 1 of each firmware root are private to that instance. +const T8140_FIRMWARE_PRIVATE_ROOT_ENTRIES: usize = 2; +/// Entries 0..2, including the shared-L2 kernel window, are published by firmware. +const T8140_FIRMWARE_SHARED_ROOT_ENTRIES: usize = 3; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct T8140PrimaryRootSnapshot { + entries: [u64; T8140_FIRMWARE_SHARED_ROOT_ENTRIES], +} + +fn clear_stale_host_root_entries(root: &mut [u64]) { + if root.len() > T8140_FIRMWARE_SHARED_ROOT_ENTRIES { + root[T8140_FIRMWARE_SHARED_ROOT_ENTRIES..].fill(0); + } +} + +fn mirror_host_root_entries(primary: &[u64], secondary: &mut [u64]) -> usize { + let mut mirrored = 0; + for (source, destination) in primary + .iter() + .zip(secondary.iter_mut()) + .skip(T8140_FIRMWARE_PRIVATE_ROOT_ENTRIES) + { + if *source != 0 && *destination != *source { + *destination = *source; + mirrored += 1; + } + } + mirrored +} + +impl Uat { + /// Create the T8152 UAT. The firmware never writes its half of the + /// handoff page, so the page is run in [`UatFirmware::NoPeerRoots`] and a + /// live peer is refused. + /// + /// `crash_views` gives the DATA-relative offsets of the GFX and GFX1 + /// RTKit crash views. The G18G offset is not yet known; G18 passes `None` + /// and that is the only supported value here. + pub(crate) fn new_t8152( + dev: &driver::AsahiDevice, + crash_views: Option<[u64; 2]>, + ) -> Result { + let node = dev.as_ref().of_node().ok_or(EINVAL)?; + let mut regions = [(0u64, 0u64); 4]; + for (index, (name, size)) in Iterator::chain( + T8152_FIRMWARE_DATA.into_iter(), + T8152_FIRMWARE_SHARED_RO, + ) + .enumerate() + { + let region = node.reserved_mem_region_to_resource_byname(name)?; + let start = region.start(); + let end = start.checked_add(region.size()).ok_or(EOVERFLOW)?; + if start < T8152_DRAM_BASE + || region.size() != size + || start & UAT_PGMSK as u64 != 0 + || end > T8152_DRAM_END + { + dev_err!( + dev.as_ref(), + "MMU: T8152 {} reservation {:#x}+{:#x} does not match the 26A428 layout ({:#x} bytes in J873 DRAM)\n", + name, + start, + region.size(), + size + ); + return Err(EINVAL); + } + if regions[..index].iter().any(|&(s, e)| start < e && s < end) { + dev_err!(dev.as_ref(), "MMU: T8152 {} overlaps another firmware reservation\n", name); + return Err(EINVAL); + } + regions[index] = (start, end); + } + // Module 1134 maps optional crash views here; G18 never requests them. + if crash_views.is_some() { + return Err(ENOTSUPP); + } + Self::new( + dev, + T8152_GEOMETRY, + false, + UatFirmware::NoPeerRoots { + secondary_root_offset: T8152_SECONDARY_ROOT_OFFSET, + }, + ) + } + + /// Whether this is the T8152 UAT; identity only. + pub(crate) fn is_t8152_geometry(&self) -> bool { + matches!(self.inner.firmware, UatFirmware::NoPeerRoots { .. }) + && self.geometry.ias == T8152_GEOMETRY.ias + && self.geometry.oas == T8152_GEOMETRY.oas + } + + /// Prepare the firmware-published upper roots for host mappings. + /// + /// Each firmware instance publishes three entries during RTKit boot: two + /// private code/data entries and the shared L2 entry used by the kernel VA + /// window. Host mappings must be added beneath that live entry, not beneath + /// a root image constructed before the processor starts. + pub(crate) fn prepare_t8140_firmware_high_roots( + &self, + ) -> Result { + if !self.is_t8152_geometry() { + return Err(EINVAL); + } + let primary_phys = self.kernel_vm.ttb()?; + let secondary_phys = self.inner.lock()?.secondary_ttb1.ok_or(EIO)?; + // SAFETY: both roots are pages of the bootloader-reserved `pagetables` + // region, which outlives this Uat. + let primary = unsafe { Page::borrow_phys(&primary_phys) }.ok_or(EIO)?; + // SAFETY: as above. + let secondary = unsafe { Page::borrow_phys(&secondary_phys) }.ok_or(EIO)?; + let mut primary_entries = [0u64; T8140_FIRMWARE_SHARED_ROOT_ENTRIES]; + let mut secondary_entries = [0u64; T8140_FIRMWARE_SHARED_ROOT_ENTRIES]; + + primary.with_pointer_into_page(0, UAT_PGSZ, |pointer| { + // SAFETY: the pointer covers the whole root page. + let entries = unsafe { + core::slice::from_raw_parts_mut( + pointer as *mut u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + primary_entries.copy_from_slice(&entries[..T8140_FIRMWARE_SHARED_ROOT_ENTRIES]); + Ok(()) + })?; + secondary.with_pointer_into_page(0, UAT_PGSZ, |pointer| { + // SAFETY: the pointer covers the whole root page. + let entries = unsafe { + core::slice::from_raw_parts( + pointer as *const u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + secondary_entries.copy_from_slice(&entries[..T8140_FIRMWARE_SHARED_ROOT_ENTRIES]); + Ok(()) + })?; + { + let node = self.dev.as_ref().of_node().ok_or(EINVAL)?; + let shared_l2 = node.reserved_mem_region_to_resource_byname(c_str!("shared-l2"))?; + let mask = ((1u64 << self.geometry.oas) - 1) & !(UAT_PGMSK as u64); + if shared_l2.size() != UAT_PGSZ as u64 + || Iterator::chain(primary_entries.iter(), secondary_entries.iter()).any(|pte| pte & 3 != 3) + || primary_entries[2] & mask != shared_l2.start() + || secondary_entries[2] & mask != shared_l2.start() + || primary_entries[0] & mask == secondary_entries[0] & mask + { + dev_err!( + self.dev.as_ref(), + "{} firmware-private roots/shared L2 do not match the native topology\n", + "T8152" + ); + return Err(EIO); + } + } + primary.with_pointer_into_page(0, UAT_PGSZ, |pointer| { + // SAFETY: the pointer covers the whole root page. + let entries = unsafe { core::slice::from_raw_parts_mut( + pointer as *mut u64, UAT_PGSZ / core::mem::size_of::()) }; + clear_stale_host_root_entries(entries); + Ok(()) + })?; + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + dev_info!( + self.dev.as_ref(), + "MMU: GFX high root {:#x} entries [{:#x}, {:#x}, {:#x}]; GFX1 root {:#x} entries [{:#x}, {:#x}, {:#x}]\n", + primary_phys, + primary_entries[0], + primary_entries[1], + primary_entries[2], + secondary_phys, + secondary_entries[0], + secondary_entries[1], + secondary_entries[2] + ); + Ok(T8140PrimaryRootSnapshot { + entries: primary_entries, + }) + } + + /// Confirm construction retained the three firmware-published entries. + pub(crate) fn confirm_t8140_primary_high_root( + &self, + snapshot: T8140PrimaryRootSnapshot, + ) -> Result { + if !self.is_t8152_geometry() { + return Err(EINVAL); + } + let primary_phys = self.kernel_vm.ttb()?; + // SAFETY: the root is a page of the bootloader-reserved `pagetables` region. + let primary = unsafe { Page::borrow_phys(&primary_phys) }.ok_or(EIO)?; + let mut current = [0u64; T8140_FIRMWARE_SHARED_ROOT_ENTRIES]; + primary.with_pointer_into_page(0, UAT_PGSZ, |pointer| { + // SAFETY: the pointer covers the whole root page. + let entries = unsafe { + core::slice::from_raw_parts( + pointer as *const u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + current.copy_from_slice(&entries[..T8140_FIRMWARE_SHARED_ROOT_ENTRIES]); + Ok(()) + })?; + if current != snapshot.entries { + dev_err!( + self.dev.as_ref(), + "MMU: GFX high-root firmware entries changed during graph construction: before {:?}, after {:?}\n", + snapshot.entries, + current + ); + return Err(EIO); + } + dev_info!( + self.dev.as_ref(), + "MMU: GFX high-root firmware entries preserved across graph construction\n" + ); + Ok(()) + } + + /// Mirror host-visible upper-root entries into GFX1's private top table. + /// Entries 0 and 1 remain owned by that firmware instance. + pub(crate) fn mirror_t8140_secondary_high_root(&self) -> Result { + if !self.is_t8152_geometry() { + return Err(EINVAL); + } + let primary_phys = self.kernel_vm.ttb()?; + let secondary_phys = self.inner.lock()?.secondary_ttb1.ok_or(EIO)?; + // SAFETY: both roots are pages of the bootloader-reserved `pagetables` region. + let primary = unsafe { Page::borrow_phys(&primary_phys) }.ok_or(EIO)?; + // SAFETY: as above. + let secondary = unsafe { Page::borrow_phys(&secondary_phys) }.ok_or(EIO)?; + let mirrored = primary.with_pointer_into_page(0, UAT_PGSZ, |primary_pointer| { + secondary.with_pointer_into_page(0, UAT_PGSZ, |secondary_pointer| { + // SAFETY: each pointer covers its whole root page; the pages differ. + let primary_entries = unsafe { + core::slice::from_raw_parts( + primary_pointer as *const u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + // SAFETY: as above. + let secondary_entries = unsafe { + core::slice::from_raw_parts_mut( + secondary_pointer as *mut u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + Ok(mirror_host_root_entries(primary_entries, secondary_entries)) + }) + })?; + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + dev_info!( + self.dev.as_ref(), + "MMU: mirrored {} host high-root entries into GFX1 at {:#x}\n", + mirrored, + secondary_phys + ); + Ok(mirrored) + } + + /// Unpublished, driver-private VM for the qualified G18 compute fixture. + /// Its VA-zero allocation is required by the saved native user payload. + /// This allocates page tables only; no context slot or TTB root is written. + pub(crate) fn new_t8152_compute_fixture_vm(&self) -> Result { + if !self.is_t8152_geometry() { return Err(ENODEV); } + // The user-VM allocator of a non-Handoff UAT starts at VA 0, as the + // module-1134 internal fixture VM's did. + self.new_vm(0x8152_0001, IOVA_USER_BASE..T8152_GEOMETRY.user_range().end)? + .with_t8152_lifetime() + } + + /// G18's 26A428 HAL400 exposes 64 context IDs (8c44c50..8c44c58). + /// The T8152 UAT already reserves the native firmware roots in 0..2; + /// use a separate, retained lower/empty-upper pair. This claims a slot + /// only. Root publication remains explicit and bounded by the owning G18 + /// firmware session's lifetime. + pub(crate) fn prepare_t8152_execution_context( + &self, vm: &Vm, + ) -> Result { + if !self.is_t8152_geometry() { + return Err(ENODEV); + } + { + let owner = vm.inner.exec_lock(None, false)?; + if owner.is_kernel || !core::ptr::eq( + Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner), + ) { + return Err(EINVAL); + } + } + // Module 1134 claims the lowest free context in 3..63 with a CAS on + // its own bitmap; here the same bitmap is the shared execution-context + // set, seeded with 0..2 for NoPeerRoots and taken under the UAT lock. + let context = { + let mut inner = self.inner.lock()?; + let free = !inner.execution_contexts & !0b111u64; + if free == 0 { + return Err(ENOSPC); + } + let index = free.trailing_zeros() as usize; + inner.execution_contexts |= 1u64 << index; + index + }; + // Construct the slot guard first so a failed private-root allocation + // returns its claim. This high VM is never exposed for mappings. + let slot = M5ContextSlot { inner: self.inner.clone(), context }; + let range = IOVA_USER_BASE..T8152_GEOMETRY.user_range().end; + let high = self.new_vm((0x8152u64 << 16) + 0x1000 + context as u64, range)? + .with_t8152_lifetime()?; + let low = vm.clone(); + let roots = context_roots(context, &low, &high)?; + Ok(M5ExecutionContext { slot, low, high, roots, published: false }) + } +} + +struct M5ContextSlot { + inner: Arc, + context: usize, +} + +impl Drop for M5ContextSlot { + fn drop(&mut self) { + // An unexpected published root is not ours to retract or recycle. + // Keep its bit quarantined rather than hand it to another client. + let Ok(mut inner) = self.inner.lock() else { return; }; + let slot = &inner.ttbs()[self.context]; + let empty = slot.ttb0.load(Ordering::Acquire) == 0 + && slot.ttb1.load(Ordering::Acquire) == 0; + if empty { + inner.execution_contexts &= !(1u64 << self.context); + } + } +} + +/// Retained M5 application root lease, shared by render and compute queues. +/// +/// Native M5/G18 compute has a private lower root and a distinct empty upper +/// root. Its firmware-global root is not copied into the application slot. +/// Firmware owns contexts 0..2. Application contexts are claimed before any +/// root becomes visible, and the lease retains both VM roots. The submission +/// owner must retain the lease until completion or confirmed processor stop. +pub(crate) struct M5ExecutionContext { + slot: M5ContextSlot, + low: Vm, + high: Vm, + /// The tagged roots of `low` and `high`, computed when `low` was set + /// (`Vm::ttb` is fallible on this driver; module 1134 recomputed them). + roots: [u64; 2], + published: bool, +} + +fn context_roots(context: usize, low: &Vm, high: &Vm) -> Result<[u64; 2]> { + let tag = (context as u64) << TTBR_ASID_SHIFT | TTBR_VALID; + Ok([low.ttb()? | tag, high.ttb()? | tag]) +} + +impl M5ExecutionContext { + pub(crate) fn vm(&self) -> &Vm { &self.low } + + pub(crate) fn context_id(&self) -> u8 { self.slot.context as u8 } + + pub(crate) fn matches(&self, vm: &Vm) -> bool { + core::ptr::eq(&*self.low.inner, &*vm.inner) + } + + pub(crate) fn roots(&self) -> [u64; 2] { + self.roots + } + + /// Publish only after all command-owned mappings have been prepared. + /// An occupied slot is never replaced, even by a second M5 probe owner. + pub(crate) fn publish(&mut self) -> Result { + if self.published { return Err(EBUSY); } + let roots = self.roots(); + { + let inner = self.slot.inner.lock()?; + let slot = &inner.ttbs()[self.slot.context]; + if slot.ttb0.load(Ordering::Acquire) != 0 + || slot.ttb1.load(Ordering::Acquire) != 0 { + return Err(EBUSY); + } + slot.ttb0.store(roots[0], Ordering::Release); + slot.ttb1.store(roots[1], Ordering::Release); + self.published = true; + } + fence(Ordering::SeqCst); + mem::tlbi_asid(self.context_id()); + mem::sync(); + Ok(()) + } + + pub(crate) fn is_current(&self) -> bool { + if !self.published { return false; } + let roots = self.roots(); + let Ok(inner) = self.slot.inner.lock() else { return false; }; + let slot = &inner.ttbs()[self.slot.context]; + slot.ttb0.load(Ordering::Acquire) == roots[0] + && slot.ttb1.load(Ordering::Acquire) == roots[1] + } + + /// Retarget this lease at another application VM without giving up the + /// hardware context slot. Any published roots are retracted and the ASID + /// invalidated first, exactly as on drop; the caller has already verified + /// that no work of the previous VM is outstanding. Publication of the new + /// lower root is explicit again, once the new owner's graph is prepared. + pub(crate) fn rebind(&mut self, vm: &Vm) -> Result { + { + let owner = vm.inner.exec_lock(None, false)?; + if owner.is_kernel || !core::ptr::eq( + Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.slot.inner), + ) { + return Err(EINVAL); + } + } + let roots = context_roots(self.slot.context, vm, &self.high)?; + self.retract(); + self.low = vm.clone(); + self.roots = roots; + Ok(()) + } + + fn retract(&mut self) { + if !self.published { return; } + let roots = self.roots(); + if let Ok(inner) = self.slot.inner.lock() { + let slot = &inner.ttbs()[self.slot.context]; + // Only our exact pair may be retracted. VM owners remain alive + // through root invalidation and the final ASID synchronization. + if slot.ttb0.load(Ordering::Acquire) == roots[0] + && slot.ttb1.load(Ordering::Acquire) == roots[1] { + slot.ttb0.store(0, Ordering::Release); + slot.ttb1.store(0, Ordering::Release); + } + } + fence(Ordering::SeqCst); + mem::tlbi_asid(self.context_id()); + mem::sync(); + self.published = false; + } +} + +impl Drop for M5ExecutionContext { + fn drop(&mut self) { + self.retract(); + } +} + +impl Vm { + /// Attach the sticky status and job lifetime that every module-1134 VM + /// had, and nothing else: no residency gate (no idle reclaim or shrinker), + /// no T8140 context-view bindings and no shared-footprint tracking. Use + /// this instead of [`Vm::with_status`] for every VM of the T8152 UAT. + pub(crate) fn with_t8152_lifetime(mut self) -> Result { + if self.status.is_none() { + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + } + self.lifetime = Some(lifetime::VmLifetime::new()?); + Ok(self) + } + + /// Application VA interval reserved by VM_CREATE for driver allocations. + pub(crate) fn kernel_range(&self) -> Range { self.kernel_range.clone() } + + /// The module-1134 name of [`Vm::retain_job`]. + pub(crate) fn retain_t8140_job(&self) -> Result { + self.retain_job() + } + + /// Reserve a checked number of trailing unmapped pages for M5 resources. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_in_range_with_guards( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_pages: usize, + ) -> Result { + let guard_size = guard_pages.checked_mul(UAT_PGSZ).ok_or(EOVERFLOW)?; + self.map_in_range_with_guard_size( + gem, object_range, alignment, range, prot, guard_size, + ) + } + + /// Map a GEM object at a specific address and reserve `guard_pages` + /// trailing unmapped pages in the same allocator node, so an alias of a + /// guarded allocation keeps its hardware guard gap in this Vm too. + pub(crate) fn map_at_with_guards( + &self, + addr: u64, + size: usize, + gem: ARef, + prot: Prot, + guard_pages: usize, + ) -> Result { + let reserved = guard_pages.checked_mul(UAT_PGSZ) + .and_then(|guard| size.checked_add(guard)).ok_or(EOVERFLOW)?; + self.map_at_reserving(addr, size, gem, prot, reserved) + } +} + +impl KernelMapping { + /// Borrow the mapped GEM subrange through a temporary CPU mapping. + /// + /// The caller must serialize CPU access with firmware or GPU writes. This + /// helper only owns the kernel VMap lifetime and applies the object offset + /// carried by this mapping. + pub(crate) fn with_cpu_bytes( + &self, + f: impl FnOnce(&mut [u8]) -> Result, + ) -> Result { + let gem: &gem::Object = self.0._gem.as_deref().ok_or(EINVAL)?; + let end = self + .0 + .offset + .checked_add(self.0.mapped_size) + .ok_or(EOVERFLOW)?; + if end > gem.size() { + return Err(ERANGE); + } + let vmap = gem.vmap::()?; + // SAFETY: the VMap covers the complete GEM allocation and the checked + // subrange is retained by this KernelMapping. + let bytes = unsafe { + core::slice::from_raw_parts_mut( + vmap.as_mut_ptr().add(self.0.offset), + self.0.mapped_size, + ) + }; + f(bytes) + } + + /// Map this shared GEM subrange into the end of another mapping's guards. + /// The caller must keep GPU work quiescent until the complete resource + /// update is published. Existing mappings are never removed or relocated. + pub(crate) fn map_alias_into_guard( + &self, anchor: &mut KernelMapping, prot: Prot, + ) -> Result { + let size = self.size(); + let gem: &gem::Object = self.0._gem.as_deref().ok_or(EINVAL)?; + if size == 0 || size & UAT_PGMSK != 0 || prot.is_cached_noncoherent() + || !gem.is_kernel() + || self.0.offset.checked_add(size).ok_or(EOVERFLOW)? > gem.size() + { + return Err(EINVAL); + } + let available = anchor.0.size().checked_sub(anchor.size() as u64).ok_or(EINVAL)?; + if size as u64 > available { return Err(ENOSPC); } + let sgt = Arc::new(gem.owned_sg_table()?, GFP_KERNEL)?; + let owner_ref = anchor.0.owner.clone(); + let mut owner = owner_ref.exec_lock(Some(gem), false)?; + let vm_bo = owner_ref.obtain_bo(gem)?; + { + let mut guard = vm_bo.inner().inner.lock(); + if guard.sgt.is_none() { guard.sgt.replace(sgt); } + } + let mut vm_bo = Some(MappingBo::new(vm_bo, owner_ref.clone())); + // As in map_at_with_guards, the fallible allocator operation must not + // own DMA-reservation references whose Drop would reacquire this lock. + let node = anchor.0.as_mut().split_tail(size as u64, KernelMappingInner { + owner: owner_ref.clone(), uat_inner: owner.uat_inner.clone(), prot, + bo: None, _gem: None, offset: self.0.offset, mapped_size: size, + }); + let mut node = match node { + Ok(node) => node, + Err(error) => { drop(owner); return Err(error); } + }; + { + let inner = node.as_mut().inner_mut(); + inner.bo = vm_bo.take(); + inner._gem = Some(gem::KernelMappingPin::new(gem)); + } + let mapped = owner.map_node(&node, prot); + if let Err(error) = mapped { + // Restore reservation coverage before the temporary mapping is + // dropped. Its normal Drop unmaps partial PTEs and invalidates + // every live context; the detached node cannot free the guard. + let rollback = anchor.0.as_mut().rejoin_tail(node.as_mut()); + drop(owner); + drop(KernelMapping::new(node)); + rollback?; + return Err(error); + } + drop(owner); + Ok(KernelMapping::new(node)) + } + + /// Undo a staged guard-tail alias before publication. Consume tails in + /// reverse growth order so each remains adjacent to its original anchor. + /// The caller owns the tail and must ensure no GPU work can reference it. + pub(crate) fn rollback_guard_tail(&mut self, mut tail: KernelMapping) -> Result { + if tail.0.prot.is_cached_noncoherent() { return Err(EINVAL); } + self.0.as_mut().rejoin_tail(tail.0.as_mut())?; + // KernelMapping::drop removes the PTEs and flushes live contexts. Its + // now-invalid MM node leaves the restored guard reservation intact. + drop(tail); + Ok(()) + } +} From 22bb2ace308cdd9129e1ecf8a2707ca84e0952fd Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 04:18:58 -0500 Subject: [PATCH 006/117] drm/asahi: g18: parse this driver's command layout The earlier out-of-tree M6 interface used a 272-byte render and an 88-byte compute command. Parse this driver's instead: struct drm_asahi_cmd_render (448 bytes) and struct drm_asahi_cmd_compute (200 bytes), zero-extending shorter payloads. - prior_vdm/prior_cdm move to 320/328 (render) and 136/144 (compute). They stay trivially satisfied: G18 retires each command before the next one starts. - scs_layout moves to 128. - Words G18 does not implement and does not report must be zero: timestamp copies, ray queries, the compute auxiliary allocation and intersection tables. - With DRM_ASAHI_RENDER_G18_RESOURCES (now bit 25) or DRM_ASAHI_COMPUTE_G18_RESOURCES (now bit 19) the payload is the complete command followed by the resources, and must be complete. USES_TEXTURE_STATE is now bit 18. The translation after parsing is unchanged. --- drivers/gpu/drm/asahi/g18/uapi/command.rs | 78 ++++++++++++++++------- 1 file changed, 55 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/asahi/g18/uapi/command.rs b/drivers/gpu/drm/asahi/g18/uapi/command.rs index 4c11e2de8ddb5..a0810a50644d1 100644 --- a/drivers/gpu/drm/asahi/g18/uapi/command.rs +++ b/drivers/gpu/drm/asahi/g18/uapi/command.rs @@ -31,15 +31,40 @@ use g17_submission::{ }; pub(crate) const UAPI_COMMAND_HEADER_SIZE: usize = 8; -pub(crate) const UAPI_RENDER_SIZE: usize = 272; -pub(crate) const UAPI_COMPUTE_SIZE: usize = 88; -pub(crate) const UAPI_G18_COMPUTE_SIZE: usize = 136; -pub(crate) const COMPUTE_G18_RESOURCES: u32 = 1 << 2; -/// `drm_asahi_cmd_render_g18`: the render payload plus its 80-byte suffix. +/// `sizeof(struct drm_asahi_cmd_render)`. Shorter payloads are zero-extended. +pub(crate) const UAPI_RENDER_SIZE: usize = 448; +/// `drm_asahi_cmd_render::ts_frag_end_copies`, which G18 does not implement: +/// the words must be zero. +const RENDER_TIMESTAMP_COPIES: core::ops::Range = 0x100..0x140; +/// `drm_asahi_cmd_render::prior_vdm` and `prior_cdm`. +const RENDER_PRIOR_VDM: usize = 0x140; +const RENDER_PRIOR_CDM: usize = 0x148; +/// `drm_asahi_cmd_render::ray_query`. G18 has no ray queries: the words +/// must be zero. +const RENDER_RAY_QUERY: core::ops::Range = 0x150..UAPI_RENDER_SIZE; +/// `sizeof(struct drm_asahi_cmd_render_g18)`: the complete render payload +/// followed by the 80-byte `drm_asahi_g18_render_resources`. pub(crate) const UAPI_G18_RENDER_SIZE: usize = UAPI_RENDER_SIZE + 80; +/// `sizeof(struct drm_asahi_cmd_compute)`. Shorter payloads are zero-extended. +pub(crate) const UAPI_COMPUTE_SIZE: usize = 200; +/// `drm_asahi_cmd_compute::ts_end_copies`, which G18 does not implement. +const COMPUTE_TIMESTAMP_COPIES: core::ops::Range = 0x40..0x80; +/// `drm_asahi_cmd_compute::scs_layout`. +const COMPUTE_SCS_LAYOUT: usize = 0x80; +/// `drm_asahi_cmd_compute::prior_vdm` and `prior_cdm`. +const COMPUTE_PRIOR_VDM: usize = 0x88; +const COMPUTE_PRIOR_CDM: usize = 0x90; +/// `cdm_auxiliary_allocation` through `intersection_default_control`, which +/// G18 does not implement: the words must be zero. +const COMPUTE_UNSUPPORTED: core::ops::Range = 0x98..UAPI_COMPUTE_SIZE; +/// `sizeof(struct drm_asahi_cmd_compute_g18)`: the complete compute payload +/// followed by the 64-byte `drm_asahi_g18_compute_resources`. +pub(crate) const UAPI_G18_COMPUTE_SIZE: usize = UAPI_COMPUTE_SIZE + 64; +/// `DRM_ASAHI_COMPUTE_G18_RESOURCES`. +pub(crate) const COMPUTE_G18_RESOURCES: u32 = 1 << 19; /// `DRM_ASAHI_RENDER_G18_RESOURCES`. Only the G18 adapter may consume and /// strip it; the shared translator's RENDER_FLAGS keeps refusing it. -pub(crate) const RENDER_G18_RESOURCES: u32 = 1 << 21; +pub(crate) const RENDER_G18_RESOURCES: u32 = 1 << 25; pub(crate) const UAPI_ATTACHMENT_SIZE: usize = 24; pub(crate) const UAPI_MAX_ATTACHMENTS: usize = 16; pub(crate) const UAPI_MAX_HARDWARE_COMMANDS: u32 = 64; @@ -371,18 +396,21 @@ pub(crate) struct UapiRenderCommand { impl UapiRenderCommand { fn parse(raw: &[u8]) -> Result { - // Legacy payloads are zero-extended exactly as before. A flagged G18 - // suffix must be complete; padding could hide a truncated field. + // Shorter payloads are zero-extended. A flagged G18 suffix must be + // complete; padding could hide a truncated field. Words G18 does not + // implement (timestamp copies, ray queries) must be zero, as must + // anything after the structure or, when flagged, its suffix. let has_resources = raw.len() >= 4 && get_u32(raw, 0) & RENDER_G18_RESOURCES != 0; if has_resources && raw.len() < UAPI_G18_RENDER_SIZE { return Err(UapiTranslateError::PayloadTruncated); } - if !has_resources && raw.len() > UAPI_RENDER_SIZE - && raw[UAPI_RENDER_SIZE..].iter().any(|byte| *byte != 0) + let raw = padded_payload::(raw)?; + let unused_tail = if has_resources { UAPI_G18_RENDER_SIZE } else { UAPI_RENDER_SIZE }; + if raw[RENDER_TIMESTAMP_COPIES].iter().chain(&raw[RENDER_RAY_QUERY]) + .chain(&raw[unused_tail..]).any(|byte| *byte != 0) { return Err(UapiTranslateError::PayloadTail); } - let raw = padded_payload::(raw)?; let mut g18_resources = [0; 20]; if has_resources { for (i, word) in g18_resources.iter_mut().enumerate() { @@ -422,8 +450,8 @@ impl UapiRenderCommand { stencil_clear: get_u32(&raw, 0xcc), vertex_timestamps: timestamps(&raw, 0xd0), fragment_timestamps: timestamps(&raw, 0xe0), - prior_vdm: get_u64(&raw, 0x100), - prior_cdm: get_u64(&raw, 0x108), + prior_vdm: get_u64(&raw, RENDER_PRIOR_VDM), + prior_cdm: get_u64(&raw, RENDER_PRIOR_CDM), g18_resources, }) } @@ -446,23 +474,27 @@ pub(crate) struct UapiComputeCommand { impl UapiComputeCommand { fn parse(raw: &[u8]) -> Result { - // Legacy compute payloads are zero-extended as before. A flagged - // metadata payload must be complete; padding it could hide truncation - // of a valid zero-valued field or reserved word. + // Shorter compute payloads are zero-extended. A flagged metadata + // payload must be complete; padding it could hide truncation of a + // valid zero-valued field or reserved word. Words G18 does not + // implement (timestamp copies, the auxiliary allocation, intersection + // tables) must be zero, as must anything after the structure or, + // when flagged, its suffix. let has_resources = raw.len() >= 4 && get_u32(raw, 0) & COMPUTE_G18_RESOURCES != 0; if has_resources && raw.len() < UAPI_G18_COMPUTE_SIZE { return Err(UapiTranslateError::PayloadTruncated); } - if !has_resources && raw.len() > UAPI_COMPUTE_SIZE - && raw[UAPI_COMPUTE_SIZE..].iter().any(|byte| *byte != 0) + let raw = padded_payload::(raw)?; + let unused_tail = if has_resources { UAPI_G18_COMPUTE_SIZE } else { UAPI_COMPUTE_SIZE }; + if raw[COMPUTE_TIMESTAMP_COPIES].iter().chain(&raw[COMPUTE_UNSUPPORTED]) + .chain(&raw[unused_tail..]).any(|byte| *byte != 0) { return Err(UapiTranslateError::PayloadTail); } - let raw = padded_payload::(raw)?; let mut g18_resources = [0; 16]; if has_resources { for (i, word) in g18_resources.iter_mut().enumerate() { - *word = get_u32(&raw, 72 + i * 4); + *word = get_u32(&raw, UAPI_COMPUTE_SIZE + i * 4); } } Ok(Self { @@ -473,10 +505,10 @@ impl UapiComputeCommand { sampler_heap: get_u64(&raw, 0x18), helper: helper(&raw, 0x20), timestamps: timestamps(&raw, 0x30), - scs_layout: get_u64(&raw, 0x40), + scs_layout: get_u64(&raw, COMPUTE_SCS_LAYOUT), g18_resources, - prior_vdm: if has_resources { 0 } else { get_u64(&raw, 0x48) }, - prior_cdm: if has_resources { 0 } else { get_u64(&raw, 0x50) }, + prior_vdm: get_u64(&raw, COMPUTE_PRIOR_VDM), + prior_cdm: get_u64(&raw, COMPUTE_PRIOR_CDM), }) } } From ae7fed1bf625797f30cefa2efffd8e770c0a67b5 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 04:18:58 -0500 Subject: [PATCH 007/117] drm/asahi: mmu: add the T8152 no-peer UAT mode T8152's two GPU coprocessors publish their own page-table roots and never touch the handoff page. Add UatFirmware::NoPeerRoots and the hooks it needs: - refuse a live firmware peer in the handoff page; - prepare, confirm and mirror the firmware-published high roots (mmu/t8152.rs); - preflight GPUVM maps against driver-owned aliases; - per-page range TLB invalidation; - downgrade cached mappings to uncached before unmapping them, with no cache flush request; - execution-context leases on contexts 3..63 instead of the slot allocator; - guard-page mappings and guard-tail aliases, with the GEM helpers they use. T8152 uses the default UatQuirks, the policy the earlier M6 port ran this UAT with. Everything new is under CONFIG_DRM_ASAHI_G18, so other GPUs' code is unchanged. Based on the earlier out-of-tree M6 port's T8152 UAT code, adapted to this tree's UAT code. --- drivers/gpu/drm/asahi/gem.rs | 21 +++++ drivers/gpu/drm/asahi/mmu.rs | 120 +++++++++++++++++++++++++-- drivers/gpu/drm/asahi/mmu/t8152.rs | 129 ++++++++++++++--------------- 3 files changed, 196 insertions(+), 74 deletions(-) diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index f100820a0c75e..551e801ba2a29 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -210,6 +210,21 @@ impl ObjectRef { vm.map_at(addr, self.gem.size(), &self.gem, prot, guard) } + + /// Map at a specific address, reserving `guard_pages` unmapped pages + /// after the object in that Vm's allocator (T8152, module 1134). + #[cfg(CONFIG_DRM_ASAHI_G18)] + pub(crate) fn map_at_with_guards( + &mut self, + vm: &crate::mmu::Vm, + addr: u64, + prot: mmu::Prot, + guard_pages: usize, + ) -> Result { + validate_vm_binding(&self.gem, vm)?; + + vm.map_at_with_guards(addr, self.gem.size(), &self.gem, prot, guard_pages) + } } pub(crate) struct AsahiObjConfig { @@ -395,6 +410,12 @@ impl Drop for KernelMappingPin { } impl AsahiObject { + /// Whether this is a kernel-created object. + #[cfg(CONFIG_DRM_ASAHI_G18)] + pub(crate) fn is_kernel(&self) -> bool { + self.kernel + } + /// Whether CPU mappings use the cacheable WRITEBACK policy. pub(crate) fn is_writeback(&self) -> bool { self.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK != 0 diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index f146b2762d812..494e0730c67d7 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -116,8 +116,15 @@ mod lifetime; mod ppl; mod residency; mod shared; +#[cfg(CONFIG_DRM_ASAHI_G18)] +mod t8152; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; +#[cfg(CONFIG_DRM_ASAHI_G18)] +pub(crate) use t8152::M5ExecutionContext; +/// The module-1134 name of the job guard G18 holds while a job uses a VM. +#[cfg(CONFIG_DRM_ASAHI_G18)] +pub(crate) type T8140VmJobGuard = VmJobGuard; pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region @@ -164,6 +171,28 @@ pub(crate) enum UatFirmware { /// Offset of the secondary instance's kernel root in the `pagetables` region. secondary_root_offset: Option, }, + /// Two coprocessor instances that never touch the handoff region and publish their roots like + /// [`UatFirmware::PublishedRoots`], without the context-1 kernel alias (T8152/G18G, 26A428). + /// + /// A live firmware peer in the handoff region is refused, and the high roots are prepared, + /// confirmed and mirrored by `mmu::t8152`. + #[cfg(CONFIG_DRM_ASAHI_G18)] + NoPeerRoots { + /// Offset of the secondary instance's kernel root in the `pagetables` region. + secondary_root_offset: u64, + }, +} + +impl UatFirmware { + /// Whether this is the T8152 no-peer firmware. + #[inline(always)] + const fn is_no_peer(&self) -> bool { + #[cfg(CONFIG_DRM_ASAHI_G18)] + if let UatFirmware::NoPeerRoots { .. } = self { + return true; + } + false + } } /// UAT policy of a SoC where it differs from the A18 Pro's. @@ -581,6 +610,14 @@ impl gpuvm::DriverGpuVm for VmInner { if ctx.dev.is_none() { return Err(ENODEV); } + // T8152 (module 1134): GPUVM cannot see driver-owned KernelMappings, and G18 places + // driver aliases in user VMs outside the VM_CREATE kernel range. Preflight the full + // object range in the authoritative page table (EBUSY on any populated leaf) and + // allocate every intermediate table before the first scatterlist leaf is written. + if self.uat_inner.firmware.is_no_peer() { + self.page_table + .prepare_map(map_addr..map_addr.checked_add(map_length as u64).ok_or(EOVERFLOW)?)?; + } // Bind preparation retained the table's backing before taking the VM reservation. // Never acquire the object reservation from inside the GPUVM map callback. let segments = op.vm_bo().data().segments.lock(); @@ -874,6 +911,9 @@ impl VmInner { UatFirmware::PublishedRoots { .. } => { mem::tlbi_range_or_asid(asid, iova as usize, size) } + // T8152: the per-page range encoding module 1134 issues. + #[cfg(CONFIG_DRM_ASAHI_G18)] + UatFirmware::NoPeerRoots { .. } => mem::tlbi_range(asid, iova as usize, size), } } @@ -906,9 +946,14 @@ impl VmInner { if contexts == 0 { return; } + let no_peer = self.uat_inner.firmware.is_no_peer(); for ctx in 0..UAT_NUM_CTX { if contexts & (1 << ctx) != 0 { - mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); + if no_peer { + mem::tlbi_range(ctx as u8, iova as usize, size); + } else { + mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); + } } } mem::sync(); @@ -1043,6 +1088,9 @@ pub(crate) struct Vm { gpuvm: ARef>, dummy_obj: ARef, binding: Arc>, + /// The kernel range this VM was created with (`t8152::Vm::kernel_range`). + #[cfg(CONFIG_DRM_ASAHI_G18)] + kernel_range: Range, } no_debug!(Vm); @@ -1298,13 +1346,13 @@ impl TimestampVmap { /// Prepared CPU timestamp access with its own vmap; destruction happens after completion /// signalling. -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] pub(crate) struct TimestampMapping { mapping: TimestampVmap, offset: usize, } -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] impl TimestampMapping { /// Prepare one bounded CPU timestamp destination on retained GEM backing. pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { @@ -1355,7 +1403,7 @@ impl KernelMapping { /// use a complete atomic store; byte stores support unaligned destinations without /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { let gem = self.0._gem.as_ref().ok_or(EINVAL)?; TimestampMapping::new(gem, self.timestamp_offset(offset)?) @@ -1399,6 +1447,7 @@ impl KernelMapping { let mut mutated = false; let result = (|| { if self.0.uat_inner.firmware == UatFirmware::Handoff + || self.0.uat_inner.firmware.is_no_peer() || prefix == 0 || prefix > self.size() || (prefix | self.size()) & UAT_PGMSK != 0 @@ -1449,6 +1498,27 @@ impl KernelMapping { (result, mutated) } + /// Downgrade this mapping's PTEs to uncached, without a firmware cache flush (T8152). + #[cfg(CONFIG_DRM_ASAHI_G18)] + fn remap_uncached_only(&mut self) { + let mut owner = self.0.owner.lock_inner(); + let prot = self.0.prot.as_uncached(); + let _mutation = owner.mapping_mutation(); + if owner + .page_table + .reprot_pages(self.iova_range(), prot) + .is_err() + { + dev_err!( + owner.dev.as_ref(), + "MMU: remap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + mem::sync(); + } + /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the /// coprocessor cache. This is required to safely unmap cached/private mappings. fn remap_uncached_and_flush(&mut self) { @@ -1631,6 +1701,12 @@ impl Drop for KernelMapping { ); self.remap_uncached_and_flush(); } + // T8152 (module 1134): downgrade the PTEs to uncached before the unmap. The firmware + // cache flush is never armed for this firmware, so no flush request follows. + #[cfg(CONFIG_DRM_ASAHI_G18)] + if self.0.prot.is_cached_noncoherent() && self.0.uat_inner.firmware.is_no_peer() { + self.remap_uncached_only(); + } // Only VM page tables/binding state are touched here. The shared // reservation is sufficient and avoids fallible exec allocation in Drop. @@ -1972,6 +2048,14 @@ impl Handoff { /// Initialize the handoff region fn init(&self, firmware: UatFirmware, slot: u32) -> Result { + #[cfg(CONFIG_DRM_ASAHI_G18)] + if firmware.is_no_peer() + && (self.magic_fw.load(Ordering::Acquire) != 0 + || self.lock_fw.load(Ordering::Acquire) != 0) + { + pr_err!("T8152 handoff: unexpected firmware peer; refusing no-peer mode\n"); + return Err(ENODEV); + } self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); self.cur_slot.store(slot, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); @@ -2154,6 +2238,8 @@ impl Vm { None }; let bind_sequence = Arc::new(AtomicU64::new(0), GFP_KERNEL)?; + #[cfg(CONFIG_DRM_ASAHI_G18)] + let vm_kernel_range = kernel_range.clone(); let unique_gpuvm = gpuvm::GpuVm::new::( c"Asahi::GpuVm", &dev, @@ -2191,6 +2277,8 @@ impl Vm { status: None, dummy_obj: dummy_obj.gem.clone(), binding, + #[cfg(CONFIG_DRM_ASAHI_G18)] + kernel_range: vm_kernel_range, }) } @@ -3341,6 +3429,11 @@ impl Uat { /// Binds a `Vm` to a slot, preferring the last used one. pub(crate) fn bind(&self, vm: &Vm) -> Result { + // T8152 uses the module-1134 M5 execution-context leases (`mmu::t8152`); the slot + // allocator would claim contexts the leases own. + if self.inner.firmware.is_no_peer() { + return Err(ENOTSUPP); + } self.bind_inner(vm, true) } @@ -3458,7 +3551,9 @@ impl Uat { /// Allocates a stable application context without changing the VM bind slot. /// Both roots and the generation remain unchanged for the lease's lifetime. pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { - if self.inner.firmware == UatFirmware::Handoff { + // NoPeerRoots: T8152 publishes `t8152::M5ExecutionContext` leases (private empty upper + // root) instead of kernel-root upper halves. + if self.inner.firmware == UatFirmware::Handoff || self.inner.firmware.is_no_peer() { return Err(EINVAL); } { @@ -3689,7 +3784,13 @@ impl Uat { shared <- new_mutex!( UatShared { kernel_ttb1: 0, - execution_contexts: !EXECUTION_CONTEXT_MASK, + // NoPeerRoots: contexts 3..63 are the M5 execution-context leases of + // `mmu::t8152`; 0..2 belong to the driver and the firmware. + execution_contexts: if firmware.is_no_peer() { + 0b111 + } else { + !EXECUTION_CONTEXT_MASK + }, context_generations: [0; UAT_NUM_CTX], binding_generations: [0; UAT_NUM_CTX], binding_published: [false; UAT_NUM_CTX], @@ -3747,6 +3848,13 @@ impl Uat { UatFirmware::PublishedRoots { secondary_root_offset: None, } => (UAT_PGSZ, None), + #[cfg(CONFIG_DRM_ASAHI_G18)] + UatFirmware::NoPeerRoots { + secondary_root_offset, + } => ( + secondary_root_offset as usize + UAT_PGSZ, + Some(ttb1 + secondary_root_offset), + ), }; if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); diff --git a/drivers/gpu/drm/asahi/mmu/t8152.rs b/drivers/gpu/drm/asahi/mmu/t8152.rs index 55c481b33d6d9..90c05fc56b09f 100644 --- a/drivers/gpu/drm/asahi/mmu/t8152.rs +++ b/drivers/gpu/drm/asahi/mmu/t8152.rs @@ -85,7 +85,7 @@ impl Uat { dev: &driver::AsahiDevice, crash_views: Option<[u64; 2]>, ) -> Result { - let node = dev.as_ref().of_node().ok_or(EINVAL)?; + let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; let mut regions = [(0u64, 0u64); 4]; for (index, (name, size)) in Iterator::chain( T8152_FIRMWARE_DATA.into_iter(), @@ -121,13 +121,16 @@ impl Uat { if crash_views.is_some() { return Err(ENOTSUPP); } + // The A18 Pro policy is the one module 1134 ran this UAT with: no + // table cleaning, range TLBIs, the handoff lock and slot 0. Self::new( - dev, + dev.clone(), T8152_GEOMETRY, false, UatFirmware::NoPeerRoots { secondary_root_offset: T8152_SECONDARY_ROOT_OFFSET, }, + UatQuirks::DEFAULT, ) } @@ -150,8 +153,8 @@ impl Uat { if !self.is_t8152_geometry() { return Err(EINVAL); } - let primary_phys = self.kernel_vm.ttb()?; - let secondary_phys = self.inner.lock()?.secondary_ttb1.ok_or(EIO)?; + let primary_phys = self.kernel_vm.ttb(); + let secondary_phys = self.inner.lock().secondary_ttb1.ok_or(EIO)?; // SAFETY: both roots are pages of the bootloader-reserved `pagetables` // region, which outlives this Uat. let primary = unsafe { Page::borrow_phys(&primary_phys) }.ok_or(EIO)?; @@ -183,7 +186,7 @@ impl Uat { Ok(()) })?; { - let node = self.dev.as_ref().of_node().ok_or(EINVAL)?; + let node = self.dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; let shared_l2 = node.reserved_mem_region_to_resource_byname(c_str!("shared-l2"))?; let mask = ((1u64 << self.geometry.oas) - 1) & !(UAT_PGMSK as u64); if shared_l2.size() != UAT_PGSZ as u64 @@ -235,7 +238,7 @@ impl Uat { if !self.is_t8152_geometry() { return Err(EINVAL); } - let primary_phys = self.kernel_vm.ttb()?; + let primary_phys = self.kernel_vm.ttb(); // SAFETY: the root is a page of the bootloader-reserved `pagetables` region. let primary = unsafe { Page::borrow_phys(&primary_phys) }.ok_or(EIO)?; let mut current = [0u64; T8140_FIRMWARE_SHARED_ROOT_ENTRIES]; @@ -272,8 +275,8 @@ impl Uat { if !self.is_t8152_geometry() { return Err(EINVAL); } - let primary_phys = self.kernel_vm.ttb()?; - let secondary_phys = self.inner.lock()?.secondary_ttb1.ok_or(EIO)?; + let primary_phys = self.kernel_vm.ttb(); + let secondary_phys = self.inner.lock().secondary_ttb1.ok_or(EIO)?; // SAFETY: both roots are pages of the bootloader-reserved `pagetables` region. let primary = unsafe { Page::borrow_phys(&primary_phys) }.ok_or(EIO)?; // SAFETY: as above. @@ -332,7 +335,7 @@ impl Uat { return Err(ENODEV); } { - let owner = vm.inner.exec_lock(None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !core::ptr::eq( Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner), ) { @@ -343,7 +346,7 @@ impl Uat { // its own bitmap; here the same bitmap is the shared execution-context // set, seeded with 0..2 for NoPeerRoots and taken under the UAT lock. let context = { - let mut inner = self.inner.lock()?; + let mut inner = self.inner.lock(); let free = !inner.execution_contexts & !0b111u64; if free == 0 { return Err(ENOSPC); @@ -373,7 +376,7 @@ impl Drop for M5ContextSlot { fn drop(&mut self) { // An unexpected published root is not ours to retract or recycle. // Keep its bit quarantined rather than hand it to another client. - let Ok(mut inner) = self.inner.lock() else { return; }; + let mut inner = self.inner.lock(); let slot = &inner.ttbs()[self.context]; let empty = slot.ttb0.load(Ordering::Acquire) == 0 && slot.ttb1.load(Ordering::Acquire) == 0; @@ -402,7 +405,7 @@ pub(crate) struct M5ExecutionContext { fn context_roots(context: usize, low: &Vm, high: &Vm) -> Result<[u64; 2]> { let tag = (context as u64) << TTBR_ASID_SHIFT | TTBR_VALID; - Ok([low.ttb()? | tag, high.ttb()? | tag]) + Ok([low.ttb() | tag, high.ttb() | tag]) } impl M5ExecutionContext { @@ -411,7 +414,7 @@ impl M5ExecutionContext { pub(crate) fn context_id(&self) -> u8 { self.slot.context as u8 } pub(crate) fn matches(&self, vm: &Vm) -> bool { - core::ptr::eq(&*self.low.inner, &*vm.inner) + core::ptr::eq(&*self.low.gpuvm, &*vm.gpuvm) } pub(crate) fn roots(&self) -> [u64; 2] { @@ -424,7 +427,7 @@ impl M5ExecutionContext { if self.published { return Err(EBUSY); } let roots = self.roots(); { - let inner = self.slot.inner.lock()?; + let inner = self.slot.inner.lock(); let slot = &inner.ttbs()[self.slot.context]; if slot.ttb0.load(Ordering::Acquire) != 0 || slot.ttb1.load(Ordering::Acquire) != 0 { @@ -443,7 +446,7 @@ impl M5ExecutionContext { pub(crate) fn is_current(&self) -> bool { if !self.published { return false; } let roots = self.roots(); - let Ok(inner) = self.slot.inner.lock() else { return false; }; + let inner = self.slot.inner.lock(); let slot = &inner.ttbs()[self.slot.context]; slot.ttb0.load(Ordering::Acquire) == roots[0] && slot.ttb1.load(Ordering::Acquire) == roots[1] @@ -456,7 +459,7 @@ impl M5ExecutionContext { /// lower root is explicit again, once the new owner's graph is prepared. pub(crate) fn rebind(&mut self, vm: &Vm) -> Result { { - let owner = vm.inner.exec_lock(None, false)?; + let owner = vm.gpuvm.lock_inner(); if owner.is_kernel || !core::ptr::eq( Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.slot.inner), ) { @@ -473,7 +476,8 @@ impl M5ExecutionContext { fn retract(&mut self) { if !self.published { return; } let roots = self.roots(); - if let Ok(inner) = self.slot.inner.lock() { + { + let inner = self.slot.inner.lock(); let slot = &inner.ttbs()[self.slot.context]; // Only our exact pair may be retracted. VM owners remain alive // through root invalidation and the final ASID synchronization. @@ -541,47 +545,23 @@ impl Vm { &self, addr: u64, size: usize, - gem: ARef, + gem: &gem::Object, prot: Prot, guard_pages: usize, ) -> Result { - let reserved = guard_pages.checked_mul(UAT_PGSZ) - .and_then(|guard| size.checked_add(guard)).ok_or(EOVERFLOW)?; - self.map_at_reserving(addr, size, gem, prot, reserved) + if size != gem.size() { + return Err(EINVAL); + } + let guard = guard_pages.checked_mul(UAT_PGSZ).ok_or(EOVERFLOW)?; + let reserved = size.checked_add(guard).ok_or(EOVERFLOW)?; + let end = addr.checked_add(reserved as u64).ok_or(EOVERFLOW)?; + // The range holds exactly one placement, so this is a fixed-address + // mapping of the whole object followed by its guard reservation. + self.map_in_range_lowest(gem, 0..size, UAT_PGSZ as u64, addr..end, prot, guard) } } impl KernelMapping { - /// Borrow the mapped GEM subrange through a temporary CPU mapping. - /// - /// The caller must serialize CPU access with firmware or GPU writes. This - /// helper only owns the kernel VMap lifetime and applies the object offset - /// carried by this mapping. - pub(crate) fn with_cpu_bytes( - &self, - f: impl FnOnce(&mut [u8]) -> Result, - ) -> Result { - let gem: &gem::Object = self.0._gem.as_deref().ok_or(EINVAL)?; - let end = self - .0 - .offset - .checked_add(self.0.mapped_size) - .ok_or(EOVERFLOW)?; - if end > gem.size() { - return Err(ERANGE); - } - let vmap = gem.vmap::()?; - // SAFETY: the VMap covers the complete GEM allocation and the checked - // subrange is retained by this KernelMapping. - let bytes = unsafe { - core::slice::from_raw_parts_mut( - vmap.as_mut_ptr().add(self.0.offset), - self.0.mapped_size, - ) - }; - f(bytes) - } - /// Map this shared GEM subrange into the end of another mapping's guards. /// The caller must keep GPU work quiescent until the complete resource /// update is published. Existing mappings are never removed or relocated. @@ -598,43 +578,56 @@ impl KernelMapping { } let available = anchor.0.size().checked_sub(anchor.size() as u64).ok_or(EINVAL)?; if size as u64 > available { return Err(ENOSPC); } - let sgt = Arc::new(gem.owned_sg_table()?, GFP_KERNEL)?; let owner_ref = anchor.0.owner.clone(); - let mut owner = owner_ref.exec_lock(Some(gem), false)?; - let vm_bo = owner_ref.obtain_bo(gem)?; - { - let mut guard = vm_bo.inner().inner.lock(); - if guard.sgt.is_none() { guard.sgt.replace(sgt); } - } + // As in Vm::map_at_reserving: hold the backing and prepare the VM BO + // outside the VM reservation, and end the bound-device lease first. + let (_bind_lease, vm_bo) = { + let gate = crate::binding::lookup(&owner_ref.lock_inner().dev)?; + let binding = gate.enter()?; + let dev = binding.device(); + let lease = crate::gem::held_sg_table(gem, dev)?.into_lease(); + let bo = owner_ref.obtain( + gem, + pin_init!(VmBoData { + newest_bind: AtomicU64::new(0), + segments <- new_mutex!(None), + }), + )?; + prepare_sgt_segments(dev, gem, &bo, &lease)?; + (lease, bo) + }; + // SAFETY: no other UniqRefGpuVm exists, as in Vm::map_at_reserving. + let mut vm_exec = unsafe { owner_ref.exec_lock_vm(None, false)? }; + let inner = vm_exec.unique_gpuvm.data(); let mut vm_bo = Some(MappingBo::new(vm_bo, owner_ref.clone())); // As in map_at_with_guards, the fallible allocator operation must not // own DMA-reservation references whose Drop would reacquire this lock. let node = anchor.0.as_mut().split_tail(size as u64, KernelMappingInner { - owner: owner_ref.clone(), uat_inner: owner.uat_inner.clone(), prot, + owner: owner_ref.clone(), uat_inner: inner.uat_inner.clone(), prot, bo: None, _gem: None, offset: self.0.offset, mapped_size: size, }); let mut node = match node { Ok(node) => node, - Err(error) => { drop(owner); return Err(error); } + Err(error) => { drop(vm_exec); return Err(error); } }; { - let inner = node.as_mut().inner_mut(); - inner.bo = vm_bo.take(); - inner._gem = Some(gem::KernelMappingPin::new(gem)); + let payload = node.as_mut().inner_mut(); + payload.bo = vm_bo.take(); + payload._gem = Some(gem::KernelMappingPin::new(gem)); } - let mapped = owner.map_node(&node, prot); + let mapped = inner.map_node(&node, prot); if let Err(error) = mapped { // Restore reservation coverage before the temporary mapping is // dropped. Its normal Drop unmaps partial PTEs and invalidates // every live context; the detached node cannot free the guard. let rollback = anchor.0.as_mut().rejoin_tail(node.as_mut()); - drop(owner); - drop(KernelMapping::new(node)); + drop(vm_exec); + drop(KernelMapping(ManuallyDrop::new(node))); rollback?; return Err(error); } - drop(owner); - Ok(KernelMapping::new(node)) + drop(vm_exec); + Ok(KernelMapping(ManuallyDrop::new(node))) } /// Undo a staged guard-tail alias before publication. Consume tails in From e06fb54f322aa8fb68fded6ba412ba45b4ac0bb5 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 04:18:58 -0500 Subject: [PATCH 008/117] drm/asahi: g18: move the backend onto the current driver core Adapt the imported backend to this driver. - It implements gpu::Gpu and queue::Queue as they are now. It uses the tracked-submission model, which is what the earlier port's batched VM_BIND and copy-before-sync submit amounted to. - Its timestamps are the shared CPU-written timestamp buffers, published at retirement. Their CPU path is now also built for G18. - The DRM device type, the IoMem/devres, MMIO, VMap and firmware request APIs, and the module's name are updated. - Its module parameters live in g18/params.rs. They are registered the way kernel::module! registers parameters, but only when the backend is built. - The T8152 SoC entry moves to hw/t8152.rs. - The G18-only PRIME import hook is dropped: this driver already gives core-created objects their driver data. --- drivers/gpu/drm/asahi/g18/boot/mod.rs | 92 +++--- .../gpu/drm/asahi/g18/boot/render_stage.rs | 8 +- drivers/gpu/drm/asahi/g18/boot/session.rs | 10 +- drivers/gpu/drm/asahi/g18/buffer/object.rs | 30 +- drivers/gpu/drm/asahi/g18/buffer/pool.rs | 26 +- .../drm/asahi/g18/buffer/pool_owner_probe.rs | 6 +- .../asahi/g18/buffer/pool_shared_compute.rs | 8 +- drivers/gpu/drm/asahi/g18/compat/dep_stats.rs | 2 +- .../gpu/drm/asahi/g18/compat/dvfs_parity.rs | 2 +- drivers/gpu/drm/asahi/g18/compat/e2e.rs | 2 +- drivers/gpu/drm/asahi/g18/compat/gem_stats.rs | 2 +- drivers/gpu/drm/asahi/g18/compat/host_path.rs | 10 +- drivers/gpu/drm/asahi/g18/compat/mem_path.rs | 2 +- .../gpu/drm/asahi/g18/compat/queue_slots.rs | 2 +- .../asahi/g18/compat/recovery_isolation.rs | 2 +- .../gpu/drm/asahi/g18/compat/recovery_path.rs | 2 +- .../gpu/drm/asahi/g18/compat/stackwatch.rs | 2 +- .../gpu/drm/asahi/g18/compat/timing_buffer.rs | 20 +- drivers/gpu/drm/asahi/g18/debug.rs | 4 +- drivers/gpu/drm/asahi/g18/drm.rs | 308 ++++++------------ drivers/gpu/drm/asahi/g18/fw/initdata.rs | 2 +- drivers/gpu/drm/asahi/g18/fw/render_params.rs | 2 +- drivers/gpu/drm/asahi/g18/hw.rs | 91 +----- .../gpu/drm/asahi/g18/initdata/application.rs | 2 +- .../asahi/g18/initdata/application_compute.rs | 14 +- drivers/gpu/drm/asahi/g18/initdata/mod.rs | 30 +- drivers/gpu/drm/asahi/g18/mod.rs | 1 + drivers/gpu/drm/asahi/g18/params.rs | 121 +++++++ drivers/gpu/drm/asahi/g18/probe.rs | 174 ++++++---- drivers/gpu/drm/asahi/g18/queue/graphics.rs | 14 +- .../asahi/g18/queue/graphics_application.rs | 2 +- drivers/gpu/drm/asahi/g18/queue/probe.rs | 8 +- drivers/gpu/drm/asahi/g18/queue/scheduler.rs | 8 +- drivers/gpu/drm/asahi/g18/regs.rs | 17 +- drivers/gpu/drm/asahi/g18/runtime/compute.rs | 6 +- .../drm/asahi/g18/runtime/compute_payload.rs | 14 +- .../gpu/drm/asahi/g18/runtime/graphics_job.rs | 30 +- .../g18/runtime/graphics_job_application.rs | 6 +- .../drm/asahi/g18/runtime/graphics_memory.rs | 6 +- .../gpu/drm/asahi/g18/uapi/render_policy.rs | 12 +- drivers/gpu/drm/asahi/hw/t8152.rs | 96 ++++++ drivers/gpu/drm/asahi/timestamp.rs | 44 ++- 42 files changed, 662 insertions(+), 578 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g18/params.rs create mode 100644 drivers/gpu/drm/asahi/hw/t8152.rs diff --git a/drivers/gpu/drm/asahi/g18/boot/mod.rs b/drivers/gpu/drm/asahi/g18/boot/mod.rs index 232b43acc6eb7..7cb051ec680f6 100644 --- a/drivers/gpu/drm/asahi/g18/boot/mod.rs +++ b/drivers/gpu/drm/asahi/g18/boot/mod.rs @@ -5,7 +5,7 @@ //! Plan milestone M1 step 1.8 plus the M0 probes. Derived from //! `m5_live_boot.rs`; the M5 (T8142) path is not touched by anything here. //! -//! The stage comes from `asahi.g18_initdata_test`. The driver's T8152 probe +//! The stage comes from `aurora.g18_initdata_test`. The driver's T8152 probe //! gate ([`crate::driver`]) has already sampled the three GPU PSD services, //! mapped SGX and, only if the sgx PSD proved the block powered, read the ID //! registers. It hands everything over in a [`T8152Probe`]. @@ -59,7 +59,7 @@ //! crashlog header and dumps what it finds. use crate::{ - driver::AsahiDevice, + driver::AsahiDrmDevice, g18::probe::{G18Runtime, T8152Probe}, g18::initdata::G18FirmwareMemory, g18::runtime::compute::G18PreparedComputeCommand, @@ -76,10 +76,8 @@ use core::{ use kernel::{ bindings, c_str, device::Core, - devres::Devres, error::{from_err_ptr, to_result}, - io::{mem::IoMem, Io, Mmio, MmioRaw}, - iosys_map::IoSysMapRef, + io::{mem::DevresIoMem, Io, IoSysMap, Mmio, MmioRaw, Region}, macros::vtable, new_mutex, platform, prelude::*, @@ -395,11 +393,11 @@ const fn psd_resource_name(service: regs::T8152PsdService) -> &'static CStr { /// 90..93 call it. struct PsdBlock { service: regs::T8152PsdService, - io: Pin>>>, + io: DevresIoMem, } impl PsdBlock { - fn map(pdev: &platform::Device, service: regs::T8152PsdService) -> Result { + fn map(pdev: &platform::Device>, service: regs::T8152PsdService) -> Result { let name = psd_resource_name(service); let resource = pdev.resource_by_name(name).ok_or(ENODEV)?; if resource.start() != service.physical_base() || resource.size() != PSD_BLOCK_SIZE { @@ -415,7 +413,7 @@ impl PsdBlock { return Err(EINVAL); } let request = pdev.io_request_by_name(name).ok_or(ENODEV)?; - let io = KBox::pin_init(request.iomap(), GFP_KERNEL)?; + let io = request.iomap()?.into_devres()?; Ok(Self { service, io }) } @@ -463,7 +461,7 @@ fn psd_line(sample: ®s::T8152PsdSample) -> (u32, u32, u32, u32, &'static str) ) } -fn log_psd(pdev: &platform::Device, label: &str, sample: ®s::T8152PsdSample) { +fn log_psd(pdev: &platform::Device>, label: &str, sample: ®s::T8152PsdSample) { let (id, state, power, ack, verdict) = psd_line(sample); dev_info!( pdev.as_ref(), @@ -478,7 +476,7 @@ fn log_psd(pdev: &platform::Device, label: &str, sample: ®s::T8152PsdSa } /// Fresh read-only samples of all three GPU services. -fn read_all_psd(pdev: &platform::Device) -> Result<[regs::T8152PsdSample; 3]> { +fn read_all_psd(pdev: &platform::Device>) -> Result<[regs::T8152PsdSample; 3]> { let mut samples = [regs::T8152PsdSample { service: regs::T8152PsdService::Sgx, state: 0, @@ -613,7 +611,7 @@ impl G18Provider { /// word when the block is not powered. Call only after [`Self::validate`]. fn postoffice_status( &self, - pdev: &platform::Device, + pdev: &platform::Device>, ) -> Result<(regs::T8152PsdSample, Option)> { let psd = regs::t8152_read_psd(pdev, self.role.psd())?; if !psd.powered() { @@ -629,10 +627,10 @@ impl G18Provider { if window == 0 || removing { return Err(ENODEV); } - let raw = MmioRaw::::new(window, POSTOFFICE_STATUS_WINDOW)?; + let raw = MmioRaw::>::new_region(window, POSTOFFICE_STATUS_WINDOW)?; // SAFETY: `raw` lies inside the provider's 0x80-byte window and // covers exactly the single status read below. - let status = unsafe { Mmio::from_raw(&raw) }.relaxed().read32(POSTOFFICE_STATUS); + let status = unsafe { Mmio::from_raw(raw) }.relaxed().read32(POSTOFFICE_STATUS); Ok((psd, Some(status))) } } @@ -877,7 +875,7 @@ impl Drop for PhysMap { /// Per-role callback state. Callbacks only publish atomics and log. struct G18RoleState { - dev: ARef, + dev: ARef, role: G18Role, /// The role's `__DATA` reservation (physical base, size). data: (u64, u64), @@ -915,7 +913,7 @@ impl G18RoleState { /// Scan one reservation every CRASHLOG_SCAN_STEP for a `CLHE` crashlog /// and dump what it finds. Read-only mapping. Returns the count found. fn scan_crashlog(&self, label: &str, region: (u64, u64), reason: &str) -> u32 { - let dev: &kernel::device::Device = self.dev.as_ref(); + let dev: &kernel::device::Device = self.dev.as_ref().as_ref(); let name = self.role.name(); let (base, size) = region; let Ok(size) = usize::try_from(size) else { @@ -970,7 +968,7 @@ impl rtkit::Buffer for G18NoRtkitBuffer { Err(ENOTSUPP) } - fn buf(&mut self) -> Result> { + fn buf(&mut self) -> Result> { Err(ENOTSUPP) } } @@ -1028,7 +1026,7 @@ impl rtkit::Operations for G18RtkitOps { match crashlog { Some(bytes) if !bytes.is_empty() => { data.crash_dumped.store(true, Ordering::Release); - dump_crashlog(data.dev.as_ref(), data.role.name(), "rtkit", bytes); + dump_crashlog(data.dev.as_ref().as_ref(), data.role.name(), "rtkit", bytes); } // Firmware-owned buffer: the RTKit core logged its DVA. _ => data.dump_data_crashlog("rtkit crash callback, firmware-owned buffer"), @@ -1056,7 +1054,7 @@ struct G18Session { application_job: Option>, uat: Arc, _registers: regs::Resources, - dev: ARef, + dev: ARef, stage: u32, } @@ -1072,7 +1070,7 @@ fn cntvct() -> u64 { impl G18Session { fn device(&self) -> &kernel::device::Device { - self.dev.as_ref() + self.dev.as_ref().as_ref() } fn any_crashed(&self) -> bool { @@ -1116,7 +1114,7 @@ impl G18Session { Err(error) => failure = Some(error), } } - log_cpu_state(self.dev.as_ref(), role, "after-stop", &self.roles[role.index()].provider); + log_cpu_state(self.dev.as_ref().as_ref(), role, "after-stop", &self.roles[role.index()].provider); } failure.map_or(Ok(()), Err) } @@ -1145,8 +1143,8 @@ impl G18Session { // Defense in depth for any future caller that bypasses driver probe. if self.stage == STAGE_RENDER { crate::g18::boot::render_stage::admit(self.device(), - *crate::module_parameters::g18_render_variant.value(), - *crate::module_parameters::g18_render_profile.value())?; + crate::g18::params::g18_render_variant.value(), + crate::g18::params::g18_render_profile.value())?; } self.boot_rtkit()?; if self.stage == STAGE_RTKIT { @@ -1188,7 +1186,7 @@ impl G18Session { for role in G18Role::START_ORDER { let dev = self.dev.clone(); let slot = &mut self.roles[role.index()]; - log_cpu_state(dev.as_ref(), role, "before-start", &slot.provider); + log_cpu_state(dev.as_ref().as_ref(), role, "before-start", &slot.provider); // The provider re-reads the role's PSD block and refuses an // unpowered block; it refuses without the CPU-control approval. let (control, _) = slot.provider.cpu_state()?; @@ -1198,7 +1196,7 @@ impl G18Session { dev_info!(dev.as_ref(), "G18_ASC {} adopted: RUN already set, start skipped\n", role.name()); } slot.started = true; - log_cpu_state(dev.as_ref(), role, "after-start", &slot.provider); + log_cpu_state(dev.as_ref().as_ref(), role, "after-start", &slot.provider); let started = Instant::::now(); let booted = Pin::new(&mut slot.rtkit).boot(); let elapsed = started.elapsed().as_micros_ceil(); @@ -1241,7 +1239,7 @@ impl G18Session { fsleep(Delta::from_millis(10)); } for role in G18Role::ALL { - log_cpu_state(self.dev.as_ref(), role, "after-syslog-window", &self.roles[role.index()].provider); + log_cpu_state(self.dev.as_ref().as_ref(), role, "after-syslog-window", &self.roles[role.index()].provider); dev_info!( self.device(), "G18_STAGE2 role={} non-system messages={}\n", @@ -1266,7 +1264,7 @@ impl G18Session { self.render_job.as_mut().ok_or(EIO)?)?; } if matches!(self.stage, STAGE_QUEUE_REGISTER | STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD) - || (self.stage == STAGE_DRM && *crate::module_parameters::g18_admit_compute.value() == 1) { + || (self.stage == STAGE_DRM && crate::g18::params::g18_admit_compute.value() == 1) { self.queue_probe = Some(G18QueueProbe::new(&self.dev, &self.uat, matches!(self.stage, STAGE_SCHEDULER_BIND | STAGE_POOL_BIND | STAGE_COMPUTE_ADD | STAGE_DRM))?); } @@ -1511,7 +1509,7 @@ impl G18Session { dev_info!(self.dev.as_ref(), "G18_COMPUTE completion outer={:?} inner={:?} stamps={:x?} times={:x?} event_masks={:x?} scheduler={:?} output={:?}\n", outer, inner, completion.stamps, completion.timestamps, masks, scheduler, output); - if *crate::module_parameters::g18_compute_variant.value() == 2 { + if crate::g18::params::g18_compute_variant.value() == 2 { let pixels = memory.triangle_output()?; dev_info!(self.dev.as_ref(), "G18_COMPUTE_TRIANGLE output={:?}\n", pixels); if !pixels.passed() { return Err(EIO); } @@ -1520,7 +1518,7 @@ impl G18Session { return Ok(()); } // g18_compute_variant=1 dispatches nothing: every output must stay poisoned. - let empty = *crate::module_parameters::g18_compute_variant.value() == 1; + let empty = crate::g18::params::g18_compute_variant.value() == 1; let expected = if empty { (0, 4096) } else { (4096, 0) }; if output.checked != 4096 || (output.correct, output.untouched) != expected { if empty { @@ -1608,7 +1606,7 @@ impl G18Session { dev_info!(self.dev.as_ref(), "G18_RENDER_PIXELS sha256={:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x}{:08x} background={} poison={}\n", pixels.0[0],pixels.0[1],pixels.0[2],pixels.0[3],pixels.0[4],pixels.0[5],pixels.0[6],pixels.0[7],pixels.1,pixels.2); self.check_faults("render pixels")?; - dev_info!(self.device(), "G18_RENDER_H1_PASS variant={} completion=independent pixels=exact owners=retained\n",*crate::module_parameters::g18_render_variant.value()); + dev_info!(self.device(), "G18_RENDER_H1_PASS variant={} completion=independent pixels=exact owners=retained\n",crate::g18::params::g18_render_variant.value()); return Ok(()); } if step%1000==999 { @@ -1686,8 +1684,8 @@ impl G18Runtime for G18Owner { // Stage 1: read-only probe // --------------------------------------------------------------------------- -fn stage_read_only(pdev: &platform::Device, probe: &T8152Probe) { - let dev: &kernel::device::Device = pdev.as_ref(); +fn stage_read_only(pdev: &platform::Device>, probe: &T8152Probe) { + let dev: &kernel::device::Device> = pdev.as_ref(); for sample in probe.psd.iter() { log_psd(pdev, "stage1 (probe sample)", sample); } @@ -1740,7 +1738,7 @@ fn stage_read_only(pdev: &platform::Device, probe: &T8152Probe) { /// Native PSD::getPsdAck order: poll ACK, then read STATE once. STATE /// clears ACK on J873; reading it while waiting loses the acknowledgement. -fn wait_psd_state(pdev: &platform::Device, block: &PsdBlock, target: u32) -> Result { +fn wait_psd_state(pdev: &platform::Device>, block: &PsdBlock, target: u32) -> Result { let started = Instant::::now(); for poll in 0..PSD_POLLS { if let Some(sample) = block.acknowledged_sample()? { @@ -1776,7 +1774,7 @@ fn claim_psd_request(dev: &kernel::device::Device, bit: u32) -> Result { Ok(()) } -fn map_psd_blocks(pdev: &platform::Device) -> Result> { +fn map_psd_blocks(pdev: &platform::Device>) -> Result> { let mut blocks = KVec::new(); for service in regs::T8152PsdService::ALL { blocks.push(PsdBlock::map(pdev, service)?, GFP_KERNEL)?; @@ -1793,8 +1791,8 @@ fn map_psd_blocks(pdev: &platform::Device) -> Result> { /// left as requested and the power state is reported UNKNOWN, to be /// restored by stage 93 under its own approval. The SGX ID is read only if /// sgx then proves powered. -fn stage_psd_power_on(pdev: &platform::Device, probe: &T8152Probe, service: regs::T8152PsdService) -> Result { - let dev: &kernel::device::Device = pdev.as_ref(); +fn stage_psd_power_on(pdev: &platform::Device>, probe: &T8152Probe, service: regs::T8152PsdService) -> Result { + let dev: &kernel::device::Device> = pdev.as_ref(); dev_warn!(dev, "G18_PSD stage {}: PMGR2 PSD power-on of service {} only (REQUIRES USER APPROVAL)\n", probe.stage, service.service_id()); let blocks = map_psd_blocks(pdev)?; @@ -1855,8 +1853,8 @@ fn stage_psd_power_on(pdev: &platform::Device, probe: &T8152Probe, service /// polling ACK before STATE. Refuses if a service is neither active nor /// idle, or if a firmware stage ran this boot. Stops at the first /// failure and reports the power state UNKNOWN. -fn stage_psd_restore_idle(pdev: &platform::Device) -> Result { - let dev: &kernel::device::Device = pdev.as_ref(); +fn stage_psd_restore_idle(pdev: &platform::Device>) -> Result { + let dev: &kernel::device::Device> = pdev.as_ref(); dev_warn!(dev, "G18_PSD stage 93: PMGR2 PSD idle restore (REQUIRES USER APPROVAL)\n"); let blocks = map_psd_blocks(pdev)?; let mut samples = [regs::T8152PsdSample { service: regs::T8152PsdService::Sgx, state: 0, ack: 0 }; 3]; @@ -1901,8 +1899,8 @@ fn stage_psd_restore_idle(pdev: &platform::Device) -> Result { // Stages 2..6 // --------------------------------------------------------------------------- -fn prepare_firmware_session(pdev: &platform::Device, probe: T8152Probe) -> Result { - let dev: &kernel::device::Device = pdev.as_ref(); +fn prepare_firmware_session(pdev: &platform::Device>, probe: T8152Probe) -> Result { + let dev: &kernel::device::Device> = pdev.as_ref(); let stage = probe.stage; dev_warn!(dev, "G18 stage {}: starts the GPU ASC CPUs (REQUIRES USER APPROVAL)\n", stage); if !probe.firmware_profile_admitted { @@ -1997,7 +1995,7 @@ fn prepare_firmware_session(pdev: &platform::Device, probe: T8152Probe) -> crash_dumped: AtomicBool::new(false), messages: AtomicU32::new(0), }, GFP_KERNEL)?; - let rtkit = rtkit::RtKit::::new_g17p(drm.as_ref(), None, role.index(), state.clone()) + let rtkit = rtkit::RtKit::::new_g17p(drm.as_ref().as_ref(), None, role.index(), state.clone()) .map_err(|error| no_probe_defer(dev, role.name(), "RTKit mailbox", error))?; let provider = providers.next().ok_or(EINVAL)?; slots.push(G18RoleSlot { role, provider, rtkit, started: false }, GFP_KERNEL)?; @@ -2027,12 +2025,16 @@ fn prepare_firmware_session(pdev: &platform::Device, probe: T8152Probe) -> // module stays loaded so RTKit callbacks keep valid code until reboot. G18_FIRMWARE_ATTEMPTED.store(true, Ordering::Release); // SAFETY: THIS_MODULE is this module; the reference is never dropped. - unsafe { bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; + unsafe { + bindings::__module_get( + ::THIS_MODULE.as_ptr(), + ) + }; Ok(Retained(Some(session))) } -fn stage_firmware(pdev: &platform::Device, probe: T8152Probe) -> Result> { - let dev: &kernel::device::Device = pdev.as_ref(); +fn stage_firmware(pdev: &platform::Device>, probe: T8152Probe) -> Result> { + let dev: &kernel::device::Device> = pdev.as_ref(); let stage = probe.stage; let mut retained = prepare_firmware_session(pdev, probe)?; if stage == STAGE_RENDER { @@ -2107,7 +2109,7 @@ fn stage_firmware(pdev: &platform::Device, probe: T8152Probe) -> Result= 1`. -pub(crate) fn start(pdev: &platform::Device, probe: T8152Probe) -> Result> { +pub(crate) fn start(pdev: &platform::Device>, probe: T8152Probe) -> Result> { let result = match probe.stage { STAGE_READ_ONLY => { stage_read_only(pdev, &probe); @@ -2125,6 +2127,6 @@ pub(crate) fn start(pdev: &platform::Device, probe: T8152Probe) -> Result< _ => Err(EINVAL), }; // Belt and braces: no stage may ever defer the GPU probe. - let dev: &kernel::device::Device = pdev.as_ref(); + let dev: &kernel::device::Device> = pdev.as_ref(); result.map_err(|error| no_probe_defer(dev, "GPU", "stage", error)) } diff --git a/drivers/gpu/drm/asahi/g18/boot/render_stage.rs b/drivers/gpu/drm/asahi/g18/boot/render_stage.rs index 51e2ff13c3369..163111f5ee368 100644 --- a/drivers/gpu/drm/asahi/g18/boot/render_stage.rs +++ b/drivers/gpu/drm/asahi/g18/boot/render_stage.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! H1 admission and retained-owner construction. H2 is deliberately excluded. //! The caller must execute every runtime readback gate before publication. -use crate::{driver::AsahiDevice, g18::runtime::graphics_job::G18GraphicsJob, +use crate::{driver::AsahiDrmDevice, g18::runtime::graphics_job::G18GraphicsJob, g18::fw::render_assembly::RenderVariant, mmu}; use kernel::prelude::*; @@ -33,9 +33,9 @@ pub(crate) fn admit(dev: &kernel::device::Device, variant: u32, profile: u32) -> } #[inline(never)] -pub(crate) fn prepare(dev: &AsahiDevice, uat: &mmu::Uat) -> Result> { - let variant = *crate::module_parameters::g18_render_variant.value(); - let profile = *crate::module_parameters::g18_render_profile.value(); +pub(crate) fn prepare(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result> { + let variant = crate::g18::params::g18_render_variant.value(); + let profile = crate::g18::params::g18_render_profile.value(); let options = options(variant, profile)?; G18GraphicsJob::from_payload(dev, uat, profile, options) } diff --git a/drivers/gpu/drm/asahi/g18/boot/session.rs b/drivers/gpu/drm/asahi/g18/boot/session.rs index 7ca41483103cb..a44fe7f872f22 100644 --- a/drivers/gpu/drm/asahi/g18/boot/session.rs +++ b/drivers/gpu/drm/asahi/g18/boot/session.rs @@ -9,7 +9,7 @@ mod recovery; use recovery::{Failure, JobEnd, JobFailure, JobPath}; use core::{ops::Range,sync::atomic::AtomicI32}; use crate::{g18::uapi::application::{G18ApplicationRender,G18RenderJobSink,G18RenderRetirement}, - g18::drm::{self as g18_drm,G18DrmRuntime},gem, + g18::drm::G18DrmRuntime, g18::runtime::graphics_job::application::ApplicationJob}; pub(crate) struct ApplicationOwner { @@ -131,10 +131,6 @@ impl G18DrmRuntime for ApplicationOwner { if self.failed.load(Ordering::Acquire) {return Err(EIO);} self.session.lock().0.as_ref().ok_or(EIO)?.uat.new_vm(id,kernel_range)?.with_t8152_lifetime() } - fn map_timestamp_buffer(&self,bo:gem::ObjectRef,range:Range)->Result { - if self.failed.load(Ordering::Acquire) {return Err(EIO);} - g18_drm::map_timestamp_buffer_in(&self.session.lock().0.as_ref().ok_or(EIO)?.uat,bo,range) - } } impl G18RenderJobSink for ApplicationOwner { fn submit_render(&self,vm:&mmu::Vm,render:&G18ApplicationRender)->Result { @@ -147,7 +143,7 @@ impl G18RenderJobSink for ApplicationOwner { } } -pub(crate) fn start_drm(pdev:&platform::Device,probe:T8152Probe) +pub(crate) fn start_drm(pdev:&platform::Device>,probe:T8152Probe) ->Result<(Arc,Arc)> { if probe.stage!=STAGE_DRM {return Err(EINVAL);} let retained=prepare_firmware_session(pdev,probe)?; @@ -216,7 +212,7 @@ impl G18Session { job.job.queues().prepare_application_pass(&self.dev,&pass,frame)?; self.poll_idle_faults("application pre-kick")?; let job=self.application_job.as_mut().ok_or(EIO)?; - let traced=job.traced() || *crate::module_parameters::g18_graphics_stall_trace.value()!=0; + let traced=job.traced() || crate::g18::params::g18_graphics_stall_trace.value()!=0; let roots=job.publish()?; let records=job.job.queues().publish_application_pass(frame)?; let outer_after=self.memory.as_mut().ok_or(EIO)?.publish_application_outer(&records,frame)?; diff --git a/drivers/gpu/drm/asahi/g18/buffer/object.rs b/drivers/gpu/drm/asahi/g18/buffer/object.rs index 006e57f9647cc..6697a5b3fa105 100644 --- a/drivers/gpu/drm/asahi/g18/buffer/object.rs +++ b/drivers/gpu/drm/asahi/g18/buffer/object.rs @@ -16,7 +16,17 @@ use core::ops::Range; use kernel::prelude::*; use super::live::g17p_live_write_u32; -use crate::{driver::AsahiDevice, gem, mmu}; +use crate::{driver::AsahiDrmDevice, gem, mmu}; + +/// Zero a new kernel object through its cached CPU mapping. +fn zero_object(object: &mut gem::ObjectRef) -> Result { + let size = object.size(); + let vmap = object.vmap()?; + // SAFETY: the VMap covers the object's complete allocation of `size` bytes, + // and the new object is not yet visible to the GPU or another CPU user. + unsafe { core::ptr::write_bytes(vmap.addr.cast::(), 0, size) }; + Ok(()) +} pub(crate) struct MappedObject { /// Present for an owning GPU mapping and absent for a subview retained by @@ -33,11 +43,11 @@ pub(crate) struct MappedObject { impl MappedObject { pub(crate) fn new_wc_in_range_with_guards( - dev: &AsahiDevice, vm: &mmu::Vm, range: Range, size: usize, + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: Range, size: usize, alignment: u64, prot: mmu::Prot, guard_pages: usize, ) -> Result { let mut object = gem::new_kernel_object_wc(dev, size)?; - object.vmap()?.memset(0); + zero_object(&mut object)?; let mapping = vm.map_in_range_with_guards(&object.gem, 0..object.size(), alignment, range, prot, guard_pages)?; let iova = mapping.iova(); @@ -53,7 +63,7 @@ impl MappedObject { pub(crate) fn new_wc_in_range( - dev: &AsahiDevice, + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: Range, size: usize, @@ -61,7 +71,7 @@ impl MappedObject { prot: mmu::Prot, ) -> Result { let mut object = gem::new_kernel_object_wc(dev, size)?; - object.vmap()?.memset(0); + zero_object(&mut object)?; let mapping = object.map_into_range(vm, range, alignment, prot, true)?; let iova = mapping.iova(); Ok(Self { @@ -75,14 +85,14 @@ impl MappedObject { } pub(crate) fn new_wc_at( - dev: &AsahiDevice, + dev: &AsahiDrmDevice, vm: &mmu::Vm, address: u64, size: usize, prot: mmu::Prot, ) -> Result { let mut object = gem::new_kernel_object_wc(dev, size)?; - object.vmap()?.memset(0); + zero_object(&mut object)?; let mapping = object.map_at(vm, address, prot, false)?; let iova = mapping.iova(); Ok(Self { @@ -103,7 +113,7 @@ impl MappedObject { /// backing; this helper does not synchronize. pub(crate) fn regrow_wc_at( &mut self, - dev: &AsahiDevice, + dev: &AsahiDrmDevice, vm: &mmu::Vm, address: u64, size: usize, @@ -112,7 +122,7 @@ impl MappedObject { return Ok(()); } let mut object = gem::new_kernel_object_wc(dev, size)?; - object.vmap()?.memset(0); + zero_object(&mut object)?; self.mapping = None; let mapping = object.map_at(vm, address, mmu::PROT_GPU_FW_SHARED_RW, false)?; self.iova = mapping.iova(); @@ -134,7 +144,7 @@ impl MappedObject { // SAFETY: the VMap covers this object's complete, page-rounded // allocation; construction bounds-checks the view offset and size. core::slice::from_raw_parts_mut( - vmap.as_mut_ptr().add(self.object_offset), + vmap.addr.cast::().add(self.object_offset), self.logical_size, ) }; diff --git a/drivers/gpu/drm/asahi/g18/buffer/pool.rs b/drivers/gpu/drm/asahi/g18/buffer/pool.rs index 4db53eace3bb2..a05985085b8bb 100644 --- a/drivers/gpu/drm/asahi/g18/buffer/pool.rs +++ b/drivers/gpu/drm/asahi/g18/buffer/pool.rs @@ -20,7 +20,7 @@ mod shared_compute; mod owner_probe; use crate::g18::buffer::object::MappedObject; -use crate::{driver::AsahiDevice, g18::fw::uma as g18_uma, mmu}; +use crate::{driver::AsahiDrmDevice, g18::fw::uma as g18_uma, mmu}; use core::ops::Range; use core::sync::atomic::{fence, Ordering}; use kernel::prelude::*; @@ -92,7 +92,7 @@ impl G18UmaPoolMemory { /// Separate application compute pool. The four native gallery compute /// captures use four two-MiB blocks; reserve their maximum observed result /// pair/counter for the initial bounded, non-spilling dispatch path. - pub(crate) fn new_application_compute(dev: &AsahiDevice, uat: &mmu::Uat, + pub(crate) fn new_application_compute(dev: &AsahiDrmDevice, uat: &mmu::Uat, high_range: Range) -> Result { let mut pool = Self::new_layout(dev, uat, high_range.clone(), 12, true, false, 0x40000000, 1, 0x200000)?; @@ -103,7 +103,7 @@ impl G18UmaPoolMemory { Ok(pool) } - pub(crate) fn compute_application_aliases(&mut self, dev: &AsahiDevice, + pub(crate) fn compute_application_aliases(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm) -> Result> { if self.render || ![0x40000000, 0x2_4000_0000, 0x3_4000_0000].contains(&self.low_bias) || !self.process_aliases.is_empty() { @@ -123,7 +123,7 @@ impl G18UmaPoolMemory { } pub(crate) fn new( - dev: &AsahiDevice, + dev: &AsahiDrmDevice, uat: &mmu::Uat, high_range: Range, uscs: u32, @@ -137,7 +137,7 @@ impl G18UmaPoolMemory { /// which has no room for guards: with them the directory pages alone /// exhaust the window (stage33 H1, ENOSPC). Compute keeps its guards. fn new_with_guard( - dev: &AsahiDevice, + dev: &AsahiDrmDevice, uat: &mmu::Uat, high_range: Range, uscs: u32, @@ -147,7 +147,7 @@ impl G18UmaPoolMemory { Self::new_layout(dev, uat, high_range, uscs, guard, ideal, 0, 0, 0x100000) } - fn new_layout(dev: &AsahiDevice, uat: &mmu::Uat, high_range: Range, + fn new_layout(dev: &AsahiDrmDevice, uat: &mmu::Uat, high_range: Range, uscs: u32, guard: bool, ideal: bool, low_bias: u64, hardware_slot: u32, block_bytes: u64) -> Result { if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 { @@ -289,7 +289,7 @@ impl G18UmaPoolMemory { /// Separate deterministic owner for H1. The firmware-memory constructor /// selects this INSTEAD OF the compute pool; their low windows cannot /// coexist. All compute constructor and backing policy remain unchanged. - pub(crate) fn new_render(dev: &AsahiDevice, uat: &mmu::Uat) -> Result { + pub(crate) fn new_render(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result { use crate::g18::address_map as va; let mut owner = Self::new_with_guard(dev, uat, va::GRAPHICS_UMA_HIGH..va::GRAPHICS_UMA_HIGH + 0x40000, 12, false, @@ -301,7 +301,7 @@ impl G18UmaPoolMemory { Ok(owner) } - pub(crate) fn prepare_render_backing(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + pub(crate) fn prepare_render_backing(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm) -> Result { use crate::{g18::address_map as va, g18::uapi::render_policy as policy}; let blocks = policy::uma_blocks(); @@ -352,7 +352,7 @@ impl G18UmaPoolMemory { /// A failed allocation/alias check leaves the prior empty pool untouched. /// No firmware command, process root or hardware pool binding is published. pub(crate) fn prepare_compute_backing( - &mut self, dev: &AsahiDevice, uat: &mmu::Uat, high_range: Range, + &mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, high_range: Range, plan: g18_uma::G18ComputeScratchPlan, ) -> Result<()> { if self.encoded || self.scratch_plan.is_some() { return Err(EBUSY); } @@ -425,7 +425,7 @@ impl G18UmaPoolMemory { /// while these existed only in the kernel lower VM. Holes stay unmapped. /// Call after the backing exists and before the job's roots are published. pub(crate) fn alias_into_process( - &mut self, dev: &AsahiDevice, uat: &mmu::Uat, vm: &mmu::Vm, + &mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, ) -> Result<()> { let plan = self.scratch_plan.ok_or(EINVAL)?; if !self.process_aliases.is_empty() { return Err(EBUSY); } @@ -542,7 +542,7 @@ impl G18UmaPoolMemory { /// foreign binding or a second load. pub(crate) fn load_hardware_descriptor( &mut self, - dev: &AsahiDevice, + dev: &AsahiDrmDevice, table: &mut MappedObject, binding: G18UmaPoolBinding, device_config: [u64; 4], @@ -554,7 +554,7 @@ impl G18UmaPoolMemory { /// Requires the caller's retired scheduler and old firmware binding check. /// This only replaces our own still-retained descriptor; it cannot reclaim /// an arbitrary occupied slot or reset an already encoded view in place. - pub(crate) fn replace_hardware_descriptor(&mut self, dev: &AsahiDevice, + pub(crate) fn replace_hardware_descriptor(&mut self, dev: &AsahiDrmDevice, table: &mut MappedObject, binding: G18UmaPoolBinding, previous: &mut Self, old: G18UmaPoolBinding, device_config: [u64; 4]) -> Result<[u64; 16]> @@ -572,7 +572,7 @@ impl G18UmaPoolMemory { self.write_hardware_descriptor(dev, table, binding, device_config, expected, true) } - fn write_hardware_descriptor(&mut self, dev: &AsahiDevice, table: &mut MappedObject, + fn write_hardware_descriptor(&mut self, dev: &AsahiDrmDevice, table: &mut MappedObject, binding: G18UmaPoolBinding, device_config: [u64; 4], expected: [u64; 16], retired_view: bool) -> Result<[u64; 16]> diff --git a/drivers/gpu/drm/asahi/g18/buffer/pool_owner_probe.rs b/drivers/gpu/drm/asahi/g18/buffer/pool_owner_probe.rs index 3b33415bd6ce7..ed6d3cde1d966 100644 --- a/drivers/gpu/drm/asahi/g18/buffer/pool_owner_probe.rs +++ b/drivers/gpu/drm/asahi/g18/buffer/pool_owner_probe.rs @@ -80,7 +80,7 @@ impl G18UmaPoolMemory { /// E2E injection at the last allocation boundary: a complete replacement /// is discarded before descriptor publication. Compare every old mapping, /// state byte and list entry, then let the real preparation reuse the bank. - pub(crate) fn verify_replacement_rollback(&mut self, dev: &AsahiDevice, + pub(crate) fn verify_replacement_rollback(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, plan: G18ComputePlan) -> Result { use crate::g18::address_map as va; let before = self.owner_snapshot(uat)?; @@ -103,7 +103,7 @@ impl G18UmaPoolMemory { } #[inline(never)] - pub(crate) fn verify_shared_compute_owner(dev: &AsahiDevice, uat: &mmu::Uat) -> Result { + pub(crate) fn verify_shared_compute_owner(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result { use crate::g18::address_map as va; let high = va::DYNAMIC_HIGH_START..va::DYNAMIC_HIGH_END; let mut words = [48,12,0,0,2304,24,0,0,2,1,0,0,0,0,0,0,1,0,0,128,4,0,0]; @@ -200,7 +200,7 @@ impl G18UmaPoolMemory { } #[inline(never)] - fn verify_guard_tiers(dev: &AsahiDevice, uat: &mmu::Uat, high: Range) -> Result { + fn verify_guard_tiers(dev: &AsahiDrmDevice, uat: &mmu::Uat, high: Range) -> Result { let low = DIRECTORY_BASE + 0x40000000; let mut anchor_object = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), high.clone(), mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW)?; diff --git a/drivers/gpu/drm/asahi/g18/buffer/pool_shared_compute.rs b/drivers/gpu/drm/asahi/g18/buffer/pool_shared_compute.rs index 69e57d0bb4623..f8e9a17cf7817 100644 --- a/drivers/gpu/drm/asahi/g18/buffer/pool_shared_compute.rs +++ b/drivers/gpu/drm/asahi/g18/buffer/pool_shared_compute.rs @@ -47,7 +47,7 @@ impl Drop for DirectoryTransaction<'_> { } impl G18UmaPoolMemory { - pub(crate) fn new_shared_compute(dev: &AsahiDevice, uat: &mmu::Uat, + pub(crate) fn new_shared_compute(dev: &AsahiDrmDevice, uat: &mmu::Uat, high: Range, plan: G18ComputePlan) -> Result { let mut pool = Self::new_layout(dev, uat, high.clone(), 12, @@ -59,7 +59,7 @@ impl G18UmaPoolMemory { /// A fresh FList view, retaining the previous scratch GEM objects while /// giving the directory, list, state and counter independent ownership. /// The caller must keep the old view until a verified firmware rebind. - pub(crate) fn replacement_shared_compute(&self, dev: &AsahiDevice, uat: &mmu::Uat, + pub(crate) fn replacement_shared_compute(&self, dev: &AsahiDrmDevice, uat: &mmu::Uat, high: Range, plan: G18ComputePlan) -> Result { if self.render || !self.encoded || !self.descriptor_loaded @@ -76,13 +76,13 @@ impl G18UmaPoolMemory { /// Increase an unpublished pool without moving any existing mapping. /// Every backing object, alias and list entry is staged. A failure restores /// all directory guards and leaves the previous plan and block list intact. - pub(crate) fn prepare_shared_compute(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + pub(crate) fn prepare_shared_compute(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, high: Range, plan: G18ComputePlan) -> Result { self.prepare_shared_compute_retaining(dev, uat, high, plan, None) } - fn prepare_shared_compute_retaining(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + fn prepare_shared_compute_retaining(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, high: Range, plan: G18ComputePlan, retained: Option<&Self>) -> Result { if self.encoded || self.descriptor_loaded || !self.process_aliases.is_empty() { diff --git a/drivers/gpu/drm/asahi/g18/compat/dep_stats.rs b/drivers/gpu/drm/asahi/g18/compat/dep_stats.rs index d8f1efc45ccce..9eef0d51ccdc7 100644 --- a/drivers/gpu/drm/asahi/g18/compat/dep_stats.rs +++ b/drivers/gpu/drm/asahi/g18/compat/dep_stats.rs @@ -94,7 +94,7 @@ static STATS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static STATS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_render_dep_stats" - } else { c"asahi.g17p_render_dep_stats" }), + } else { c"aurora.g17p_render_dep_stats" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] diff --git a/drivers/gpu/drm/asahi/g18/compat/dvfs_parity.rs b/drivers/gpu/drm/asahi/g18/compat/dvfs_parity.rs index 16d09f9309b5a..3ec61a902813d 100644 --- a/drivers/gpu/drm/asahi/g18/compat/dvfs_parity.rs +++ b/drivers/gpu/drm/asahi/g18/compat/dvfs_parity.rs @@ -115,7 +115,7 @@ static PARITY_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static PARITY_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_dvfs_parity" - } else { c"asahi.g17p_dvfs_parity" }), + } else { c"aurora.g17p_dvfs_parity" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] diff --git a/drivers/gpu/drm/asahi/g18/compat/e2e.rs b/drivers/gpu/drm/asahi/g18/compat/e2e.rs index d150f57305c18..173d2571ee38d 100644 --- a/drivers/gpu/drm/asahi/g18/compat/e2e.rs +++ b/drivers/gpu/drm/asahi/g18/compat/e2e.rs @@ -362,7 +362,7 @@ static E2E_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static E2E_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_e2e" - } else { c"asahi.g17p_e2e" }), + } else { c"aurora.g17p_e2e" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] diff --git a/drivers/gpu/drm/asahi/g18/compat/gem_stats.rs b/drivers/gpu/drm/asahi/g18/compat/gem_stats.rs index 22f02811b9478..8f81b2d02a575 100644 --- a/drivers/gpu/drm/asahi/g18/compat/gem_stats.rs +++ b/drivers/gpu/drm/asahi/g18/compat/gem_stats.rs @@ -81,7 +81,7 @@ static STATS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static STATS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"gem_stats" - } else { c"asahi.gem_stats" }), + } else { c"aurora.gem_stats" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] diff --git a/drivers/gpu/drm/asahi/g18/compat/host_path.rs b/drivers/gpu/drm/asahi/g18/compat/host_path.rs index fb3bf77011806..2403560feae6c 100644 --- a/drivers/gpu/drm/asahi/g18/compat/host_path.rs +++ b/drivers/gpu/drm/asahi/g18/compat/host_path.rs @@ -718,7 +718,7 @@ static PCT_E_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_host_path" - } else { c"asahi.g17p_host_path" }), + } else { c"aurora.g17p_host_path" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] @@ -731,7 +731,7 @@ static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { static PCT_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_feed_perf_pct" - } else { c"asahi.g17p_feed_perf_pct" }), + } else { c"aurora.g17p_feed_perf_pct" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] @@ -744,7 +744,7 @@ static PCT_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { static DEPTH_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_early_depth" - } else { c"asahi.g17p_early_depth" }), + } else { c"aurora.g17p_early_depth" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] @@ -757,7 +757,7 @@ static DEPTH_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { static PCT_P_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_feed_perf_pct_p" - } else { c"asahi.g17p_feed_perf_pct_p" }), + } else { c"aurora.g17p_feed_perf_pct_p" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] @@ -770,7 +770,7 @@ static PCT_P_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { static PCT_E_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_feed_perf_pct_e" - } else { c"asahi.g17p_feed_perf_pct_e" }), + } else { c"aurora.g17p_feed_perf_pct_e" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] diff --git a/drivers/gpu/drm/asahi/g18/compat/mem_path.rs b/drivers/gpu/drm/asahi/g18/compat/mem_path.rs index 535ab69caeadc..e658033088d7c 100644 --- a/drivers/gpu/drm/asahi/g18/compat/mem_path.rs +++ b/drivers/gpu/drm/asahi/g18/compat/mem_path.rs @@ -226,7 +226,7 @@ static MASK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_mem_path" - } else { c"asahi.g17p_mem_path" }), + } else { c"aurora.g17p_mem_path" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] diff --git a/drivers/gpu/drm/asahi/g18/compat/queue_slots.rs b/drivers/gpu/drm/asahi/g18/compat/queue_slots.rs index 62f57e101dcbb..0de236b4ab872 100644 --- a/drivers/gpu/drm/asahi/g18/compat/queue_slots.rs +++ b/drivers/gpu/drm/asahi/g18/compat/queue_slots.rs @@ -178,7 +178,7 @@ static SLOTS_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static SLOTS_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_queue_slots" - } else { c"asahi.g17p_queue_slots" }), + } else { c"aurora.g17p_queue_slots" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] diff --git a/drivers/gpu/drm/asahi/g18/compat/recovery_isolation.rs b/drivers/gpu/drm/asahi/g18/compat/recovery_isolation.rs index 933ad025526f7..22d7995b6a000 100644 --- a/drivers/gpu/drm/asahi/g18/compat/recovery_isolation.rs +++ b/drivers/gpu/drm/asahi/g18/compat/recovery_isolation.rs @@ -634,7 +634,7 @@ static MASK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_recovery_isolation" - } else { c"asahi.g17p_recovery_isolation" }), + } else { c"aurora.g17p_recovery_isolation" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] diff --git a/drivers/gpu/drm/asahi/g18/compat/recovery_path.rs b/drivers/gpu/drm/asahi/g18/compat/recovery_path.rs index 5ae0b7c7ac662..f35cf67d91715 100644 --- a/drivers/gpu/drm/asahi/g18/compat/recovery_path.rs +++ b/drivers/gpu/drm/asahi/g18/compat/recovery_path.rs @@ -389,7 +389,7 @@ static MASK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static MASK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_recovery_path" - } else { c"asahi.g17p_recovery_path" }), + } else { c"aurora.g17p_recovery_path" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] diff --git a/drivers/gpu/drm/asahi/g18/compat/stackwatch.rs b/drivers/gpu/drm/asahi/g18/compat/stackwatch.rs index 23d8977ed8d2f..2c74f17ef7dda 100644 --- a/drivers/gpu/drm/asahi/g18/compat/stackwatch.rs +++ b/drivers/gpu/drm/asahi/g18/compat/stackwatch.rs @@ -140,7 +140,7 @@ static SW_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static SW_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_stack_watch" - } else { c"asahi.g17p_stack_watch" }), + } else { c"aurora.g17p_stack_watch" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] diff --git a/drivers/gpu/drm/asahi/g18/compat/timing_buffer.rs b/drivers/gpu/drm/asahi/g18/compat/timing_buffer.rs index 5e4c614f196f9..611465fa139b3 100644 --- a/drivers/gpu/drm/asahi/g18/compat/timing_buffer.rs +++ b/drivers/gpu/drm/asahi/g18/compat/timing_buffer.rs @@ -142,8 +142,8 @@ unsafe extern "C" fn set(value: *const core::ffi::c_char, _: *const bindings::ke let arg = &command[..len]; match arg { b"1" | b"1\n" => { - if *crate::module_parameters::g17p_completion_trace.value() != 0 - || *crate::module_parameters::g17p_trace_depth.value() != 0 { return -22; } + if crate::g18::params::g17p_completion_trace.value() != 0 + || crate::g18::params::g17p_trace_depth.value() != 0 { return -22; } if STATE.load(Ordering::Relaxed) != 0 { return -16; } NEXT.store(0, Ordering::Relaxed); STATE.store(1, Ordering::Relaxed); @@ -179,7 +179,7 @@ static OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_timing_buffer" - } else { c"asahi.g17p_timing_buffer" }), + } else { c"aurora.g17p_timing_buffer" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] @@ -292,7 +292,7 @@ static COMPLETION_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static COMPLETION_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_completion_buffer" - } else { c"asahi.g17p_completion_buffer" }), + } else { c"aurora.g17p_completion_buffer" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] @@ -472,7 +472,7 @@ static PEEK_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static PEEK_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_fw_peek" - } else { c"asahi.g17p_fw_peek" }), + } else { c"aurora.g17p_fw_peek" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] @@ -527,7 +527,7 @@ static GRID_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static GRID_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_grid_peek" - } else { c"asahi.g17p_grid_peek" }), + } else { c"aurora.g17p_grid_peek" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] @@ -596,7 +596,7 @@ static GPC_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static GPC_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_gpc_sample" - } else { c"asahi.g17p_gpc_sample" }), + } else { c"aurora.g17p_gpc_sample" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] @@ -612,7 +612,7 @@ static DIAG_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static DIAG_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_diag_knobs" - } else { c"asahi.g17p_diag_knobs" }), + } else { c"aurora.g17p_diag_knobs" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] @@ -731,7 +731,7 @@ static MSGLOG_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static MSGLOG_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_msglog" - } else { c"asahi.g17p_msglog" }), + } else { c"aurora.g17p_msglog" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] @@ -919,7 +919,7 @@ static PSTATE_OPS: bindings::kernel_param_ops = bindings::kernel_param_ops { static PSTATE_PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g17p_stats_pstate" - } else { c"asahi.g17p_stats_pstate" }), + } else { c"aurora.g17p_stats_pstate" }), #[cfg(MODULE)] mod_: unsafe { core::ptr::from_ref(&bindings::__this_module).cast_mut() }, #[cfg(not(MODULE))] diff --git a/drivers/gpu/drm/asahi/g18/debug.rs b/drivers/gpu/drm/asahi/g18/debug.rs index 8336c9f67df60..99b28228eced1 100644 --- a/drivers/gpu/drm/asahi/g18/debug.rs +++ b/drivers/gpu/drm/asahi/g18/debug.rs @@ -6,7 +6,7 @@ //! the port's extraction ledger); only module paths //! are rewritten. use core::sync::atomic::{AtomicBool, Ordering}; -use crate::module_parameters; +use crate::g18::params; /// Set once the G18 application DRM node is live (`G18_DRM_READY`). static G18_DRM_LIVE: AtomicBool = AtomicBool::new(false); @@ -21,7 +21,7 @@ pub(crate) fn g18_mark_drm_live() { /// recovery, refusal, wait, stall and admission lines never consult this. #[inline(always)] pub(crate) fn g18_kick_log() -> bool { - *module_parameters::g18_kick_log.value() != 0 || !G18_DRM_LIVE.load(Ordering::Relaxed) + params::g18_kick_log.value() != 0 || !G18_DRM_LIVE.load(Ordering::Relaxed) } /// dev_info!() for a G18 per-kick, frame-pass or per-job resource line diff --git a/drivers/gpu/drm/asahi/g18/drm.rs b/drivers/gpu/drm/asahi/g18/drm.rs index 9e826e69d4d8a..8dfd44c87b5a0 100644 --- a/drivers/gpu/drm/asahi/g18/drm.rs +++ b/drivers/gpu/drm/asahi/g18/drm.rs @@ -6,27 +6,23 @@ //! publication guard. It differs where G18 differs: //! - render commands go through `g18_application` to a job sink the G18 //! runtime implements, not through a captured reference profile; -//! - `map_timestamp_buffer` is implemented (contract K8), so Mesa's G18G -//! screen exposes timer queries and with them GL 3.3-4.6; +//! - `map_timestamp_buffer` is implemented (contract K8) with the shared +//! CPU-written timestamp buffers, so Mesa's G18G screen exposes timer +//! queries and with them GL 3.3-4.6; //! - compute is refused unless the adapter is built with `admit_compute`. //! -//! The adapter holds no firmware state. Everything it needs from the live -//! G18 session goes through [`G18DrmRuntime`]. -//! -//! [`asahi_gem_prime_import`] is the driver's `drm_driver.gem_prime_import` -//! hook. On the G18 device it stops foreign dma-bufs from becoming bare -//! shmem objects which asahi ioctls then misread (module 1134); every other -//! device keeps the DRM core's default import, as on t6050-gpu. +//! Commands use the shared command layout (`include/uapi/drm/asahi_drm.h`); +//! `g18::uapi::command` parses it. The adapter holds no firmware state. +//! Everything it needs from the live G18 session goes through +//! [`G18DrmRuntime`]. use core::ops::Range; -use core::sync::atomic::{fence, AtomicBool, AtomicPtr, Ordering}; -use kernel::{bindings, c_str, macros::vtable, new_mutex, prelude::*, static_lock_class, +use kernel::{bindings, macros::vtable, new_mutex, prelude::*, static_lock_class, dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, - sync::{Arc, Mutex}, uapi, xarray}; -use crate::{alloc, file, g18::uapi::command as g17_uapi, g18::hw, - g18::address_map as va, + sync::{Arc, Mutex, SpinLock}, uapi, xarray}; +use crate::{alloc, driver::AsahiRegData, file, g18::uapi::command as g17_uapi, g18::hw, g18::uapi::application::{self as g18_application, G18ApplicationRefusal, G18RenderJobSink, G18RenderRetirement}, - gem, gpu, g18::fw::vm_layout as m5_vm, mmu, queue}; + gem, gpu, g18::fw::vm_layout as m5_vm, mmu, queue, stats, stats_raw, timestamp}; const USER_END: u64 = (1 << 42) - 2 * mmu::UAT_PGSZ as u64; /// Largest compute control stream admitted, in bytes. The kernel does not read @@ -37,10 +33,36 @@ const USER_END: u64 = (1 << 42) - 2 * mmu::UAT_PGSZ as u64; pub(crate) const G18_COMPUTE_STREAM_BYTES: u64 = 64 * 1024; /// `DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE`: the dispatch binds texture or image /// state, so the command takes the native texture compute mode (d62a, -/// `g18_application_compute_resources.rs`). The only compute flag G18 admits. -/// Spelled as its value, `(1 << 1)` in include/uapi/drm/asahi_drm.h, because -/// the incremental module build reuses pinned bindgen output that predates it. -const G18_COMPUTE_TEXTURE_STATE: u32 = 1 << 1; +/// `g18_application_compute_resources.rs`). +const G18_COMPUTE_TEXTURE_STATE: u32 = + uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE as u32; + +// `g18::uapi::command` parses by byte offset (it is also built on the host); +// keep its constants equal to the UAPI header. +const _: () = { + use core::mem::{offset_of, size_of}; + assert!(g17_uapi::UAPI_RENDER_SIZE == size_of::()); + assert!(g17_uapi::UAPI_COMPUTE_SIZE == size_of::()); + assert!(g17_uapi::UAPI_G18_RENDER_SIZE == size_of::()); + assert!(g17_uapi::UAPI_G18_COMPUTE_SIZE == size_of::()); + assert!(offset_of!(uapi::drm_asahi_cmd_render_g18, resources) == g17_uapi::UAPI_RENDER_SIZE); + assert!(offset_of!(uapi::drm_asahi_cmd_compute_g18, resources) == g17_uapi::UAPI_COMPUTE_SIZE); + assert!(size_of::() == 80); + assert!(size_of::() == 64); + assert!(offset_of!(uapi::drm_asahi_cmd_render, ray_query) == 0x150); + assert!(offset_of!(uapi::drm_asahi_cmd_compute, prior_vdm) == 0x88); + assert!(offset_of!(uapi::drm_asahi_cmd_render, ts_frag_end_copies) == 0x100); + assert!(offset_of!(uapi::drm_asahi_cmd_render, prior_vdm) == 0x140); + assert!(offset_of!(uapi::drm_asahi_cmd_render, prior_cdm) == 0x148); + assert!(offset_of!(uapi::drm_asahi_cmd_compute, ts_end_copies) == 0x40); + assert!(offset_of!(uapi::drm_asahi_cmd_compute, scs_layout) == 0x80); + assert!(offset_of!(uapi::drm_asahi_cmd_compute, prior_cdm) == 0x90); + assert!(offset_of!(uapi::drm_asahi_cmd_compute, cdm_auxiliary_allocation) == 0x98); + assert!(g17_uapi::RENDER_G18_RESOURCES + == uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_G18_RESOURCES as u32); + assert!(g17_uapi::COMPUTE_G18_RESOURCES + == uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_G18_RESOURCES as u32); +}; // Identity Mesa's G18G bring-up gate accepts (`agx_chip.c:471-477`) and the // drm-shim `g18g` profile publishes (`asahi_noop.c:129-147`), measured on @@ -55,29 +77,9 @@ const G18_USC_GENERATION: u32 = 3; const G18_REVISION_B1: u32 = hw::GpuRevision::B1 as u32; /// `DRM_ASAHI_GPU_HAL_300`: the UAPI has no HAL400 value yet and Mesa's /// G18G gate requires 300. Apple's kext is PI_400. -const G18_UAPI_HAL_GENERATION: u32 = 300; - -/// Kernel-VM window for K8 timestamp buffers. It must stay outside every -/// fixed G18 reservation; the assertion below re-checks that whenever the -/// address map changes. -pub(crate) const G18_TIMESTAMP_KERNEL_RANGE: Range = - (va::KERNEL_BASE + 0xc600_0000)..(va::KERNEL_BASE + 0xc700_0000); - -const fn timestamp_range_is_free() -> bool { - let (start, end) = (G18_TIMESTAMP_KERNEL_RANGE.start, G18_TIMESTAMP_KERNEL_RANGE.end); - let mut i = 0; - while i < va::REGIONS.len() { - let region = va::REGIONS[i]; - if start < region.base + region.bytes && region.base < end { - return false; - } - i += 1; - } - true -} -const _: () = assert!(timestamp_range_is_free()); +const G18_UAPI_HAL_GENERATION: u32 = uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_300 as u32; -static QUEUE_NAME: &CStr = c_str!("g18_queue"); +static QUEUE_NAME: &CStr = c"g18_queue"; static QUEUE_CLASS_KEY: Pin<&'static kernel::sync::LockClassKey> = static_lock_class!(); /// What the adapter needs from the live G18 session. The conductor @@ -89,10 +91,6 @@ pub(crate) trait G18DrmRuntime: G18RenderJobSink { /// VM_CREATE: a user VM in the session UAT. The adapter has already /// checked `kernel_range` against the M5/Mesa compact contract. fn new_user_vm(&self, id: u64, kernel_range: Range) -> Result; - /// K8: map a timestamp BO range. [`map_timestamp_buffer_in`] is the - /// intended implementation. - fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) - -> Result; /// Compute (K1). Only called when the adapter admits compute. fn submit_compute(&self, _vm: &mmu::Vm, _command: &g17_uapi::TranslatedComputeCommand) -> Result { @@ -100,17 +98,6 @@ pub(crate) trait G18DrmRuntime: G18RenderJobSink { } } -/// K8 mapping for [`G18DrmRuntime::map_timestamp_buffer`]: the G17P manager -/// body (`g17_manager.rs:3949-3962`) in the G18 kernel VM, inside -/// [`G18_TIMESTAMP_KERNEL_RANGE`]. The adapter writes timestamps through the -/// mapping's CPU view; the firmware-shared GPU view is what the file layer -/// retains and what firmware-written timestamps will use later. -pub(crate) fn map_timestamp_buffer_in(uat: &mmu::Uat, mut bo: gem::ObjectRef, - range: Range) -> Result { - bo.map_range_into_range(uat.kernel_vm(), range, G18_TIMESTAMP_KERNEL_RANGE, - mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RW, false) -} - /// Adapter options fixed at registration. #[derive(Debug, Copy, Clone, Default)] pub(crate) struct G18DrmConfig { @@ -162,11 +149,12 @@ impl G18DrmGpu { // Module 1134 reached G18 through its `DrmGpu` trait (drm_gpu.rs) and // computed GET_PARAMS in file.rs from `DrmGpu::params`; here the same facts -// fill `drm_asahi_params_global` through `gpu::Gpu::get_params`. file.rs -// adds VM_STATUS (supports_vm_status), leaves out VM_STATUS_MIRROR -// (supports_vm_status_mirror is false, as in 1134) and INDEPENDENT_QUEUES -// (no queue limits), and adds SOFT_FAULTS when fault_control is 0xb. +// fill `drm_asahi_params_global` through `gpu::Gpu::get_params`. The shared +// file layer adds VM_STATUS and VM_STATUS_MIRROR (supports_vm_status) and +// SOFT_FAULTS when fault_control is 0xb; there are no queue limits. impl gpu::Gpu for G18DrmGpu { + fn stats_snapshot(&self) -> Option> { None } + fn raw_stats_snapshot(&self) -> Option>> { None } fn as_any(&self) -> &dyn core::any::Any { self } fn arc_as_any(self: Arc) -> Arc { self } fn manager(&self) -> Option<&dyn gpu::GpuManager> { None } @@ -196,28 +184,15 @@ impl gpu::Gpu for G18DrmGpu { if self.config.admit_graphics_scratch { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH as u64; } - params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES as u64; + params.features |= uapi::DRM_ASAHI_FEATURE_G18_NATIVE_TILE_SAMPLES as u64; Ok(()) } fn supports_vm_status(&self) -> bool { true } - fn supports_vm_status_mirror(&self) -> bool { false } - /// Module 1134 validated and resolved every VM_BIND op before committing. - fn batched_vm_bind(&self) -> bool { true } - /// Module 1134 recorded `submission_error()` (EIO once crashed) on every VM_STATUS query. - fn submission_error(&self) -> Option { self.is_crashed().then_some(EIO) } - /// Module 1134 copied the command buffer before it parsed the syncs. - fn copy_commands_before_syncs(&self) -> bool { true } - /// Module 1134 refused HIGH and REALTIME queues with EINVAL. - fn allows_elevated_queue_priority(&self) -> bool { false } - /// Module 1134 had no GEM_MADVISE ioctl. - fn supports_gem_madvise(&self) -> bool { false } - /// Module 1134 refused bind flag bit 4 with EINVAL, and the T8152 UAT maps no - /// sparse tiles: a failed tile bind would clear the whole range, driver aliases - /// included. - fn supports_sparse_tiles(&self) -> bool { false } - /// `DRM_IOCTL_ASAHI_GET_TIME` scales CNTPCT_EL0 by this rate - /// (`file.rs:1225-1241`); report the counter's own frequency, as G17P - /// does (`g17_drm.rs:1209-1222`). J873 runs it at 24 MHz. + /// VM_BIND validates every op before committing any, and SUBMIT copies + /// the command buffer before it resolves the syncs (module 1134 did both). + fn tracked_submission(&self) -> bool { true } + /// `DRM_IOCTL_ASAHI_GET_TIME` scales CNTPCT_EL0 by this rate; report the + /// counter's own frequency, as G17P does. J873 runs it at 24 MHz. fn base_clock_hz(&self) -> u32 { counter_frequency() as u32 } /// 42-bit T8152 user space: user range UAT_PGSZ..USER_END with the dummy /// page at USER_END, as in module 1134. @@ -228,7 +203,7 @@ impl gpu::Gpu for G18DrmGpu { let vm = self.runtime.new_user_vm(self.ids.vm.next(), kernel_range)?; // Module 1134 VM_CREATE: a simplefb alias is T8140-only; any other // GPU refuses the VM when the diagnostic parameter is set. - if *crate::module_parameters::g17p_simplefb_iova.value() != 0 { + if crate::g18::params::g17p_simplefb_iova.value() != 0 { return Err(ENOTSUPP); } Ok(vm) @@ -236,7 +211,8 @@ impl gpu::Gpu for G18DrmGpu { fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, _ualloc_priv: Arc>, - priority: u32, usc_exec_base: u64) -> Result> { + priority: u32, usc_exec_base: u64, + _usage: Arc) -> Result> { if priority > 3 { return Err(EINVAL); } let window = g17_uapi::QueueUscWindow { base: usc_exec_base, user_start: m5_vm::DRIVER_BASE, user_end: USER_END, @@ -249,13 +225,15 @@ impl gpu::Gpu for G18DrmGpu { }; Ok(KBox::new(queue, GFP_KERNEL)?) } - /// K8. Mesa binds `128 * sizeof(struct agx_timestamps)` of a WRITEBACK - /// BO at screen creation and enables timer queries only if this works. + /// K8. Mesa binds `128 * sizeof(struct agx_timestamps)` of a WRITEBACK BO + /// at screen creation and enables timer queries only if this works. G18 + /// publishes timestamps from the CPU after retirement, so the buffer needs + /// no firmware mapping. fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) - -> Result { + -> Result { if self.is_crashed() { return Err(ENODEV); } if range.is_empty() { return Err(EINVAL); } - self.runtime.map_timestamp_buffer(bo, range) + timestamp::Buffer::cpu(bo, range) } } @@ -276,21 +254,11 @@ fn counter_ticks() -> u64 { ticks } -/// GET_TIME's conversion (`file.rs:1225-1241`): floor(ticks * 1e9 / f), the -/// same rational as its gcd-reduced form. Split at whole seconds so no step -/// overflows (the remainder term is below f * 1e9 < 2^64 for any u32 rate) -/// and no 128-bit division (`__udivti3`) reaches the module. -fn ticks_to_ns(ticks: u64, frequency: u64) -> u64 { - const NSEC_PER_SEC: u64 = 1_000_000_000; - (ticks / frequency).wrapping_mul(NSEC_PER_SEC) - .wrapping_add(ticks % frequency * NSEC_PER_SEC / frequency) -} - struct G18SubmissionFence; #[vtable] impl FenceOps for G18SubmissionFence { - fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } - fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("g18-submit") } + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c"aurora" } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c"g18-submit" } } /// Keep the original sync vector: creating a second output list would add an @@ -328,58 +296,18 @@ impl g17_uapi::GpuAddressSpace for UserAddressSpace<'_> { } } -/// One UAPI timestamp destination inside a bound timestamp buffer. -struct TimestampSlot { - mapping: Arc, - offset: usize, -} - -impl TimestampSlot { - /// Resolve before any GPU work, so a bad handle fails the submit cleanly. - fn resolve(objects: Pin<&xarray::XArray>>, - timestamp: g17_uapi::UapiTimestamp) -> Result> { - if timestamp.handle == 0 { - return if timestamp.offset == 0 { Ok(None) } else { Err(EINVAL) }; - } - let guard = objects.lock(); - let object = guard.get(timestamp.handle.try_into()?).ok_or(ENOENT)?.clone(); - core::mem::drop(guard); - match object { - file::Object::TimestampBuffer(mapping) => { - let offset: usize = timestamp.offset.try_into()?; - if offset.checked_add(8).ok_or(EINVAL)? > mapping.size() { - return Err(ERANGE); - } - Ok(Some(Self { mapping, offset })) - } - } - } - - /// Publish one timestamp in the UAPI unit, GET_TIME nanoseconds. - fn write(&self, ticks: u64, frequency: u64) -> Result { - let value = ticks_to_ns(ticks, frequency); - self.mapping.with_cpu_bytes(|bytes| { - let end = self.offset.checked_add(8).ok_or(EINVAL)?; - bytes.get_mut(self.offset..end).ok_or(EINVAL)? - .copy_from_slice(&value.to_le_bytes()); - Ok(()) - }).inspect_err(|error| pr_err!( - "G18 timestamp not published: {:?} offset={:#x} mapping={:#x}\n", - error, self.offset, self.mapping.size()))?; - fence(Ordering::Release); - Ok(()) - } -} - -/// Write the requested stage timestamps after retirement. Without job stage -/// ticks, the CPU bracket `[t0, t1]` around the synchronous job bounds every -/// stage: starts get t0, ends get t1. -fn publish_timestamps(slots: &[Option], stage_ticks: &[u64]) -> Result { - let frequency = counter_frequency(); - for (slot, &ticks) in slots.iter().zip(stage_ticks) { - if let Some(slot) = slot { slot.write(ticks, frequency)?; } +/// A parsed timestamp pair in the UAPI's own type, for the shared resolver. +fn uapi_timestamps(timestamps: g17_uapi::UapiTimestamps) -> uapi::drm_asahi_timestamps { + uapi::drm_asahi_timestamps { + start: uapi::drm_asahi_timestamp { + handle: timestamps.start.handle, + offset: timestamps.start.offset, + }, + end: uapi::drm_asahi_timestamp { + handle: timestamps.end.handle, + offset: timestamps.end.offset, + }, } - Ok(()) } struct G18Queue { @@ -392,7 +320,8 @@ struct G18Queue { } impl queue::Queue for G18Queue { - fn submit(&mut self, _submission_id: u64, syncs: KVec, + fn submit(&mut self, _reg_data: &AsahiRegData<'_>, _submission_id: u64, + syncs: KVec, in_sync_count: usize, cmdbuf_raw: &[u8], objects: Pin<&xarray::XArray>>) -> Result { if in_sync_count > syncs.len() { return Err(EINVAL); } @@ -447,18 +376,21 @@ impl G18Queue { // any earlier submission) has completed when its waiter begins. for command in admitted { if self.runtime.is_crashed() { return Err(ENODEV); } + // Without job stage ticks, the CPU bracket `[t0, t1]` around the + // synchronous job bounds every stage: starts get t0, ends get t1. let t0 = counter_ticks(); match command { Admitted::Render(render, slots) => { let retired = self.runtime.submit_render(&self.vm, &render)?; let t1 = counter_ticks(); - publish_timestamps(&slots, &retired.stage_ticks.unwrap_or([t0, t1, t0, t1]))?; + slots.publish(retired.stage_ticks.unwrap_or([t0, t1, t0, t1]), + counter_frequency()); } Admitted::Compute(command, slots) => { let retired = self.runtime.submit_compute(&self.vm, &command)?; let t1 = counter_ticks(); let [start, _, _, end] = retired.stage_ticks.unwrap_or([t0, t1, t0, t1]); - publish_timestamps(&slots, &[start, end])?; + slots.publish([start, end, 0, 0], counter_frequency()); } } } @@ -519,12 +451,10 @@ impl G18Queue { pr_err!("G18 render admission rejected: job-owned fields set\n"); return Err(EINVAL); } - let slots = [ - TimestampSlot::resolve(objects, payload.vertex_timestamps.start)?, - TimestampSlot::resolve(objects, payload.vertex_timestamps.end)?, - TimestampSlot::resolve(objects, payload.fragment_timestamps.start)?, - TimestampSlot::resolve(objects, payload.fragment_timestamps.end)?, - ]; + let slots = timestamp::Destinations::resolve_unaligned(objects, &[ + uapi_timestamps(payload.vertex_timestamps), + uapi_timestamps(payload.fragment_timestamps), + ])?; Ok(Admitted::Render(render, slots)) } @@ -629,18 +559,16 @@ impl G18Queue { })?; translated.g18_texture_state = payload.flags & G18_COMPUTE_TEXTURE_STATE != 0; translated.g18_resource_descriptor = resource_descriptor; - let slots = [ - TimestampSlot::resolve(objects, payload.timestamps.start)?, - TimestampSlot::resolve(objects, payload.timestamps.end)?, - ]; + let slots = timestamp::Destinations::resolve_unaligned(objects, + &[uapi_timestamps(payload.timestamps)])?; Ok(Admitted::Compute(KBox::new(translated, GFP_KERNEL)?, slots)) } } /// An admitted command, heap-held so the submit frame stays small. enum Admitted { - Render(KBox, [Option; 4]), - Compute(KBox, [Option; 2]), + Render(KBox, timestamp::Destinations), + Compute(KBox, timestamp::Destinations), } /// Log a render refusal with the fields that decided it and map it to the @@ -653,57 +581,3 @@ fn refuse_render(payload: &g17_uapi::UapiRenderCommand, refusal: G18ApplicationR payload.stencil.base, payload.stencil_clear, payload.sampler_count); if refusal.is_invalid() { EINVAL } else { ENOTSUPP } } - -static FOREIGN_IMPORT_REPORTED: AtomicBool = AtomicBool::new(false); - -/// Parent `struct device` of the G18 DRM device, set before it registers. -static G18_PARENT: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); - -/// Record the platform device of the G18 DRM device, so that -/// [`asahi_gem_prime_import`] applies module 1134's import rule to it alone. -pub(crate) fn note_g18_parent(parent: *mut bindings::device) { - G18_PARENT.store(parent, Ordering::Release); -} - -/// `drm_driver.gem_prime_import` for asahi. A device other than G18 gets the -/// DRM core's default import, exactly what the core does without this hook -/// (t6050-gpu has none). The G18 device re-imports only dma-bufs that it -/// exported. Anything else is refused with EINVAL, the DRM -/// core's answer for a driver without import support -/// (`drm_prime.c:987-988`). -/// -/// Without this hook the core's default import builds a bare C -/// `drm_gem_shmem_object` (no `gem_create_object`, `shmem.rs:335-339`), -/// and every asahi ioctl that looks a handle up reads it as an -/// `Object` (`file.rs:989`, `shmem.rs:320-327`). -/// -/// # Safety -/// -/// The DRM core calls this with a valid `drm_device` and a dma-buf it holds -/// a reference on, for the duration of the call. -pub(crate) unsafe extern "C" fn asahi_gem_prime_import( - dev: *mut bindings::drm_device, - dma_buf: *mut bindings::dma_buf, -) -> *mut bindings::drm_gem_object { - // SAFETY: `dev` is a valid drm_device (caller contract); its parent is - // set at allocation and does not change while the device exists. - let parent = unsafe { (*dev).dev }; - if parent.is_null() || parent != G18_PARENT.load(Ordering::Acquire) { - // SAFETY: caller contract; this is the call the DRM core makes - // itself when a driver has no gem_prime_import hook. - return unsafe { bindings::drm_gem_prime_import(dev, dma_buf) }; - } - // SAFETY: both pointers are valid (caller contract). The helper compares - // `dma_buf->ops` with DRM's own PRIME ops before it reads `priv` as a GEM - // object (`drm_prime.c:945-951`). - if unsafe { bindings::drm_gem_is_prime_exported_dma_buf(dev, dma_buf) } { - // SAFETY: for a dma-buf this device exported, the helper only takes - // a reference on the existing GEM object (`drm_prime.c:977-983`), - // which is an `Object` created by this driver. - return unsafe { bindings::drm_gem_prime_import(dev, dma_buf) }; - } - if !FOREIGN_IMPORT_REPORTED.swap(true, Ordering::Relaxed) { - pr_warn!("asahi: refusing a dma-buf another driver exported; only asahi's own buffers can be imported\n"); - } - EINVAL.to_ptr() -} diff --git a/drivers/gpu/drm/asahi/g18/fw/initdata.rs b/drivers/gpu/drm/asahi/g18/fw/initdata.rs index a1e5e06bf25b8..ae404f123b4fb 100644 --- a/drivers/gpu/drm/asahi/g18/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g18/fw/initdata.rs @@ -2718,7 +2718,7 @@ pub(crate) fn encode_main_config_for_profile( // records are plain ASCII. Shipping macOS provisions the whole facility // and then throws the messages away (drainFirmwareLogRing is `bti c; ret`), // which is why the captured value is zero. - if *crate::module_parameters::g17p_fw_log.value() != 0 { + if crate::g18::params::g17p_fw_log.value() != 0 { put_u32(out, MAIN_FIRMWARE_LOG_ENABLE, 1); } let bundle = shared.hw_data_bundle; diff --git a/drivers/gpu/drm/asahi/g18/fw/render_params.rs b/drivers/gpu/drm/asahi/g18/fw/render_params.rs index 893d174936fbc..65bd30235ecf4 100644 --- a/drivers/gpu/drm/asahi/g18/fw/render_params.rs +++ b/drivers/gpu/drm/asahi/g18/fw/render_params.rs @@ -1516,7 +1516,7 @@ fn apply_g17p_fragment_oracle(raw: &mut [u8]) { /// FRG captures hold 0. Diagnostic parameter `g17p_frg_hwdsid_clear`. #[cfg(not(test))] fn g17p_frg_hwdsid_mode() -> u8 { - u8::from(*crate::module_parameters::g17p_frg_hwdsid_clear.value() == 0 + u8::from(crate::g18::params::g17p_frg_hwdsid_clear.value() == 0 && !crate::g18::compat::timing_buffer::diag_knob(crate::g18::compat::timing_buffer::DIAG_FRG_222D_CLEAR)) } diff --git a/drivers/gpu/drm/asahi/g18/hw.rs b/drivers/gpu/drm/asahi/g18/hw.rs index efc30b8c15949..cc6728b8583c2 100644 --- a/drivers/gpu/drm/asahi/g18/hw.rs +++ b/drivers/gpu/drm/asahi/g18/hw.rs @@ -1,10 +1,10 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! T8152 SoC configuration and the native G18G registry facts. +//! G18G revision and identity types, and the T8152 SoC entry. //! //! Items copied verbatim from the J873 module-1134 sources (provenance in //! the port's extraction ledger); only module paths -//! are rewritten. +//! are rewritten. The SoC entry is in `hw::t8152`. use kernel::prelude::*; pub(crate) use crate::g18::identity::{GpuGen, GpuHalGeneration, GpuVariant}; @@ -80,88 +80,5 @@ pub(crate) struct GpuIdConfig { pub(crate) gpc_perf_state_control: u32, } -/// Identification-only configuration for one AGX3 SoC. -#[derive(Debug)] -pub(crate) struct SocConfig { - /// Chip ID in hex format (e.g. 0x6050 for t6050). - pub(crate) chip_id: u32, - /// Decoded design generation for this chip. - pub(crate) gpu_gen: GpuGen, - /// Decoded variant for this chip. - pub(crate) gpu_variant: GpuVariant, - /// Hardware family byte expected at `0xd04000[31:24]`. - pub(crate) hw_family: u8, - /// Hardware variant byte expected at `0xd04000[23:16]`. - pub(crate) hw_variant: u8, - /// Die count expected at `0xd04010[19:16]`. - pub(crate) num_dies: u32, - /// Input-address bits per UAT root (42 on every AGX3 part). - pub(crate) uat_ias: u8, - /// Output address space (42-bit on every AGX3 part). - pub(crate) uat_oas: u32, -} - -const fn agx3( - chip_id: u32, - gpu_gen: GpuGen, - gpu_variant: GpuVariant, - hw_family: u8, - hw_variant: u8, - num_dies: u32, -) -> SocConfig { - SocConfig { - chip_id, - gpu_gen, - gpu_variant, - hw_family, - hw_variant, - num_dies, - uat_ias: 42, - uat_oas: 42, - } -} - -// G18 (M6 family). Family byte 0xc, variant 2: 26A428 -// `AGX·PI_400·G·A0·Accelerator::readChipInfo` @ 0xfffffe0008c3d3a0 compares -// the family byte with 0xc @ 0xfffffe0008c3d49c and gives variant 2 the -// 12-core G configuration @ 0xfffffe0008c3d4bc..0x4c8. Native J873 registry -// (AGXAcceleratorG18G): gpu_gen=18, gpu_var=G, usc_gen=3, is_sksm=1, -// num_cores=12, num_gps=6, num_mgpus=2, core_mask_list=(63,63), -// num_frags=12, kickid_qid_shift=40, kickid_qid_mask=127. The live ID -// registers have not been read yet (the GPU PSD reads idle at cold boot), so -// the T8152 probe identifies the part by chip id and compatible, and only -// logs a mismatching register decode. The ADT uat-vaddr-size is 43 on both -// J813 and J873: 42 bits per root plus the TTBR-select bit, as on T8142. -/// Apple M6 (G18G, J873 Mac mini). -pub(crate) const T8152: SocConfig = agx3(0x8152, GpuGen::G18, GpuVariant::G, 0xc, 2, 1); - -/// Native J873 `AGXAcceleratorG18G` registry facts (macOS 26A428 ioreg). -/// -/// These are identity facts, not firmware tuning: they size the core-mask -/// decode and cross-check the live ID registers once the GPU is powered. -#[allow(dead_code)] -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) struct G18RegistryFacts { - pub(crate) num_cores: u32, - pub(crate) num_gps: u32, - pub(crate) num_mgpus: u32, - /// Per-mGPU (cluster) core masks, `core_mask_list`. - pub(crate) core_mask_list: [u32; 2], - pub(crate) num_frags: u32, - pub(crate) usc_generation: u32, - pub(crate) kickid_qid_shift: u32, - pub(crate) kickid_qid_mask: u32, -} - -/// J873 registry facts (PROVEN from the native ioreg). -#[allow(dead_code)] -pub(crate) const T8152_REGISTRY: G18RegistryFacts = G18RegistryFacts { - num_cores: 12, - num_gps: 6, - num_mgpus: 2, - core_mask_list: [0x3f, 0x3f], - num_frags: 12, - usc_generation: 3, - kickid_qid_shift: 40, - kickid_qid_mask: 127, -}; +// The T8152 SoC entry and its registry facts live with the other SoCs. +pub(crate) use crate::hw::t8152::{G18RegistryFacts, SocConfig, T8152, T8152_REGISTRY}; diff --git a/drivers/gpu/drm/asahi/g18/initdata/application.rs b/drivers/gpu/drm/asahi/g18/initdata/application.rs index 9f0d800f76aee..9db38c60458a9 100644 --- a/drivers/gpu/drm/asahi/g18/initdata/application.rs +++ b/drivers/gpu/drm/asahi/g18/initdata/application.rs @@ -8,7 +8,7 @@ impl G18FirmwareMemory { self.uma_pool.application_aliases(&self.uat,vm) } - pub(crate) fn audit_application(&mut self, dev: &AsahiDevice, job: &mut ApplicationJob) -> Result { + pub(crate) fn audit_application(&mut self, dev: &AsahiDrmDevice, job: &mut ApplicationJob) -> Result { let frame=job.frame();let context=u32::from(job.job.context_id()); let pb=job.job.parameter_buffer().verify_resident(&mut self.pbdesc,context,frame.generation)?; let uma=self.uma_pool.verify_application_descriptor(&mut self.uma,j873_sma_device_words(2,6)?,frame.generation==1)?; diff --git a/drivers/gpu/drm/asahi/g18/initdata/application_compute.rs b/drivers/gpu/drm/asahi/g18/initdata/application_compute.rs index 0875b61333836..7dba393568828 100644 --- a/drivers/gpu/drm/asahi/g18/initdata/application_compute.rs +++ b/drivers/gpu/drm/asahi/g18/initdata/application_compute.rs @@ -7,7 +7,7 @@ use core::sync::atomic::Ordering; // Application-compute-only knobs K1-K5 (compute diagnostics) and // g18_app_d62a (case 10). Unlike // kernel::module! parameters (perm 0, set once at load), these are 0644 with -// the C uint ops, so /sys/module/asahi/parameters/g18_app_* may be changed +// the C uint ops, so /sys/module/aurora/parameters/g18_app_* may be changed // between submissions. prepare_application_compute reads each exactly once // per submission; nothing else reads them, so graphics and the stage8 fixture // are unaffected. Registration otherwise follows g18_render_policy.rs. @@ -22,7 +22,7 @@ macro_rules! app_knob { static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { kernel::c_str!($name) - } else { kernel::c_str!(concat!("asahi.", $name)) }), + } else { kernel::c_str!(concat!("aurora.", $name)) }), // Module lifetime and parameter serialization are managed by the // kernel, identically to the kernel::module! registration. A raw // address avoids a reference to the mutable extern static. @@ -130,7 +130,7 @@ impl ApplicationComputeKnobs { } fn apply_plan(&mut self, plan: crate::g18::uapi::compute_plan::G18ComputePlan, - dev: &AsahiDevice) -> Result { + dev: &AsahiDrmDevice) -> Result { // Fixed knobs remain available for legacy diagnostics. Never combine // them with shader-derived ownership or silently ignore an override. if self.sma_pair != 0x6_0010 || self.sma_units != 0 { @@ -145,7 +145,7 @@ impl ApplicationComputeKnobs { /// Refuse, before any owner changes, values that would make the command /// inconsistent. Nothing is published for a refused submission. - fn validate(&self, dev: &AsahiDevice) -> Result { + fn validate(&self, dev: &AsahiDrmDevice) -> Result { let refusal = if self.a731 > 1 { Some(("g18_app_a731", self.a731, "admitted 0 (fixture) or 1 (native)")) } else if !SMA_PAIRS.contains(&self.pair()) { @@ -199,7 +199,7 @@ pub(super) struct ApplicationCompute { } impl G18FirmwareMemory { - pub(crate) fn prepare_application_compute(&mut self, dev: &AsahiDevice, + pub(crate) fn prepare_application_compute(&mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm, command: &TranslatedComputeCommand) -> Result<(G18PreparedComputeCommand, G18UmaPoolBinding, u32, ApplicationComputeKnobs)> { if command.sampler_heap != 0 || command.sampler_count != 0 @@ -307,7 +307,7 @@ impl G18FirmwareMemory { Ok(prepared) } - fn prepare_application_compute_attempt(&mut self, dev: &AsahiDevice, + fn prepare_application_compute_attempt(&mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm, command: &TranslatedComputeCommand, mut knobs: ApplicationComputeKnobs, fail_checkpoint: u32, replacement: Option) @@ -475,7 +475,7 @@ impl G18FirmwareMemory { /// Called only after the session checked the old firmware binding and the /// retired scheduler. Both views remain owned if any later step fails. - pub(crate) fn install_application_compute_pool(&mut self, dev: &AsahiDevice) -> Result { + pub(crate) fn install_application_compute_pool(&mut self, dev: &AsahiDrmDevice) -> Result { let job = &mut **self.application_compute.as_mut().ok_or(EIO)?; if let Some(previous) = job.previous_pool.as_mut() { let old = job.previous_binding.ok_or(EIO)?; diff --git a/drivers/gpu/drm/asahi/g18/initdata/mod.rs b/drivers/gpu/drm/asahi/g18/initdata/mod.rs index 3d3c195002a24..05c0d1c17633e 100644 --- a/drivers/gpu/drm/asahi/g18/initdata/mod.rs +++ b/drivers/gpu/drm/asahi/g18/initdata/mod.rs @@ -45,7 +45,7 @@ use crate::g18::buffer::pool::{G18UmaPoolBinding, G18UmaPoolMemory}; use crate::g18::queue::scheduler::G18SchedulerMemory; use crate::g18::runtime::compute::{G18ComputeMemory, G18PreparedComputeCommand}; use crate::g18::runtime::compute_payload::G18ComputePayloadMemory; -use crate::{driver::AsahiDevice, g18::fw::initdata as init, mmu}; +use crate::{driver::AsahiDrmDevice, g18::fw::initdata as init, mmu}; use core::sync::atomic::{fence, Ordering}; use kernel::prelude::*; use kernel::sync::Arc; @@ -218,7 +218,7 @@ impl G18FirmwareMemory { /// until both processors stop, including when this construction fails. #[inline(never)] pub(crate) fn new_with_uat( - dev: &AsahiDevice, + dev: &AsahiDrmDevice, uat: Arc, boot_timestamp: u64, render: bool, @@ -226,7 +226,7 @@ impl G18FirmwareMemory { if !uat.is_t8152_geometry() { return Err(ENODEV); } - match *crate::module_parameters::g18_compute_owner_test.value() { + match crate::g18::params::g18_compute_owner_test.value() { 0 => {}, 1 => G18UmaPoolMemory::verify_shared_compute_owner(dev, &uat)?, _ => return Err(EINVAL), @@ -247,7 +247,7 @@ impl G18FirmwareMemory { }; // Page-table writes only. No register contents are accessed. registers.push( - uat.kernel_vm().map_io(virtual_address, physical.try_into()?, span.try_into()?, + uat.kernel_vm().map_io(dev.as_ref().as_ref(), virtual_address, physical.try_into()?, span.try_into()?, protection)?, GFP_KERNEL, )?; @@ -418,7 +418,7 @@ impl G18FirmwareMemory { pub(crate) fn uma_pool(&self) -> &G18UmaPoolMemory { &self.uma_pool } - pub(crate) fn prepare_render(&mut self, dev: &AsahiDevice, + pub(crate) fn prepare_render(&mut self, dev: &AsahiDrmDevice, job: &mut crate::g18::runtime::graphics_job::G18GraphicsJob) -> Result { self.uma_pool.prepare_render_backing(dev, &self.uat, job.vm())?; let pool = self.uma_pool.prepare_binding()?; @@ -428,7 +428,7 @@ impl G18FirmwareMemory { Ok(pool) } - pub(crate) fn audit_render(&mut self, dev: &AsahiDevice, + pub(crate) fn audit_render(&mut self, dev: &AsahiDrmDevice, job: &mut crate::g18::runtime::graphics_job::G18GraphicsJob) -> Result { let pb = job.parameter_buffer().verify_before_kick(&mut self.pbdesc)?; if pb.counter != 1 { return Err(EIO); } @@ -484,7 +484,7 @@ impl G18FirmwareMemory { /// 0x12cdc..0x12de8 skips tag15 hardware queue setup when it is nonzero. /// Native macOS uses host queue programming. This first Linux submission /// sends tag15/tag14 and performs no host KSMFE MMIO, so choose one owner. - pub(crate) fn use_firmware_queue_programming(&mut self, dev: &AsahiDevice) -> Result { + pub(crate) fn use_firmware_queue_programming(&mut self, dev: &AsahiDrmDevice) -> Result { let policy = SHARED_BASE + 0x2724; let trace = PRIVATE_BASE + init::G18G_26A428_PRIVATE_STATUS_A[0] as u64; let progress = REGION_C_BASE + 0x9ac; @@ -499,7 +499,7 @@ impl G18FirmwareMemory { // exact branches. Opt-in graphics diagnosis additionally enables // bit9 (global 0x1093e5 bit1), the native stall diagnostic category. // audit-g18-graphics-stall.py executes that original predicate. - let graphics_stall_trace = *crate::module_parameters::g18_graphics_stall_trace.value() == 1; + let graphics_stall_trace = crate::g18::params::g18_graphics_stall_trace.value() == 1; let trace_mask = if graphics_stall_trace { 0x203 } else { 3 }; self.private.write_u32(trace, trace_mask)?; // b000 refresh 0x352c/0x3688 copies region-C+0x9ac into @@ -524,7 +524,7 @@ impl G18FirmwareMemory { /// Retain a private shader/input/output VM. This does not select a process /// context or make its roots visible to firmware, and never queues work. - pub(crate) fn prepare_compute_payload(&mut self, dev: &AsahiDevice) + pub(crate) fn prepare_compute_payload(&mut self, dev: &AsahiDrmDevice) -> Result<&mut G18ComputePayloadMemory> { if self.compute_payload.is_some() { return Err(EBUSY); } self.compute_payload = Some(G18ComputePayloadMemory::load(dev, &self.uat)?); @@ -533,14 +533,14 @@ impl G18FirmwareMemory { /// Connect the retained payload VM to the command/channel owner. Context /// roots remain unpublished until the complete command has been prepared. - pub(crate) fn bind_compute_process(&mut self, dev: &AsahiDevice) -> Result { + pub(crate) fn bind_compute_process(&mut self, dev: &AsahiDrmDevice) -> Result { let payload = self.compute_payload.as_ref().ok_or(EINVAL)?; self.compute.bind_process(dev, &self.uat, payload.vm()) } /// Allocate the captured job's scratch pool while its state is unpublished. /// The returned register inputs are valid only with this retained owner. - pub(crate) fn prepare_compute_scratch(&mut self, dev: &AsahiDevice) + pub(crate) fn prepare_compute_scratch(&mut self, dev: &AsahiDrmDevice) -> Result { let plan = self.compute_payload.as_ref().ok_or(EINVAL)?.scratch_plan()?; self.uma_pool.prepare_compute_backing(dev, &self.uat, DYNAMIC_START..DYNAMIC_END, plan)?; @@ -551,7 +551,7 @@ impl G18FirmwareMemory { /// Every step is still unpublished: the caller must subsequently bind the /// pool, publish the process roots, encode the owned SKSM entry and only /// then publish a kick. Errors leave any partial owners retained here. - pub(crate) fn prepare_addition(&mut self, dev: &AsahiDevice) + pub(crate) fn prepare_addition(&mut self, dev: &AsahiDrmDevice) -> Result<(G18PreparedComputeCommand, G18UmaPoolBinding)> { use crate::g18::fw::compute::G18ComputeCompletionState; use crate::g18::queue::probe::QUEUE_ID; @@ -835,7 +835,7 @@ impl G18FirmwareMemory { /// rings. Records are copied before their slot is released; each drain is /// bounded. Returns whether an event record of the J813 recovery type was /// seen. This is the milestone-1 heartbeat (plan step 1.8, stage 4). - pub(crate) fn drain_reports(&mut self, dev: &AsahiDevice) -> Result { + pub(crate) fn drain_reports(&mut self, dev: &AsahiDrmDevice) -> Result { let mut recovery = false; for (role, status_a) in init::G18G_26A428_PRIVATE_STATUS_A.into_iter().enumerate() { for channel in 0..2 { @@ -914,7 +914,7 @@ impl G18FirmwareMemory { /// Export after the transaction and stop requests, never while polling a /// GPU job. Completeness covers consumed records, not a possibly live producer. - pub(crate) fn log_trace_journal(&self, dev: &AsahiDevice) { + pub(crate) fn log_trace_journal(&self, dev: &AsahiDrmDevice) { for role in 0..2 { let journal = &self.trace_journal[role]; let consumed = self.report_counts[role][1]; @@ -937,7 +937,7 @@ impl G18FirmwareMemory { } /// One-line construction summary for the bring-up log. - pub(crate) fn log_summary(&self, dev: &AsahiDevice) { + pub(crate) fn log_summary(&self, dev: &AsahiDrmDevice) { dev_info!(dev.as_ref(), "G18G initdata: roots {:#x}/{:#x} version {:#x}; shared {:#x}+{:#x}; private {:#x}+{:#x}; {} register windows; {} unproven fields\n", self.root(false), self.root(true), init::G18G_26A428_INITDATA_VERSION_WORD, diff --git a/drivers/gpu/drm/asahi/g18/mod.rs b/drivers/gpu/drm/asahi/g18/mod.rs index 14317928821e2..23e6f2a76a62c 100644 --- a/drivers/gpu/drm/asahi/g18/mod.rs +++ b/drivers/gpu/drm/asahi/g18/mod.rs @@ -18,6 +18,7 @@ pub(crate) mod fixture; pub(crate) mod fw; #[allow(dead_code)] pub(crate) mod initdata; +pub(crate) mod params; pub(crate) mod queue; pub(crate) mod runtime; pub(crate) mod uapi; diff --git a/drivers/gpu/drm/asahi/g18/params.rs b/drivers/gpu/drm/asahi/g18/params.rs new file mode 100644 index 0000000000000..555c9959ead2f --- /dev/null +++ b/drivers/gpu/drm/asahi/g18/params.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Module parameters of the G18 backend. +//! +//! They are registered exactly as `kernel::module!` registers the driver's +//! own parameters (a `__param` entry with `perm` 0 and the `parmtype`/`parm` +//! modinfo strings, prefixed by `aurora.` when built in), but only when the +//! backend is built, so other configurations keep their parameter set. The +//! names, types, defaults and descriptions are those of the J873 kernel. + +#![allow(non_upper_case_globals)] + +use kernel::module_param::{ModuleParamAccess, PARAM_OPS_U32, PARAM_OPS_U64}; + +/// Copy `bytes` into an array of its exact length, for a `.modinfo` string. +const fn modinfo(bytes: &[u8]) -> [u8; N] { + let mut out = [0; N]; + let mut i = 0; + while i < N { + out[i] = bytes[i]; + i += 1; + } + out +} + +macro_rules! g18_param { + ($name:ident: $ty:ident = $default:expr, $ops:ident, $description:literal) => { + pub(crate) static $name: ModuleParamAccess<$ty> = ModuleParamAccess::new($default); + + const _: () = { + #[link_section = "__param"] + #[used(compiler)] + static PARAM: kernel::module_param::KernelParam = + kernel::module_param::KernelParam::new(kernel::bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + kernel::c_str!(stringify!($name)) + } else { + kernel::c_str!(concat!("aurora.", stringify!($name))) + }), + // `__this_module` is constructed by the kernel at load time and is not + // freed until the module is unloaded. A raw address avoids a reference to + // the mutable extern static. + #[cfg(MODULE)] + mod_: &raw mut kernel::bindings::__this_module, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&$ops), + perm: 0, // Will not appear in sysfs + level: -1, + flags: 0, + __bindgen_anon_1: kernel::bindings::kernel_param__bindgen_ty_1 { + arg: $name.as_void_ptr(), + }, + }); + + #[cfg(MODULE)] + const TYPE: &[u8] = + concat!("parmtype=", stringify!($name), ":", stringify!($ty), "\0").as_bytes(); + #[cfg(not(MODULE))] + const TYPE: &[u8] = + concat!("aurora.parmtype=", stringify!($name), ":", stringify!($ty), "\0") + .as_bytes(); + #[cfg_attr(not(target_os = "macos"), link_section = ".modinfo")] + #[used(compiler)] + static TYPE_INFO: [u8; TYPE.len()] = modinfo(TYPE); + + #[cfg(MODULE)] + const TEXT: &[u8] = + concat!("parm=", stringify!($name), ":", $description, "\0").as_bytes(); + #[cfg(not(MODULE))] + const TEXT: &[u8] = + concat!("aurora.parm=", stringify!($name), ":", $description, "\0").as_bytes(); + #[cfg_attr(not(target_os = "macos"), link_section = ".modinfo")] + #[used(compiler)] + static TEXT_INFO: [u8; TEXT.len()] = modinfo(TEXT); + }; + }; +} + +g18_param!(g18_initdata_test: u32 = 10, PARAM_OPS_U32, + "T8152/J873 G18G bring-up stage. 0 refuses before MMIO/DMA. 1=read-only powered-resource probe. 2=RTKit. 3=initdata ACKs. 4=openings, idle policy, 60 s observation. 5=empty queue. 6=queue plus scheduler binding. 7=owned empty UMA pool binding. 8=compute diagnostic. 9=H1 render replay. 10=firmware plus application DRM render node. Stages 5/6/7 then observe without GPU jobs. Firmware stages require prior power, loader receipt and apple_mailbox.ascwrap_v8_cpu_control=1; all owners remain retained until reboot. 90=SGX power-on; 92=GFX1 power-on; 91=GFX power-on; 93=idle requests. Power stages are never implied. Mutating stages require user-authorized bring-up. Other values refused"); +g18_param!(g18_compute_variant: u32 = 0, PARAM_OPS_U32, + "Stage 8 compute diagnostic. 0=captured compute-add. 1=empty stream: captured barrier and terminate replace the launch; all output must remain poisoned. 2=64x64 compute triangle using a separately verified private shader patch; no TA/3D or Mesa. Variants 1 and 2 require g18_initdata_test=8. Other values are refused"); +g18_param!(g18_compute_owner_test: u32 = 0, PARAM_OPS_U32, + "J873 allocation-only E2E before firmware publication: 1 verifies shared-compute backing, guarded directory growth, allocator rollback and process aliases. Never submits GPU work. Other nonzero values refused"); +g18_param!(g18_render_variant: u32 = 0, PARAM_OPS_U32, + "Stage 9 H1 diagnostic: 0=colour, 2=terminated VDM plus background clear. Other variants refused before MMIO. One asynchronous worker pass, captured/M5 H1 policy, all owners retained; see the H1 render-runtime notes"); +g18_param!(g18_render_profile: u32 = 1, PARAM_OPS_U32, + "Stage 9 G18GFX01 v2 profile: 1=colour-a, 2=colour-b, 4=fragment-witness (colour only). Gears profile3 is offline only and refused by H1. Every other stage, including the default stage10, requires 1"); +g18_param!(g18_admit_compute: u32 = 1, PARAM_OPS_U32, + "G18 DRM adapter: 1 admits single-dispatch compute commands (K1). 0 refuses them"); +g18_param!(g18_admit_msaa4: u32 = 1, PARAM_OPS_U32, + "G18 DRM adapter (g18_initdata_test=10 only): 1 (default) admits exactly the native 4x MSAA render (4 samples, 32x16 utiles, sample positions 0xeaa26e26, 8-byte samples, one layer, no depth/stencil) with its native 4x command fields; Mesa needs AGX_G18_MSAA4=1 to submit it. 0 refuses every multisampled render"); +g18_param!(g18_admit_graphics_scratch: u32 = 1, PARAM_OPS_U32, + "G18 DRM adapter: 1 (default) advertises DRM_ASAHI_FEATURE_G18_GRAPHICS_SCRATCH and admits single-sample renders carrying the G18 render resource suffix; the kernel derives both TA/3D private-memory launch pairs and selects the native ideal or minimum result that fits the owned render pool. 0 refuses the suffix"); +g18_param!(g18_usc_statistics: u32 = 0, PARAM_OPS_U32, + "Stage 9 colour fixture only: 1 enables owned TA/3D SMAPP statistics pages. Raw memory statistics, not shader invocation counts. Default 0"); +g18_param!(g18_sma_minimum: u32 = 1, PARAM_OPS_U32, + "Stage9/profile4 or stage10/profile1, variant0: 1 probes fixed native minimum TA/fragment resource words with unchanged owned backing. Default 1"); +g18_param!(g18_graphics_stall_trace: u32 = 0, PARAM_OPS_U32, + "Stage9/profile4 or stage10/profile1, variant0: 1 enables firmware category9 diagnostics and the native TA mode2 progress sampler. No timeout changes. Default 0"); +g18_param!(g18_kick_log: u32 = 0, PARAM_OPS_U32, + "Stage10 application DRM log volume: 0 (default) omits the per-kick, frame-pass and per-job resource lines (G18_APP_KICK, G18_APP_FRAME_PASS, G18_APP_COMPUTE_KNOBS, G18_COMPUTE_*, G18_SHARED_COMPUTE_*, G18 UMA/compute-context lines, G18_POOL_DESCRIPTOR_CURSOR, role-0 type-1 G18_EVENT) once G18_DRM_READY is logged; 1 prints them as before. Startup, fault, recovery, refusal, wait, stall, admission and sampled summary lines are always printed"); +g18_param!(g18_vdm_high_alias: u32 = 0, PARAM_OPS_U32, + "Stage9/profile4/variant0 with minimum SMA and stall tracing: 1 fetches unchanged native VDM bytes through the Gallium-address alias. Diagnostic only. Default 0"); +g18_param!(g18_ppp_high_alias: u32 = 0, PARAM_OPS_U32, + "With the stage9 high-VDM control: 1 aliases unchanged native PPP pages into the Gallium range and relocates only their VDM pointers. Diagnostic only. Default 0"); +g18_param!(g18_sma_ideal: u32 = 0, PARAM_OPS_U32, + "Stage9 variant0 profile4 only: 1 probes native ideal graphics resource policy with 40MiB backing and precommitted directory growth. Default 0"); +g18_param!(g18_native_barrier: u32 = 0, PARAM_OPS_U32, + "Stage9 variant0 profile4 with ideal SMA only: 1 probes the native same-group TA-to-3D barrier kind3. Default 0"); +g18_param!(g17p_simplefb_iova: u64 = 0, PARAM_OPS_U64, + "Diagnostic simplefb GPU address override; 0 disables it"); +g18_param!(g17p_fw_log: u32 = 0, PARAM_OPS_U32, + "Enable the GPU firmware log ring"); +g18_param!(g17p_frg_hwdsid_clear: u32 = 0, PARAM_OPS_U32, + "Diagnostic: 1 writes FRG descriptor +0x222d as 0 (native light captures); 0 preserves 1"); +g18_param!(g17p_trace_depth: u32 = 0, PARAM_OPS_U32, + "Trace the first N compute submissions per queue; 0 disables success-path tracing"); +g18_param!(g17p_completion_trace: u32 = 0, PARAM_OPS_U32, + "Completion logging level: 0 off, 1 summary, 2 detailed"); diff --git a/drivers/gpu/drm/asahi/g18/probe.rs b/drivers/gpu/drm/asahi/g18/probe.rs index 9bae60ce5a16d..42c3988597e34 100644 --- a/drivers/gpu/drm/asahi/g18/probe.rs +++ b/drivers/gpu/drm/asahi/g18/probe.rs @@ -1,20 +1,18 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! T8152 probe: parameter gates, device preparation and the stage-10 DRM path. +//! T8152 probe: the G18 platform driver, its parameter gates, device +//! preparation and the stage-10 DRM path. //! //! Items copied from the J873 module-1134 driver.rs (provenance in -//! the port's extraction ledger), then adapted to -//! this driver's core: the DRM device is allocated with -//! `Device::new_uninit` and its data published with `init_data` (module 1134 -//! used a no-op initializer the release path skipped), `AsahiData` is this -//! driver's (no G17 work items, no SGX resources) and the G18 adapter is a -//! `gpu::Gpu`. The parameter gates and every log line are unchanged. +//! the port's extraction ledger), then adapted to this driver's core: G18 is +//! its own platform driver, like G16; the shared probe body +//! ([`AsahiPlatformDriver::probe_g18`]) owns the DRM device and its +//! registration, and the G18 adapter is a `gpu::Gpu`. The parameter gates and +//! every log line are unchanged. use kernel::{ - c_str, device::Core, dma::{Device, DmaMask}, - drm, of, platform, prelude::*, @@ -24,10 +22,69 @@ use kernel::{ use crate::g18::boot as g18_live_boot; use crate::g18::{hw, identity, regs}; use crate::{ - driver::{AsahiData, AsahiDevice}, + driver::{AsahiDrmDevice, AsahiPlatformDriver, AsahiPlatformDriverData}, gpu, }; +/// Name of the G18 platform driver. +pub(crate) const DRIVER_NAME: &CStr = c"aurora-g18"; + +kernel::of_device_table!( + OF_TABLE, + ::IdInfo, + [(of::DeviceId::new(c"apple,agx-t8152"), &crate::hw::t8152::T8152)] +); + +/// The G18 platform driver: the shared [`AsahiPlatformDriver`] probed through +/// [`AsahiPlatformDriver::probe_g18`]. +pub(crate) struct Driver; + +/// What a bound G18 device holds until unbind. +pub(crate) struct G18Bound<'bound> { + /// The firmware owner, shut down on unbind before devres releases its resources. + pub(crate) runtime: Arc, + /// The DMA binding of a diagnostic stage, which has no DRM device. + pub(crate) binding: Option, + /// The shared driver data of the application stage's render node. + pub(crate) driver: Option>, +} + +impl platform::Driver for Driver { + type IdInfo = &'static crate::hw::t8152::SocConfig; + type Data<'bound> = G18Bound<'bound>; + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + /// The firmware session is retained until reboot once started. + fn suppress_bind_attrs() -> bool { + true + } + + fn probe<'bound>( + pdev: &'bound platform::Device>, + info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound { + let soc = *info.ok_or(ENODEV)?; + AsahiPlatformDriver::probe_g18(pdev, soc) + } + + fn unbind<'bound>(_pdev: &'bound platform::Device>, this: Pin<&Self::Data<'bound>>) { + this.runtime.shutdown(); + if let Some(binding) = &this.binding { + binding.close(); + } + if let Some(driver) = &this.driver { + driver.close_binding(); + } + } +} + +/// A started G18 session: its owner and, for the application stage, the GPU +/// the render node is registered over. +pub(crate) struct G18Started { + pub(crate) runtime: Arc, + pub(crate) gpu: Option>, +} + /// Owner object returned by the G18 live-boot module. /// /// The live-boot module implements this for its session type. The driver @@ -68,9 +125,9 @@ pub(crate) struct T8152Probe { pub(crate) registers: regs::Resources, /// Hardware identity, only when the SGX PSD proved the block powered. pub(crate) observed: Option, - /// Unregistered DRM device whose data slot is still uninitialized, as - /// on M5. The live-boot module initializes it before any registration. - pub(crate) drm: ARef, + /// The DRM device, not yet registered. The live-boot module allocates + /// its kernel objects on it; the shared probe registers it at stage 10. + pub(crate) drm: ARef, } /// Loader receipt the G18G firmware stages require. The loader writes it @@ -84,11 +141,8 @@ const G18G_FIRMWARE_PROFILE: &[u8] = b"G18G-RTKit-3514.0.15.release\0"; /// default. Unlike the M5 default it also skips the read-only SGX identity /// probe, because J873 cold-boots with the GPU PSD services idle, where an /// SGX read can raise an SError. -pub(crate) fn probe_t8152( - pdev: &platform::Device, - soc: &'static hw::agx3::SocConfig, -) -> Result> { - let stage = *crate::module_parameters::g18_initdata_test.value(); +pub(crate) fn admit_t8152(pdev: &platform::Device>) -> Result { + let stage = crate::g18::params::g18_initdata_test.value(); if stage == 0 { dev_info!( pdev.as_ref(), @@ -104,7 +158,7 @@ pub(crate) fn probe_t8152( ); return Err(EINVAL); } - let variant = *crate::module_parameters::g18_compute_variant.value(); + let variant = crate::g18::params::g18_compute_variant.value(); if variant > 2 || (variant != 0 && stage != g18_live_boot::STAGE_COMPUTE_ADD) { dev_err!( pdev.as_ref(), @@ -113,11 +167,11 @@ pub(crate) fn probe_t8152( ); return Err(EINVAL); } - let render_variant = *crate::module_parameters::g18_render_variant.value(); - let render_profile = *crate::module_parameters::g18_render_profile.value(); - let sma_minimum = *crate::module_parameters::g18_sma_minimum.value(); - let sma_ideal = *crate::module_parameters::g18_sma_ideal.value(); - let native_barrier = *crate::module_parameters::g18_native_barrier.value(); + let render_variant = crate::g18::params::g18_render_variant.value(); + let render_profile = crate::g18::params::g18_render_profile.value(); + let sma_minimum = crate::g18::params::g18_sma_minimum.value(); + let sma_ideal = crate::g18::params::g18_sma_ideal.value(); + let native_barrier = crate::g18::params::g18_native_barrier.value(); if native_barrier > 1 || (native_barrier != 0 && (sma_ideal != 1 || sma_minimum != 0 || stage != crate::g18::boot::render_stage::STAGE_RENDER || render_variant != 0 || render_profile != 4)) { dev_err!(pdev.as_ref(), "G18G: native barrier probe requires ideal SMA stage9 variant0 profile4; refused before MMIO\n"); @@ -133,19 +187,19 @@ pub(crate) fn probe_t8152( let minimum_probe = render_variant == 0 && ((stage == crate::g18::boot::render_stage::STAGE_RENDER && render_profile == 4) || (stage == g18_live_boot::STAGE_DRM && render_profile == 1)); - let stall_trace = *crate::module_parameters::g18_graphics_stall_trace.value(); + let stall_trace = crate::g18::params::g18_graphics_stall_trace.value(); if stall_trace > 1 || (stall_trace != 0 && !minimum_probe) { dev_err!(pdev.as_ref(), "G18G: graphics stall tracing requires stage9/profile4 or stage10/profile1, variant0; refused before MMIO\n"); return Err(EINVAL); } - let high_vdm = *crate::module_parameters::g18_vdm_high_alias.value(); + let high_vdm = crate::g18::params::g18_vdm_high_alias.value(); if high_vdm > 1 || (high_vdm != 0 && (stage != crate::g18::boot::render_stage::STAGE_RENDER || render_variant != 0 || render_profile != 4 || sma_minimum != 1 || sma_ideal != 0 || stall_trace != 1)) { dev_err!(pdev.as_ref(), "G18G: high VDM alias requires stage9/profile4/variant0, minimum SMA and stall tracing; refused before MMIO\n"); return Err(EINVAL); } - let high_ppp = *crate::module_parameters::g18_ppp_high_alias.value(); + let high_ppp = crate::g18::params::g18_ppp_high_alias.value(); if high_ppp > 1 || (high_ppp != 0 && high_vdm != 1) { dev_err!(pdev.as_ref(), "G18G: high PPP aliases require the bounded high VDM control; refused before MMIO\n"); return Err(EINVAL); @@ -154,7 +208,7 @@ pub(crate) fn probe_t8152( dev_err!(pdev.as_ref(), "G18G: SMA minimum probe requires stage9/profile4 or stage10/profile1, variant0; refused before MMIO\n"); return Err(EINVAL); } - let usc_statistics = *crate::module_parameters::g18_usc_statistics.value(); + let usc_statistics = crate::g18::params::g18_usc_statistics.value(); if usc_statistics > 1 || (usc_statistics != 0 && (stage != crate::g18::boot::render_stage::STAGE_RENDER || render_variant != 0)) { dev_err!(pdev.as_ref(), "G18G: USC statistics require the stage9 colour fixture; refused before MMIO\n"); @@ -165,52 +219,58 @@ pub(crate) fn probe_t8152( } else if render_variant != 0 || render_profile != 1 { return Err(EINVAL); } - let admit_msaa4 = *crate::module_parameters::g18_admit_msaa4.value(); + let admit_msaa4 = crate::g18::params::g18_admit_msaa4.value(); if admit_msaa4 > 1 || (admit_msaa4 != 0 && stage != g18_live_boot::STAGE_DRM) { dev_err!(pdev.as_ref(), "G18G: g18_admit_msaa4={} needs 0, or 1 with g18_initdata_test=10; refused before MMIO\n", admit_msaa4); return Err(EINVAL); } + if stage == g18_live_boot::STAGE_DRM && crate::g18::params::g18_admit_compute.value() > 1 { + return Err(EINVAL); + } + Ok(()) +} + +/// Start the stage that [`admit_t8152`] admitted, on the unregistered `drm`. +pub(crate) fn probe_t8152( + pdev: &platform::Device>, + drm: &AsahiDrmDevice, + soc: &'static hw::agx3::SocConfig, +) -> Result { + let stage = crate::g18::params::g18_initdata_test.value(); if stage == g18_live_boot::STAGE_DRM { - if *crate::module_parameters::g18_admit_compute.value() > 1 {return Err(EINVAL);} - return probe_t8152_drm(pdev, soc, stage); + return probe_t8152_drm(pdev, drm, soc, stage); } - let probe = prepare_t8152_device(pdev, soc, stage)?; - g18_live_boot_start(pdev, probe) + let probe = prepare_t8152_device(pdev, drm, soc, stage)?; + Ok(G18Started { runtime: g18_live_boot_start(pdev, probe)?, gpu: None }) } -/// Register the G18 DRM render node over a started G18 session. +/// Start the G18 session the render node is registered over. fn probe_t8152_drm( - pdev: &platform::Device, + pdev: &platform::Device>, + drm: &AsahiDrmDevice, soc: &'static hw::agx3::SocConfig, stage: u32, -) -> Result> { - let mut probe = prepare_t8152_device(pdev, soc, stage)?; - let drm = probe.drm.clone(); +) -> Result { + let mut probe = prepare_t8152_device(pdev, drm, soc, stage)?; let observed = probe.observed.take(); let (owner, runtime) = g18_live_boot::start_drm(pdev, probe)?; let config = crate::g18::drm::G18DrmConfig { - admit_compute: *crate::module_parameters::g18_admit_compute.value() != 0, - admit_msaa4: *crate::module_parameters::g18_admit_msaa4.value() != 0, + admit_compute: crate::g18::params::g18_admit_compute.value() != 0, + admit_msaa4: crate::g18::params::g18_admit_msaa4.value() != 0, admit_graphics_scratch: - *crate::module_parameters::g18_admit_graphics_scratch.value() != 0, + crate::g18::params::g18_admit_graphics_scratch.value() != 0, }; let gpu: Arc = Arc::new( crate::g18::drm::G18DrmGpu::new(runtime, observed.as_ref(), config)?, GFP_KERNEL, )?; - let data = try_pin_init!(AsahiData { gpu, pdev: pdev.into(), resources: None }); - // SAFETY: prepare_t8152_device leaves this data slot uninitialized. This - // is its sole initialization, before registration exposes file operations. - unsafe { drm.init_data(data)? }; - (*drm).gpu.init()?; - crate::g18::drm::note_g18_parent(pdev.as_ref().as_raw()); - drm::driver::Registration::new_foreign_owned(&drm, pdev.as_ref(), 0)?; - dev_info!(pdev.as_ref(), "G18_DRM registered render node (asahi, apple,agx-t8152)\n"); - Ok(owner) + gpu.init()?; + Ok(G18Started { runtime: owner, gpu: Some(gpu) }) } fn prepare_t8152_device( - pdev: &platform::Device, + pdev: &platform::Device>, + drm: &AsahiDrmDevice, soc: &'static hw::agx3::SocConfig, stage: u32, ) -> Result { @@ -240,7 +300,7 @@ fn prepare_t8152_device( let firmware_profile_admitted = match of::chosen() { Some(chosen) => { let profile: Result> = - chosen.get_property(c_str!("aurora,g18g-gpu-firmware-profile")); + chosen.get_property(c"aurora,g18g-gpu-firmware-profile"); matches!(profile, Ok(profile) if profile.as_slice() == G18G_FIRMWARE_PROFILE) } None => false, @@ -299,11 +359,9 @@ fn prepare_t8152_device( unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(soc.uat_oas)?)? }; } - // This device is deliberately unregistered and its data slot stays - // uninitialized until the stage-10 DRM path calls `init_data`; the DRM - // release path drops only initialized data. - // SAFETY: nothing reads the data slot before `init_data`. - let drm: ARef = unsafe { drm::device::Device::new_uninit(pdev.as_ref())? }; + // The device stays unregistered until the shared probe registers it at + // stage 10; diagnostic stages never register it. + let drm: ARef = drm.into(); Ok(T8152Probe { soc, identity, @@ -332,7 +390,7 @@ fn prepare_t8152_device( /// On success the driver keeps the owner and calls /// [`G18Runtime::shutdown`] on unbind. fn g18_live_boot_start( - pdev: &platform::Device, + pdev: &platform::Device>, probe: T8152Probe, ) -> Result> { g18_live_boot::start(pdev, probe) diff --git a/drivers/gpu/drm/asahi/g18/queue/graphics.rs b/drivers/gpu/drm/asahi/g18/queue/graphics.rs index 9a7b53295190a..7917fd9555153 100644 --- a/drivers/gpu/drm/asahi/g18/queue/graphics.rs +++ b/drivers/gpu/drm/asahi/g18/queue/graphics.rs @@ -177,7 +177,7 @@ pub(crate) fn g18_stage_config( #[cfg(not(test))] use crate::g18::buffer::object::MappedObject; #[cfg(not(test))] -use crate::{driver::AsahiDevice, mmu}; +use crate::{driver::AsahiDrmDevice, mmu}; #[cfg(not(test))] use core::sync::atomic::{fence, Ordering}; #[cfg(not(test))] @@ -214,8 +214,8 @@ unsafe impl Sync for DataView {} #[cfg(not(test))] impl DataView { - fn new(dev: &AsahiDevice) -> Result { - let resource = dev.as_ref().of_node().ok_or(EINVAL)? + fn new(dev: &AsahiDrmDevice) -> Result { + let resource = dev.as_ref().as_ref().of_node().ok_or(EINVAL)? .reserved_mem_region_to_resource_byname(c_str!("gfx-data"))?; if resource.size() != DATA_SIZE as u64 || resource.start() < 0x100_0000_0000 || resource.start() & 0x3fff != 0 @@ -293,7 +293,7 @@ impl G18GraphicsQueues { /// 0xfffffe0008bdd178), the shared job list and the shared control /// (AGXCommandQueueReclaimableState::init 0xfffffe0008bf9014). Nothing is /// published: all inner producers stay 0. - pub(crate) fn new(dev: &AsahiDevice, uat: &mmu::Uat, a: G18GraphicsQueueAddresses) -> Result { + pub(crate) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, a: G18GraphicsQueueAddresses) -> Result { if !uat.is_t8152_geometry() { return Err(ENODEV); } let mut entries = KVec::new(); let mut aliases = KVec::new(); @@ -458,7 +458,7 @@ impl G18GraphicsQueues { /// Compare the retained command records to the shared pure encoders and /// print their actual bytes' digest before producer 6 can be published. - pub(crate) fn audit_prepared_pass(&mut self, dev: &AsahiDevice, pass: &G18QueuedPass) -> Result { + pub(crate) fn audit_prepared_pass(&mut self, dev: &AsahiDrmDevice, pass: &G18QueuedPass) -> Result { if self.step != 4 || self.counters()? != [[3; 3]; 2] { return Err(EBUSY); } let a = self.addresses; for stage in 0..2 { @@ -736,7 +736,7 @@ impl G18ParameterBuffer { /// firmware: the PB-table row is written separately by `load_row`, and /// the tag15 that names the PB comes later still. pub(crate) fn new( - dev: &AsahiDevice, + dev: &AsahiDrmDevice, uat: &mmu::Uat, low_vm: &mmu::Vm, a: G18PbAddresses, @@ -1034,7 +1034,7 @@ impl G18ParameterBuffer { Ok(pages) } - pub(crate) fn print_h1_digests(&mut self, dev: &AsahiDevice, row: &[u8;16]) -> Result { + pub(crate) fn print_h1_digests(&mut self, dev: &AsahiDrmDevice, row: &[u8;16]) -> Result { // Context +8 is firmware-owned and separately checked. Normalize only // that field and the unwritten +1c word for host reproducibility. self.state.with_bytes_mut(|out| { diff --git a/drivers/gpu/drm/asahi/g18/queue/graphics_application.rs b/drivers/gpu/drm/asahi/g18/queue/graphics_application.rs index ab15b357a87ed..623b98d506fee 100644 --- a/drivers/gpu/drm/asahi/g18/queue/graphics_application.rs +++ b/drivers/gpu/drm/asahi/g18/queue/graphics_application.rs @@ -4,7 +4,7 @@ use super::*; use crate::g18::uapi::application_runtime::Frame; impl G18GraphicsQueues { - pub(crate) fn prepare_application_pass(&mut self, dev: &AsahiDevice, + pub(crate) fn prepare_application_pass(&mut self, dev: &AsahiDrmDevice, pass: &G18QueuedPass, frame: Frame) -> Result { let before = frame.inner_before(); if self.counters()? != [[before;3];2] diff --git a/drivers/gpu/drm/asahi/g18/queue/probe.rs b/drivers/gpu/drm/asahi/g18/queue/probe.rs index ff425b12b424e..ea739dab517b4 100644 --- a/drivers/gpu/drm/asahi/g18/queue/probe.rs +++ b/drivers/gpu/drm/asahi/g18/queue/probe.rs @@ -9,7 +9,7 @@ use crate::g18::buffer::object::MappedObject; use crate::g18::buffer::pool::G18UmaPoolBinding; use crate::g18::runtime::compute::G18PreparedComputeCommand; -use crate::{driver::AsahiDevice, mmu}; +use crate::{driver::AsahiDrmDevice, mmu}; use core::sync::atomic::{fence, Ordering}; use kernel::{c_str, io, prelude::*}; @@ -44,8 +44,8 @@ unsafe impl Send for FirmwareDataView {} unsafe impl Sync for FirmwareDataView {} impl FirmwareDataView { - fn new(dev: &AsahiDevice) -> Result { - let resource = dev.as_ref().of_node().ok_or(EINVAL)? + fn new(dev: &AsahiDrmDevice) -> Result { + let resource = dev.as_ref().as_ref().of_node().ok_or(EINVAL)? .reserved_mem_region_to_resource_byname(c_str!("gfx-data"))?; if resource.size() != DATA_SIZE as u64 || resource.start() < 0x100_0000_0000 || resource.start() & 0x3fff != 0 @@ -109,7 +109,7 @@ fn put64(out: &mut [u8], offset: usize, value: u64) { } impl G18QueueProbe { - pub(crate) fn new(dev: &AsahiDevice, uat: &mmu::Uat, bind_scheduler: bool) -> Result { + pub(crate) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, bind_scheduler: bool) -> Result { if !uat.is_t8152_geometry() { return Err(ENODEV); } let mut entry = MappedObject::new_wc_at(dev, uat.kernel_vm(), ENTRY_HIGH, ENTRY_SIZE, mmu::PROT_FW_PRIV_RW)?; diff --git a/drivers/gpu/drm/asahi/g18/queue/scheduler.rs b/drivers/gpu/drm/asahi/g18/queue/scheduler.rs index 34b91dd1d2486..b5579ba9e534e 100644 --- a/drivers/gpu/drm/asahi/g18/queue/scheduler.rs +++ b/drivers/gpu/drm/asahi/g18/queue/scheduler.rs @@ -8,7 +8,7 @@ #![allow(dead_code)] use crate::g18::buffer::object::MappedObject; -use crate::{driver::AsahiDevice, g18::fw::scheduler::*, mmu}; +use crate::{driver::AsahiDrmDevice, g18::fw::scheduler::*, mmu}; use core::ops::Range; use core::sync::atomic::{fence, Ordering}; use kernel::prelude::*; @@ -37,7 +37,7 @@ pub(crate) struct G18SchedulerMemory { } impl G18SchedulerMemory { - pub(crate) fn print_render_digests(&mut self, dev: &AsahiDevice) -> Result { + pub(crate) fn print_render_digests(&mut self, dev: &AsahiDrmDevice) -> Result { for (name, object) in [("scheduler", &mut self.records), ("submitted-pre-kick", &mut self.counters)] { object.with_bytes_mut(|bytes| { dev_info!(dev.as_ref(), "G18_RENDER_DIGEST name={} fnv={:#018x}\n",name,crate::g18::fw::graphics_payload::fnv64(bytes)); @@ -48,7 +48,7 @@ impl G18SchedulerMemory { } /// Runtime's deterministic reservation, separate from the legacy compute /// scheduler. Counters and records are independent uncached FW mappings. - pub(crate) fn new_render(dev: &AsahiDevice, uat: &mmu::Uat) -> Result { + pub(crate) fn new_render(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result { use crate::g18::address_map::GRAPHICS_SCHEDULER_START as BASE; if !uat.is_t8152_geometry() { return Err(ENODEV); } let counters = MappedObject::new_wc_at(dev, uat.kernel_vm(), BASE, @@ -62,7 +62,7 @@ impl G18SchedulerMemory { sequences: [0; G18_SCHEDULER_RECORD_COUNT], submitted: [0; G18_SCHEDULER_RECORD_COUNT] }) } - pub(crate) fn new(dev: &AsahiDevice, uat: &mmu::Uat, high_range: Range) -> Result { + pub(crate) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, high_range: Range) -> Result { if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != 0x4000 { return Err(ENODEV); } diff --git a/drivers/gpu/drm/asahi/g18/regs.rs b/drivers/gpu/drm/asahi/g18/regs.rs index ede40c1b2e16d..c96ca475263fb 100644 --- a/drivers/gpu/drm/asahi/g18/regs.rs +++ b/drivers/gpu/drm/asahi/g18/regs.rs @@ -8,8 +8,7 @@ use kernel::{ c_str, device::Core, - devres::Devres, - io::{mem::IoMem, Io}, + io::{mem::DevresIoMem, Io, IoBase}, platform, prelude::*, sync::aref::ARef, @@ -40,7 +39,7 @@ const CORE_MASKS_G14X: usize = 0xe01500; /// Device resources for this GPU instance. pub(crate) struct Resources { dev: ARef, - sgx: Pin>>>, + sgx: DevresIoMem, // The G17P SKSM FIFO lock and deferred SKSM writes are not carried: on // T8152 the firmware owns KSMFE queue setup (hwdata+0x2724 = 0) and the // host performs no SKSM MMIO. @@ -49,13 +48,13 @@ pub(crate) struct Resources { impl Resources { /// Map the required resources given our platform device. - pub(crate) fn new(pdev: &platform::Device) -> Result { + pub(crate) fn new(pdev: &platform::Device>) -> Result { // The pinned host maps accelerator device-memory index 0 and stores // it at accelerator+0x348 before every SKSM scratch access. That range // is the J700 GPU block exposed to Linux by the named `sgx` resource // (the second platform `reg` tuple, after `asc`). let sgx_req = pdev.io_request_by_name(c_str!("sgx")).ok_or(EINVAL)?; - let sgx_iomem = KBox::pin_init(sgx_req.iomap(), GFP_KERNEL)?; + let sgx_iomem = sgx_req.iomap()?.into_devres()?; Ok(Resources { // SAFETY: This device does DMA via the UAT IOMMU. @@ -73,7 +72,7 @@ impl Resources { pub(crate) fn m5_match_pmgr_boot_control(&self) -> Result { const OFFSET: usize = 0xe90000; let sgx = self.sgx.try_access().ok_or(ENODEV)?; - let res = sgx.relaxed(); + let res = (&*sgx).as_view().relaxed(); let before = res.try_read32(OFFSET)?; if before != 0x30000 && before != 0x30001 { dev_err!(self.dev.as_ref(), "M5_INIT unexpected PMGR control {:#x}\n", before); @@ -397,7 +396,7 @@ impl T8152SgxPowered { /// The block is mapped for this call alone and never claimed, so a later /// power owner can map the same service block. pub(crate) fn t8152_read_psd( - pdev: &platform::Device, + pdev: &platform::Device>, service: T8152PsdService, ) -> Result { let resource = pdev @@ -417,7 +416,7 @@ pub(crate) fn t8152_read_psd( let request = pdev .io_request_by_name(service.resource_name()) .ok_or(ENODEV)?; - let iomem = KBox::pin_init(request.iomap(), GFP_KERNEL)?; + let iomem = request.iomap()?.into_devres()?; let io = iomem.try_access().ok_or(ENODEV)?; // STATE consumes the transition ACK; preserve it in the snapshot first. let ack = io.try_read32(T8152_PSD_ACK)?; @@ -433,7 +432,7 @@ pub(crate) fn t8152_read_psd( /// (state 0). Otherwise log the sample and return [`t8152_not_powered`], so /// the caller skips every SGX read. pub(crate) fn t8152_prove_sgx_powered( - pdev: &platform::Device, + pdev: &platform::Device>, ) -> Result { let sample = t8152_read_psd(pdev, T8152PsdService::Sgx)?; if !sample.powered() { diff --git a/drivers/gpu/drm/asahi/g18/runtime/compute.rs b/drivers/gpu/drm/asahi/g18/runtime/compute.rs index 3874897459a95..8db15ce7fcf18 100644 --- a/drivers/gpu/drm/asahi/g18/runtime/compute.rs +++ b/drivers/gpu/drm/asahi/g18/runtime/compute.rs @@ -9,7 +9,7 @@ #![allow(dead_code)] use crate::{g18::queue::scheduler::G18SchedulerBinding, g18::buffer::object::MappedObject}; -use crate::{driver::AsahiDevice, g18::fw::compute::*, g18::fw::compute_memory::*, mmu}; +use crate::{driver::AsahiDrmDevice, g18::fw::compute::*, g18::fw::compute_memory::*, mmu}; use core::ops::Range; use core::sync::atomic::{fence, Ordering}; use kernel::prelude::*; @@ -99,7 +99,7 @@ impl G18ComputeMemory { } pub(crate) fn new( - dev: &AsahiDevice, + dev: &AsahiDrmDevice, uat: &mmu::Uat, high_range: Range, clusters: u32, @@ -178,7 +178,7 @@ impl G18ComputeMemory { /// Bind the payload VM while retaining both aliases and the context lease. /// A failure leaves the original command/channel and firmware roots intact. pub(crate) fn bind_process( - &mut self, dev: &AsahiDevice, uat: &mmu::Uat, vm: &mmu::Vm, + &mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, ) -> Result { if !self.unpublished_idle() { return Err(EBUSY); } let context = uat.prepare_t8152_execution_context(vm)?; diff --git a/drivers/gpu/drm/asahi/g18/runtime/compute_payload.rs b/drivers/gpu/drm/asahi/g18/runtime/compute_payload.rs index eeb9d9a15036d..614a31167a434 100644 --- a/drivers/gpu/drm/asahi/g18/runtime/compute_payload.rs +++ b/drivers/gpu/drm/asahi/g18/runtime/compute_payload.rs @@ -6,7 +6,7 @@ #![allow(dead_code)] use crate::g18::buffer::object::MappedObject; -use crate::{driver::AsahiDevice, g18::fw::compute::*, g18::fixture::compute_payload as payload, +use crate::{driver::AsahiDrmDevice, g18::fw::compute::*, g18::fixture::compute_payload as payload, g18::fixture::compute_triangle::{TrianglePatch, TriangleResult, PATCH_COUNT}, g18::fw::compute_state::{G18ComputeState, G18_COMPUTE_STATE_BYTES}, mmu}; use core::sync::atomic::{fence, Ordering}; @@ -35,11 +35,11 @@ pub(crate) struct G18ComputePayloadMemory { } impl G18ComputePayloadMemory { - pub(crate) fn load(dev: &AsahiDevice, uat: &mmu::Uat) -> Result { + pub(crate) fn load(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result { if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != payload::PAGE_BYTES { return Err(ENODEV); } - let firmware = Firmware::request(c"asahi/g18-26a428-compute-add.bin", dev.as_ref())?; + let firmware = Firmware::request(c"asahi/g18-26a428-compute-add.bin", dev.as_ref().as_ref())?; let reference = payload::ComputePayload::parse(firmware.data()).map_err(|_| EINVAL)?; let vm = uat.new_t8152_compute_fixture_vm()?; let mut pages = KVec::new(); @@ -69,7 +69,7 @@ impl G18ComputePayloadMemory { if initial.checked != 4096 || initial.untouched != 4096 || initial.correct != 0 { return Err(EIO); } - match *crate::module_parameters::g18_compute_variant.value() { + match crate::g18::params::g18_compute_variant.value() { 0 => {} 1 => owner.empty_cdm_stream(dev)?, 2 => owner.triangle_shader(dev, payload::fnv64(&firmware.data()[payload::HEADER_BYTES..]))?, @@ -104,7 +104,7 @@ impl G18ComputePayloadMemory { /// barrier and terminate at +0/+4, zero +8..+0x2f (the old launch, /// barrier and terminate) and point command +0xee0 at the new terminate. /// Every other byte, page and register is unchanged; nothing is published. - fn empty_cdm_stream(&mut self, dev: &AsahiDevice) -> Result { + fn empty_cdm_stream(&mut self, dev: &AsahiDrmDevice) -> Result { let mut command = G18ComputeCommandInputs::default(); let mut registers = G18ComputeRegisterInputs::default(); self.apply_state(&mut command, &mut registers); @@ -142,8 +142,8 @@ impl G18ComputePayloadMemory { /// Validate every original site before mutating any page. All pages and /// roots are still private; failure cannot publish a partial program. - fn triangle_shader(&mut self, dev: &AsahiDevice, base_fnv: u64) -> Result { - let firmware = Firmware::request(c"asahi/g18-26a428-compute-triangle.bin", dev.as_ref())?; + fn triangle_shader(&mut self, dev: &AsahiDrmDevice, base_fnv: u64) -> Result { + let firmware = Firmware::request(c"asahi/g18-26a428-compute-triangle.bin", dev.as_ref().as_ref())?; let patch = TrianglePatch::parse(firmware.data(), base_fnv).ok_or(EINVAL)?; let mut indices = [0usize; PATCH_COUNT]; for (i, index) in indices.iter_mut().enumerate() { diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs index cc6a35bd01669..de7e2cd51f5d2 100644 --- a/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs @@ -6,7 +6,7 @@ use crate::{g18::buffer::object::MappedObject, g18::runtime::graphics_memory::G1 g18::queue::scheduler::{G18SchedulerMemory, G18SchedulerSnapshot}, g18::queue::graphics::{G18GraphicsQueues, G18GraphicsQueueAddresses, G18ParameterBuffer, G18PbAddresses, G18PbPolicy, G18QueuedPass, G18PbReadback}}; -use crate::{driver::AsahiDevice, g18::fw::render_params::G17pRenderParameters, +use crate::{driver::AsahiDrmDevice, g18::fw::render_params::G17pRenderParameters, g18::address_map as va, g18::fw::render as regs, g18::fw::render_command as cmd, g18::fw::render_assembly::{self as assembly, EncodedJob, RenderVariant}, mmu}; use core::sync::atomic::{fence, Ordering}; @@ -31,7 +31,7 @@ pub(crate) struct G18GraphicsJob { impl G18GraphicsJob { #[inline(never)] - pub(crate) fn from_payload(dev: &AsahiDevice, uat: &mmu::Uat, + pub(crate) fn from_payload(dev: &AsahiDrmDevice, uat: &mmu::Uat, profile: u32, variant: RenderVariant) -> Result> { if (variant == RenderVariant::GearsThreeFrames) != (profile == 3) { return Err(EINVAL); } let mut program = G18GraphicsMemory::load_payload(dev, uat, profile)?; @@ -42,19 +42,19 @@ impl G18GraphicsJob { crate::g18::fw::graphics_payload::Capture::FragmentWitness => regs::G18Capture::FragmentWitness, }; let (mut p, mut controls) = regs::g18_capture_parameters(capture); - if *crate::module_parameters::g18_vdm_high_alias.value() == 1 { + if crate::g18::params::g18_vdm_high_alias.value() == 1 { let source = p.encoder; p.encoder = program.alias_vdm_for_address_probe(source)?; - dev_info!(dev, "G18_VDM_ADDRESS_CONTROL source={:#x} alias={:#x} compact={:#x} bytes=16384 contents=unchanged\n", + dev_info!(dev.as_ref(), "G18_VDM_ADDRESS_CONTROL source={:#x} alias={:#x} compact={:#x} bytes=16384 contents=unchanged\n", source, va::GRAPHICS_HIGH_VDM, p.encoder); - dev_info!(dev, "G18_RENDER_OBJECT name=vdm-address-control va={:#x} bytes={:#x} context={}\n", + dev_info!(dev.as_ref(), "G18_RENDER_OBJECT name=vdm-address-control va={:#x} bytes={:#x} context={}\n", va::GRAPHICS_HIGH_VDM, va::PAGE, program.context_id()); } - if *crate::module_parameters::g18_ppp_high_alias.value() == 1 { + if crate::g18::params::g18_ppp_high_alias.value() == 1 { program.alias_ppp_for_address_probe()?; - dev_info!(dev, "G18_PPP_ADDRESS_CONTROL records=14 pages=4 contents=unchanged vdm_changes=ppp-pointers-only\n"); + dev_info!(dev.as_ref(), "G18_PPP_ADDRESS_CONTROL records=14 pages=4 contents=unchanged vdm_changes=ppp-pointers-only\n"); for source in va::GRAPHICS_HIGH_PPP_PAGES { - dev_info!(dev, "G18_RENDER_OBJECT name=ppp-address-control-{:x} va={:#x} bytes={:#x} context={}\n", + dev_info!(dev.as_ref(), "G18_RENDER_OBJECT name=ppp-address-control-{:x} va={:#x} bytes={:#x} context={}\n", source, va::GRAPHICS_HIGH_PPP_BASE - 0x58000 + source, va::PAGE, program.context_id()); } } @@ -78,7 +78,7 @@ impl G18GraphicsJob { targets.push(target, GFP_KERNEL)?; } assembly::apply_target_plan(&mut p, &plan); - if *crate::module_parameters::g18_usc_statistics.value() != 0 { + if crate::g18::params::g18_usc_statistics.value() != 0 { if variant != RenderVariant::Color || program.context_id() >= 64 { return Err(EINVAL); } @@ -93,10 +93,10 @@ impl G18GraphicsJob { }; controls.ta.usc_statistics = encode(p.ta_status)?; controls.fragment.usc_statistics = encode(p.fragment_status)?; - dev_info!(dev, "G18_USC_STATISTICS context={} ta={:#x} fragment={:#x} backing=16384-each\n", + dev_info!(dev.as_ref(), "G18_USC_STATISTICS context={} ta={:#x} fragment={:#x} backing=16384-each\n", program.context_id(), controls.ta.usc_statistics, controls.fragment.usc_statistics); } - dev_info!(dev, "G18_RENDER_TARGET kind={} primary={:#x} tpc-allocation={:#x} native={:?}\n", + dev_info!(dev.as_ref(), "G18_RENDER_TARGET kind={} primary={:#x} tpc-allocation={:#x} native={:?}\n", crate::g18::uapi::render_policy::target_kind(), plan.sizes[0], plan.sizes[1], plan.native); let mut parameter_buffer = G18ParameterBuffer::new(dev, uat, program.vm(), G18PbAddresses { state_high: va::GRAPHICS_PB_HIGH, graph_high: va::PB_GRAPH, @@ -141,7 +141,7 @@ impl G18GraphicsJob { pub(crate) fn queues(&mut self) -> &mut G18GraphicsQueues { &mut self.queues } pub(crate) fn parameter_buffer(&mut self) -> &mut G18ParameterBuffer { &mut self.parameter_buffer } #[inline(never)] - pub(crate) fn encode_h1(&mut self, dev: &AsahiDevice) -> Result { + pub(crate) fn encode_h1(&mut self, dev: &AsahiDrmDevice) -> Result { let inputs = self.h1_policy()?; self.encode(dev, &inputs) } @@ -163,7 +163,7 @@ impl G18GraphicsJob { event_generation: [0;2], ta_barrier_flag: crate::g18::uapi::render_policy::ta_barrier_flag() }) } - pub(crate) fn audit_h1(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + pub(crate) fn audit_h1(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, pb: &G18PbReadback, uma: &[u64;16]) -> Result { let pass = self.pass()?; self.queues.audit_before_pass(uat, &pass)?; @@ -288,7 +288,7 @@ impl G18GraphicsJob { /// Caller has allocated/retained PB and pool owners and filled all native /// fields. In particular a default resource word is not an admitted policy. #[inline(never)] - pub(crate) fn encode(&mut self, dev: &AsahiDevice, inputs: &cmd::G18RenderCommandInputs) -> Result { + pub(crate) fn encode(&mut self, dev: &AsahiDrmDevice, inputs: &cmd::G18RenderCommandInputs) -> Result { if self.encoded.is_some() || self.controls.ta.resource_word == 0 || self.controls.fragment.resource_word == 0 || inputs.context_id != u32::from(self.context_id()) @@ -337,7 +337,7 @@ impl G18GraphicsJob { pub(crate) fn scheduler_snapshot(&mut self) -> Result { self.scheduler.snapshot(0) } pub(crate) fn output(&mut self) -> Result<(u64, u32)> { self.program.output() } - fn log_objects(&self, dev: &AsahiDevice) { + fn log_objects(&self, dev: &AsahiDrmDevice) { for (i, target) in self.targets.iter().enumerate() { dev_info!(dev.as_ref(), "G18_RENDER_OBJECT name={} va={:#x} bytes={:#x} context={}\n", assembly::TARGET_NAMES[i], target.iova(), target.logical_size, self.context_id()); diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs index fc9cb879637ea..5a43ad3816e62 100644 --- a/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs @@ -42,7 +42,7 @@ impl ApplicationJob { pub(crate) fn traced(&self) -> bool {self.client_frames<=3} #[inline(never)] - pub(crate) fn prepare(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + pub(crate) fn prepare(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, render: &G18ApplicationRender) -> Result { if !self.retired {return Err(EBUSY);} let frame=Frame::new(self.generation.checked_add(1).ok_or(EOVERFLOW)?).ok_or(EOVERFLOW)?; @@ -120,7 +120,7 @@ impl ApplicationJob { } #[inline(never)] - fn encode_application(&mut self, dev: &AsahiDevice, frame: Frame) -> Result { + fn encode_application(&mut self, dev: &AsahiDrmDevice, frame: Frame) -> Result { let inputs=self.application_policy(frame)?; self.job.encode(dev,&inputs) } @@ -154,7 +154,7 @@ impl ApplicationJob { let mut pass=self.job.pass()?;pass.stamps=[self.frame().stamp();2];Ok(pass) } - pub(crate) fn audit(&mut self, dev: &AsahiDevice, uat: &mmu::Uat, + pub(crate) fn audit(&mut self, dev: &AsahiDrmDevice, uat: &mmu::Uat, pb: &G18PbReadback, uma: &[u64;16], mut pages: usize) -> Result { let frame=self.frame();let pass=self.pass()?; if frame.generation==1 {self.job.queues.audit_before_pass(uat,&pass)?;} diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs index 222352d1360bd..15e24f5c0d38e 100644 --- a/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Retained G18GFX01 v2 pages. Construction never publishes roots or work. use crate::g18::buffer::object::MappedObject; -use crate::{driver::AsahiDevice, g18::address_map as va, g18::fw::graphics_payload as payload, mmu}; +use crate::{driver::AsahiDrmDevice, g18::address_map as va, g18::fw::graphics_payload as payload, mmu}; use core::sync::atomic::{fence, Ordering}; use kernel::{firmware::Firmware, prelude::*}; @@ -21,7 +21,7 @@ pub(crate) struct G18GraphicsMemory { } impl G18GraphicsMemory { - pub(crate) fn load_payload(dev: &AsahiDevice, uat: &mmu::Uat, profile: u32) -> Result { + pub(crate) fn load_payload(dev: &AsahiDrmDevice, uat: &mmu::Uat, profile: u32) -> Result { let name = match profile { 1 => c"asahi/g18-26a428-color-a-v2.bin", 2 => c"asahi/g18-26a428-color-b-v2.bin", @@ -30,7 +30,7 @@ impl G18GraphicsMemory { _ => return Err(EINVAL), }; if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != payload::PAGE_BYTES { return Err(ENODEV); } - let firmware = Firmware::request(name, dev.as_ref())?; + let firmware = Firmware::request(name, dev.as_ref().as_ref())?; let input = payload::GraphicsPayload::parse(firmware.data()).map_err(|_| EINVAL)?; if input.capture() as u32 != profile { return Err(EINVAL); } let vm = uat.new_t8152_compute_fixture_vm()?; diff --git a/drivers/gpu/drm/asahi/g18/uapi/render_policy.rs b/drivers/gpu/drm/asahi/g18/uapi/render_policy.rs index c6bb9b4f6bf42..84170ed8be79c 100644 --- a/drivers/gpu/drm/asahi/g18/uapi/render_policy.rs +++ b/drivers/gpu/drm/asahi/g18/uapi/render_policy.rs @@ -30,11 +30,11 @@ pub(crate) const IDEAL_GROWTH_OFFSET: u64 = 0x158000 - IDEAL_GROWTH_BYTES; pub(crate) static HOST_SMA_IDEAL: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false); pub(crate) fn ideal_enabled() -> bool { - #[cfg(CONFIG_RUST)] { *crate::module_parameters::g18_sma_ideal.value() == 1 } + #[cfg(CONFIG_RUST)] { crate::g18::params::g18_sma_ideal.value() == 1 } #[cfg(not(CONFIG_RUST))] { HOST_SMA_IDEAL.load(core::sync::atomic::Ordering::Relaxed) } } pub(crate) fn ta_barrier_flag() -> u32 { - #[cfg(CONFIG_RUST)] { *crate::module_parameters::g18_native_barrier.value() } + #[cfg(CONFIG_RUST)] { crate::g18::params::g18_native_barrier.value() } #[cfg(not(CONFIG_RUST))] { 0 } } pub(crate) fn uma_blocks() -> u32 { if ideal_enabled() { 40 } else { UMA_BLOCKS } } @@ -52,7 +52,7 @@ pub(crate) fn usc_required() -> [u64; 2] { if ideal_enabled() { [9 * 0x100000, 2 pub(crate) fn resource_pairs() -> [[u64; 2]; 2] { #[cfg(CONFIG_RUST)] - let minimum = *crate::module_parameters::g18_sma_minimum.value() == 1; + let minimum = crate::g18::params::g18_sma_minimum.value() == 1; #[cfg(not(CONFIG_RUST))] let minimum = HOST_SMA_MINIMUM.load(core::sync::atomic::Ordering::Relaxed); if ideal_enabled() { NATIVE_IDEAL_PAIRS } else if minimum { NATIVE_MINIMUM_PAIRS } else { [RESOURCE_PAIR; 2] } @@ -71,7 +71,7 @@ mod parameter { static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g18_render_target_kind" - } else { c"asahi.g18_render_target_kind" }), + } else { c"aurora.g18_render_target_kind" }), // SAFETY: module lifetime and parameter serialization are managed by // the kernel, identically to the kernel::module! registration. #[cfg(MODULE)] @@ -109,7 +109,7 @@ mod msaa4_fragment { static PARAM: KernelParam = KernelParam::new(bindings::kernel_param { name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { c"g18_msaa4_fragment_word" - } else { c"asahi.g18_msaa4_fragment_word" }), + } else { c"aurora.g18_msaa4_fragment_word" }), #[cfg(MODULE)] mod_: &raw mut bindings::__this_module, #[cfg(not(MODULE))] @@ -140,7 +140,7 @@ pub(crate) fn msaa4_fragment_word() -> u64 { } pub(crate) fn target_kind() -> u32 { - #[cfg(CONFIG_RUST)] { *parameter::VALUE.value() } + #[cfg(CONFIG_RUST)] { parameter::VALUE.value() } #[cfg(not(CONFIG_RUST))] { HOST_TARGET_KIND.load(core::sync::atomic::Ordering::Relaxed) } } diff --git a/drivers/gpu/drm/asahi/hw/t8152.rs b/drivers/gpu/drm/asahi/hw/t8152.rs new file mode 100644 index 0000000000000..a50477ee21c38 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8152.rs @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8152 platforms (M6, G18G). +//! +//! The M6 GPU runs on the G18 backend (`crate::g18`), which uses its own +//! firmware interface and therefore an identification-only SoC entry rather +//! than a [`super::HwConfig`]. + +use crate::g18::identity::{GpuGen, GpuVariant}; + +/// Identification-only configuration for one AGX3 SoC. +#[allow(dead_code)] +#[derive(Debug)] +pub(crate) struct SocConfig { + /// Chip ID in hex format (e.g. 0x6050 for t6050). + pub(crate) chip_id: u32, + /// Decoded design generation for this chip. + pub(crate) gpu_gen: GpuGen, + /// Decoded variant for this chip. + pub(crate) gpu_variant: GpuVariant, + /// Hardware family byte expected at `0xd04000[31:24]`. + pub(crate) hw_family: u8, + /// Hardware variant byte expected at `0xd04000[23:16]`. + pub(crate) hw_variant: u8, + /// Die count expected at `0xd04010[19:16]`. + pub(crate) num_dies: u32, + /// Input-address bits per UAT root (42 on every AGX3 part). + pub(crate) uat_ias: u8, + /// Output address space (42-bit on every AGX3 part). + pub(crate) uat_oas: u32, +} + +const fn agx3( + chip_id: u32, + gpu_gen: GpuGen, + gpu_variant: GpuVariant, + hw_family: u8, + hw_variant: u8, + num_dies: u32, +) -> SocConfig { + SocConfig { + chip_id, + gpu_gen, + gpu_variant, + hw_family, + hw_variant, + num_dies, + uat_ias: 42, + uat_oas: 42, + } +} + +// G18 (M6 family). Family byte 0xc, variant 2: 26A428 +// `AGX·PI_400·G·A0·Accelerator::readChipInfo` @ 0xfffffe0008c3d3a0 compares +// the family byte with 0xc @ 0xfffffe0008c3d49c and gives variant 2 the +// 12-core G configuration @ 0xfffffe0008c3d4bc..0x4c8. Native J873 registry +// (AGXAcceleratorG18G): gpu_gen=18, gpu_var=G, usc_gen=3, is_sksm=1, +// num_cores=12, num_gps=6, num_mgpus=2, core_mask_list=(63,63), +// num_frags=12, kickid_qid_shift=40, kickid_qid_mask=127. The live ID +// registers have not been read yet (the GPU PSD reads idle at cold boot), so +// the T8152 probe identifies the part by chip id and compatible, and only +// logs a mismatching register decode. The ADT uat-vaddr-size is 43 on both +// J813 and J873: 42 bits per root plus the TTBR-select bit, as on T8142. +/// Apple M6 (G18G, J873 Mac mini). +pub(crate) const T8152: SocConfig = agx3(0x8152, GpuGen::G18, GpuVariant::G, 0xc, 2, 1); + +/// Native J873 `AGXAcceleratorG18G` registry facts (macOS 26A428 ioreg). +/// +/// These are identity facts, not firmware tuning: they size the core-mask +/// decode and cross-check the live ID registers once the GPU is powered. +#[allow(dead_code)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G18RegistryFacts { + pub(crate) num_cores: u32, + pub(crate) num_gps: u32, + pub(crate) num_mgpus: u32, + /// Per-mGPU (cluster) core masks, `core_mask_list`. + pub(crate) core_mask_list: [u32; 2], + pub(crate) num_frags: u32, + pub(crate) usc_generation: u32, + pub(crate) kickid_qid_shift: u32, + pub(crate) kickid_qid_mask: u32, +} + +/// J873 registry facts (PROVEN from the native ioreg). +#[allow(dead_code)] +pub(crate) const T8152_REGISTRY: G18RegistryFacts = G18RegistryFacts { + num_cores: 12, + num_gps: 6, + num_mgpus: 2, + core_mask_list: [0x3f, 0x3f], + num_frags: 12, + usc_generation: 3, + kickid_qid_shift: 40, + kickid_qid_mask: 127, +}; diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index 9f13436b4caf5..fd3de19a96d29 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -3,14 +3,14 @@ //! Prepared timestamp destinations shared by native command completion paths. use crate::mmu; -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] use crate::{file, gem}; -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] use core::ops::Range; -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] use core::sync::atomic::{fence as barrier, Ordering}; use kernel::{prelude::*, sync::Arc}; -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] use kernel::{uapi, xarray}; /// A firmware-mapped timestamp buffer with a kernel CPU mapping of its backing, kept for the @@ -27,7 +27,7 @@ pub(crate) enum Buffer { /// releasing the CPU mapping takes the reservation lock: the deferral runs the release /// on the cleanup queue. Firmware(crate::cleanup::Deferred), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Cpu { // The GEM pin keeps the object's held SG table alive through CPU publication. gem: gem::KernelMappingPin, @@ -45,7 +45,7 @@ impl Buffer { } /// Retain a CPU-written byte range without publishing any firmware address. - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] pub(crate) fn cpu(bo: gem::ObjectRef, range: Range) -> Result { let size = range.end.checked_sub(range.start).ok_or(EINVAL)?; if size == 0 || (range.start | size) & mmu::UAT_PGMSK != 0 || range.end > bo.size() { @@ -61,16 +61,16 @@ impl Buffer { pub(crate) fn size(&self) -> usize { match self { Self::Firmware(buffer) => buffer.mapping.size(), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Self::Cpu { range, .. } => range.end - range.start, } } - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { match self { Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Self::Cpu { gem, range, .. } => { if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { return Err(ERANGE); @@ -85,7 +85,7 @@ impl Buffer { pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { match self { Self::Firmware(buffer) => buffer.mapping.timestamp_offset(offset), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Self::Cpu { .. } => Err(EINVAL), } } @@ -95,7 +95,7 @@ impl Buffer { pub(crate) fn write_timestamp(&self, offset: usize, value: u64) { match self { Self::Firmware(buffer) => buffer.cpu.write(offset, value), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Self::Cpu { .. } => {} } } @@ -104,13 +104,13 @@ impl Buffer { pub(crate) fn firmware_mapping(&self) -> Result> { match self { Self::Firmware(buffer) => Ok(buffer.mapping.clone()), - #[cfg(CONFIG_DRM_ASAHI_G16)] + #[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] Self::Cpu { .. } => Err(EINVAL), } } } -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] fn checked_offset(offset: u64, size: usize, aligned: bool) -> Result { let offset = usize::try_from(offset).map_err(|_| EINVAL)?; let end = offset.checked_add(size_of::()).ok_or(EINVAL)?; @@ -123,14 +123,14 @@ fn checked_offset(offset: u64, size: usize, aligned: bool) -> Result { Ok(offset) } -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] struct Destination { // Retain the bound range and backing along with its prepared CPU view. _mapping: Arc, cpu: mmu::TimestampMapping, } -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] impl Destination { fn resolve( objects: Pin<&xarray::XArray>>, @@ -165,12 +165,13 @@ impl Destination { } /// Timestamp mappings resolved and retained before any command is enqueued. -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] pub(crate) struct Destinations(Option; 4]>>); -#[cfg(CONFIG_DRM_ASAHI_G16)] +#[cfg(any(CONFIG_DRM_ASAHI_G16, CONFIG_DRM_ASAHI_G18))] impl Destinations { /// G16 retains its eight-byte destination alignment and EINVAL contract. + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn resolve_aligned( objects: Pin<&xarray::XArray>>, timestamps: &[uapi::drm_asahi_timestamps], @@ -178,6 +179,15 @@ impl Destinations { Self::resolve_inner(objects, timestamps, true) } + /// G18 keeps module 1134's rule: any in-range offset, ERANGE beyond the buffer. + #[cfg(CONFIG_DRM_ASAHI_G18)] + pub(crate) fn resolve_unaligned( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + ) -> Result { + Self::resolve_inner(objects, timestamps, false) + } + fn resolve_inner( objects: Pin<&xarray::XArray>>, timestamps: &[uapi::drm_asahi_timestamps], From d782cba5bbbf08558dab3808bf9e3b9ff34b4ed6 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 04:18:58 -0500 Subject: [PATCH 009/117] drm/asahi: build the G18 backend for the M6 Add CONFIG_DRM_ASAHI_G18. With it, the module registers an "aurora-g18" platform driver for "apple,agx-t8152", as G16 does for its SoC. Its probe: - first checks the G18 parameters; - maps SGX and allocates the DRM device and DMA binding; - starts the firmware session the parameters select; - at the application stage, registers the render node over it; - in a diagnostic stage, keeps the firmware owner without one. The firmware session is retained until reboot, so manual unbind is suppressed. --- drivers/gpu/drm/asahi/Kconfig | 13 +++++++ drivers/gpu/drm/asahi/asahi_core.rs | 7 ++++ drivers/gpu/drm/asahi/driver.rs | 57 +++++++++++++++++++++++++++++ drivers/gpu/drm/asahi/hw/mod.rs | 2 + 4 files changed, 79 insertions(+) diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 6b28266b6b4d1..c915603abeabd 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -43,6 +43,19 @@ config DRM_ASAHI_G16 An aborted receipt disables that command class until reboot. Suspend, hibernation and kexec are blocked while native graphics owns DMA. +config DRM_ASAHI_G18 + bool "Apple M6 (G18G, T8152) support (experimental)" + depends on DRM_ASAHI && ARM64 + select WANT_DEV_COREDUMP + help + Backend for the Mac mini M6 (J873) GPU and its 26A428 firmware. + The GPU is started only when the boot loader reports the matching + firmware images and enables both GFX coprocessor mailboxes; the + firmware session and its memory are then retained until reboot. + Submission is synchronous: every command retires before the next + one starts. The module parameters prefixed g18_ select the bring-up + stage and the admitted command envelope. + config DRM_ASAHI_DEBUG_ALLOCATOR bool "Use debug allocator" depends on DRM_ASAHI diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index decf16ea09647..0003c2be108e2 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -22,6 +22,8 @@ mod gem; #[cfg(CONFIG_DRM_ASAHI_G16)] mod g16; +#[cfg(CONFIG_DRM_ASAHI_G18)] +mod g18; mod gpu; mod hw; mod initdata; @@ -78,6 +80,9 @@ struct DriverModule { _g15: G15Registration, // Field drop order unregisters/joins driver producers before queue teardown. _g16: Option>>, + #[cfg(CONFIG_DRM_ASAHI_G18)] + #[pin] + _g18: kernel::driver::Registration>, #[pin] _driver: kernel::driver::Registration>, _cleanup: cleanup::QueueOwner, @@ -99,6 +104,8 @@ impl kernel::InPlaceModule for DriverModule { module, ), _g16: register_g16(module)?, + #[cfg(CONFIG_DRM_ASAHI_G18)] + _g18 <- kernel::driver::Registration::new(g18::probe::DRIVER_NAME, module), // All fallible fields are initialized. This final unit initializer cannot fail. #[cfg(CONFIG_DRM_ASAHI_G15)] _g15_ready: g15::activate(), diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index ff82d8c118e85..3df2c98aa9af4 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -481,6 +481,63 @@ impl AsahiPlatformDriver { } } +#[cfg(CONFIG_DRM_ASAHI_G18)] +impl AsahiPlatformDriver { + /// Probe the M6 GPU through the G18 backend. + /// + /// The parameter gate refuses before any MMIO, DMA, mailbox or processor access. A + /// diagnostic bring-up stage retains its firmware owner without a DRM device; the + /// application stage registers the render node over the started session. + pub(crate) fn probe_g18<'bound>( + pdev: &'bound platform::Device>, + soc: &'static hw::t8152::SocConfig, + ) -> Result> { + crate::g18::probe::admit_t8152(pdev)?; + let resources = regs::Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: The G18 adapter closes the binding in unbind before devres; probe errors + // drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; + let started = crate::g18::probe::probe_t8152(pdev, &drm, soc)?; + let Some(gpu) = started.gpu else { + return Ok(crate::g18::probe::G18Bound { + runtime: started.runtime, + binding: Some(binding), + driver: None, + }); + }; + let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); + // SAFETY: `reg` is stored in `AsahiPlatformDriverData` and dropped when the driver is + // unbound; it is never forgotten. + let reg = match unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0) } { + Ok(reg) => reg, + Err(error) => { + started.runtime.shutdown(); + return Err(error); + } + }; + dev_info!( + pdev.as_ref(), + "G18_DRM registered render node (aurora, apple,agx-t8152)\n" + ); + Ok(crate::g18::probe::G18Bound { + runtime: started.runtime, + binding: None, + driver: Some(AsahiPlatformDriverData { + binding, + _device: reg.device().into(), + _reg: reg, + g17: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, + raw_dev: core::ptr::null_mut(), + raw_stats: core::ptr::null_mut(), + stats: None, + }), + }) + } +} + impl AsahiPlatformDriverData<'_> { pub(crate) fn close_binding(&self) { self.binding.close(); diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index c54ffb5c208fe..438d071de971c 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -19,6 +19,8 @@ mod t604x; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8140; +#[cfg(CONFIG_DRM_ASAHI_G18)] +pub(crate) mod t8152; /// GPU generation enumeration. Note: Part of the UABI. #[derive(Debug, PartialEq, Copy, Clone)] From 3abc8618d0102343d0aaea091a7457d52e05a3df Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 04:18:58 -0500 Subject: [PATCH 010/117] arm64: dts: apple: t8152-j873g: add the G18G GPU Describe the M6 GPU in t8152-j873-gpu.dtsi: - the "apple,agx-t8152" node with its SGX window and PSD service blocks; - its eight reserved regions; - the GFX and GFX1 ASCWrap v8 mailboxes. Include it from t8152-j873g.dts. The reserved regions and the mailboxes stay disabled. A boot loader fills them from this boot's ADT and enables them, then writes the /chosen firmware receipt the driver checks. --- arch/arm64/boot/dts/apple/t8152-j873-gpu.dtsi | 151 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8152-j873g.dts | 2 + 2 files changed, 153 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8152-j873-gpu.dtsi diff --git a/arch/arm64/boot/dts/apple/t8152-j873-gpu.dtsi b/arch/arm64/boot/dts/apple/t8152-j873-gpu.dtsi new file mode 100644 index 0000000000000..2166be04d4111 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8152-j873-gpu.dtsi @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple Mac mini (M6), J873: G18G GPU resources of the T8152. + * + * Addresses and sizes follow the 26A428 ADT /arm-io/sgx, gfx-asc and + * gfx1-asc nodes. This only describes the hardware; it does not start the + * firmware or change the GPU's power state. + * + * Unless a loader prepares the boot, nothing here starts the GPU: + * - the driver's G18 backend binds "apple,agx-t8152", but refuses with + * ENODEV before any firmware access unless the loader's /chosen + * aurora,g18g-gpu-firmware-profile receipt matches, and before any MMIO + * with aurora.g18_initdata_test=0; + * - the eight reserved regions are boot-local. They stay disabled with a + * zero reg until the loader copies this boot's /arm-io/sgx base and size + * values into them and sets status = "okay" (as m1n1 dt_set_resvmem() + * does). The loader finds them by these node names; + * - the four firmware segment carveouts (GFX/GFX1 __DATA and + * __DATA_SHARED_RO) are no-map, so they never enter the linear map. The + * driver only takes their resources, and maps __DATA read-only to scan + * for a crash log. The four UAT regions stay mapped: the driver uses + * their pages as System RAM; + * - the GFX and GFX1 ASCWrap v8 mailboxes stay disabled. The loader enables + * both together with the /chosen aurora,g18g-gpu-firmware-profile + * property, and only after it has checked the loaded firmware images. + * Even then the provider only maps its windows: ASC CPU control is + * refused unless apple_mailbox.ascwrap_v8_cpu_control=1 is given, and the + * receive side is polled, with no interrupt enabled, unless + * apple_mailbox.ascwrap_v8_rx_irq=1 is given. + * + * No power-domain is attached. The GPU is powered by PMGR2 PSD services 3 + * (sgx), 81 (gfx-asc) and 82 (gfx1-asc). Their 0x100-byte service blocks + * are listed for state reads (state +0x80, ack +0xbc). + */ + +/* Loaders locate the node through this alias (m1n1 dt_set_gpu()). */ +&{/aliases} { + gpu = &gpu; +}; + +&{/reserved-memory} { + /* sgx gpu-region */ + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + status = "disabled"; + }; + + /* sgx gfx-shared-region (0x80000) */ + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + status = "disabled"; + }; + + /* sgx gfx-handoff */ + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + status = "disabled"; + }; + + /* sgx gfx-shared-l2-region */ + uat_l2_pagetables: uat-l2-pagetables { + reg = <0 0 0 0>; + status = "disabled"; + }; + + /* GFX __DATA (0xbc000 on 26A428) */ + gfx_data: gfx-data { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + /* GFX1 __DATA (0xb4000 on 26A428) */ + gfx1_data: gfx1-data { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + /* GFX __DATA_SHARED_RO (0x8000, firmware VA +0x10c000), never host-writable */ + gfx_data_shared_ro: gfx-data-shared-ro { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; + + /* GFX1 __DATA_SHARED_RO (0x8000, firmware VA +0x108000), never host-writable */ + gfx1_data_shared_ro: gfx1-data-shared-ro { + reg = <0 0 0 0>; + no-map; + status = "disabled"; + }; +}; + +&{/soc} { + gpu: gpu@580000000 { + compatible = "apple,agx-t8152"; + /* + * ADT sgx reg[0]. ADT reg[1] (0x580d00000/0x180000) lies inside + * it and is reached through this window. The PSD service blocks + * follow. + */ + reg = <5 0x80000000 0 0x4000000>, + <3 0x03a10300 0 0x100>, + <3 0x03a15100 0 0x100>, + <3 0x03a15200 0 0x100>; + reg-names = "sgx", "psd-sgx", "psd-gfx-asc", "psd-gfx1-asc"; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, + <&uat_l2_pagetables>, <&gfx_data>, <&gfx1_data>, + <&gfx_data_shared_ro>, <&gfx1_data_shared_ro>; + memory-region-names = "ttbs", "pagetables", "handoff", "shared-l2", + "gfx-data", "gfx1-data", + "gfx-data-shared-ro", "gfx1-data-shared-ro"; + mboxes = <&gfx_asc>, <&gfx1_asc>; + }; + + /* + * ADT gfx-asc: iop,ascwrap-v8, msgbox-mailbox-num = 1, alias, + * postoffice. Only this role's 0x80-byte post-office window + * (0x316020000 + (2 * 1 + 0) * 0x80) is claimed; the rest of the + * 0x316020000 bank belongs to PMP, DCP, DCPEXT and SCALER0. + * Interrupt index 0 is send-empty and index 1 recv-not-empty, as in + * the macOS ASCWrap v8 driver. The provider polls by default. + */ + gfx_asc: mailbox@582600000 { + compatible = "apple,t8152-gfx-ascwrap-v8"; + reg = <5 0x82600000 0 0x74000>, <5 0x82050000 0 8>, + <3 0x16020100 0 0x80>, <3 0x03a15100 0 0x100>; + reg-names = "wrapper", "iop-vbar", "mailbox", "psd"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + #mbox-cells = <0>; + status = "disabled"; + }; + + /* ADT gfx1-asc: msgbox-mailbox-num = 2, window 0x316020200. */ + gfx1_asc: mailbox@582e00000 { + compatible = "apple,t8152-gfx-ascwrap-v8"; + reg = <5 0x82e00000 0 0x74000>, <5 0x82850000 0 8>, + <3 0x16020200 0 0x80>, <3 0x03a15200 0 0x100>; + reg-names = "wrapper", "iop-vbar", "mailbox", "psd"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX1"; + #mbox-cells = <0>; + status = "disabled"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts index 0a0818d85d085..6c2e25a991e61 100644 --- a/arch/arm64/boot/dts/apple/t8152-j873g.dts +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -248,3 +248,5 @@ }; }; }; + +#include "t8152-j873-gpu.dtsi" From bcf870bb5971643ee6ae511ed679cf3f713438d6 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Tue, 27 Oct 2026 22:04:33 +0100 Subject: [PATCH 011/117] dt-bindings: nvme: apple,nvme-ans: Add T8142 compatible T8142 (M5) ANS2 behaves like T8140: separate NVMe (non-secure alias) and NVMMU windows, post-M4 queue setup, three power domains. Allow apple,t8142-nvme-ans2 with apple,t8140-nvme-ans2 as fallback and apply the three-region reg rule to the T8140 family as well. Signed-off-by: Thomas Glanzmann --- Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml index 85f0fd2b6fa1e..93388178e330e 100644 --- a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml +++ b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml @@ -26,6 +26,9 @@ properties: - items: - const: apple,t8140-nvme-ans2 - const: apple,t8132-nvme-ans2 + - items: + - const: apple,t8142-nvme-ans2 + - const: apple,t8140-nvme-ans2 - items: - enum: # Do not add additional SoC to this list. @@ -146,6 +149,7 @@ allOf: enum: - apple,t6041-nvme-ans2 - apple,t8132-nvme-ans2 + - apple,t8140-nvme-ans2 then: properties: reg: From 0dd9b105d76048a36bcac2eed279de1e20c3bfdd Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Tue, 27 Oct 2026 22:04:33 +0100 Subject: [PATCH 012/117] dt-bindings: iommu: apple,sart: Add T8142 compatible The T8142 (M5) SART is the coastguard SART with sart-version 3, the same layout as T6000. Signed-off-by: Thomas Glanzmann --- Documentation/devicetree/bindings/iommu/apple,sart.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/apple,sart.yaml b/Documentation/devicetree/bindings/iommu/apple,sart.yaml index a75b4c8f05d5b..47fb67a6906df 100644 --- a/Documentation/devicetree/bindings/iommu/apple,sart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,sart.yaml @@ -37,6 +37,7 @@ properties: - apple,t8122-sart - apple,t8132-sart - apple,t8140-sart + - apple,t8142-sart - const: apple,t6000-sart - enum: - apple,t6000-sart From 44e3ab566eb6fa281094bfece4e92d9db4748273 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:19:01 +0100 Subject: [PATCH 013/117] nvme: apple: add T8142 (M5) The J813 (MacBook Air 13-inch, M5) ANS2 is laid out like T8140's: LSQ NVMMU, separate NVMMU, IOQ registers, 64-deep queues and broken FUA. As on every post-M4 controller the final shutdown sends SHN. aura (the J813 boot loader) leaves the controller and its RTKit session running for Linux to adopt. Signed-off-by: Thomas Glanzmann --- drivers/nvme/host/apple.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 05f73eb1dd5e9..e4c9a1ccd38a0 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2728,6 +2728,18 @@ static const struct apple_nvme_hw apple_nvme_t8140_hw = { .quirks = NVME_QUIRK_BROKEN_FUA, }; +/* + * T8142 (M5) is laid out like T8140; the final shutdown sends SHN, as on + * every post-M4 controller. + */ +static const struct apple_nvme_hw apple_nvme_t8142_hw = { + .has_lsq_nvmmu = true, + .has_separate_nvmmu = true, + .needs_ioq_registers = true, + .max_queue_depth = 64, + .quirks = NVME_QUIRK_BROKEN_FUA, +}; + /* * T6040 (M4 Pro) with the 26.x ANS firmware: the T8132 controller, but the * "nvme" register block ends at 0x10000 and the linear submission queue @@ -2748,6 +2760,7 @@ static const struct of_device_id apple_nvme_of_match[] = { { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, { .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw }, { .compatible = "apple,t8140-nvme-ans2", .data = &apple_nvme_t8140_hw }, + { .compatible = "apple,t8142-nvme-ans2", .data = &apple_nvme_t8142_hw }, { .compatible = "apple,nvme-ans2", .data = &apple_nvme_t8103_hw }, {}, }; From fe84f4c6b970e30acec7f6964b544995a15245ae Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:21:37 +0100 Subject: [PATCH 014/117] arm64: dts: apple: t8142-j813: E and P states up to the ADT's top levels The J813 ADT lists E-cluster levels up to 2964 MHz (8) and P-cluster levels 11-20 above the 3516 MHz this tree stopped at. Add them; level 19 (4416 MHz) is below level 18 and left out. The SoC throttles the top levels when hot. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813.dts | 77 ++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 576c4637f37df..61f77d206af7f 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -180,6 +180,36 @@ opp-hz = /bits/ 64 <972000000>; opp-level = <2>; }; + + opp-1152000000 { + opp-hz = /bits/ 64 <1152000000>; + opp-level = <3>; + }; + + opp-1584000000 { + opp-hz = /bits/ 64 <1584000000>; + opp-level = <4>; + }; + + opp-1992000000 { + opp-hz = /bits/ 64 <1992000000>; + opp-level = <5>; + }; + + opp-2352000000 { + opp-hz = /bits/ 64 <2352000000>; + opp-level = <6>; + }; + + opp-2700000000 { + opp-hz = /bits/ 64 <2700000000>; + opp-level = <7>; + }; + + opp-2964000000 { + opp-hz = /bits/ 64 <2964000000>; + opp-level = <8>; + }; }; pcluster_opp: opp-table-1 { @@ -235,6 +265,53 @@ opp-hz = /bits/ 64 <3516000000>; opp-level = <10>; }; + + /* ADT levels 11-20, measured on J813 (level 19, 4416 MHz, is below + * level 18 and left out). The SoC throttles them when hot. */ + opp-3720000000 { + opp-hz = /bits/ 64 <3720000000>; + opp-level = <11>; + }; + + opp-3876000000 { + opp-hz = /bits/ 64 <3876000000>; + opp-level = <12>; + }; + + opp-4020000000 { + opp-hz = /bits/ 64 <4020000000>; + opp-level = <13>; + }; + + opp-4176000000 { + opp-hz = /bits/ 64 <4176000000>; + opp-level = <14>; + }; + + opp-4248000000 { + opp-hz = /bits/ 64 <4248000000>; + opp-level = <15>; + }; + + opp-4272000000 { + opp-hz = /bits/ 64 <4272000000>; + opp-level = <16>; + }; + + opp-4332000000 { + opp-hz = /bits/ 64 <4332000000>; + opp-level = <17>; + }; + + opp-4428000000 { + opp-hz = /bits/ 64 <4428000000>; + opp-level = <18>; + }; + + opp-4464000000 { + opp-hz = /bits/ 64 <4464000000>; + opp-level = <20>; + }; }; clkref: clock-ref { From 47fcf6c5123bd8fdf53fbe26fd853852c9ed75d2 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:21:50 +0100 Subject: [PATCH 015/117] arm64: dts: apple: t8142-j813: PMC voters and the CLPC The T8142 PMC (AppleT8142PMC, ADT pmc-pmgr 31 + 0x700000) takes DRAM and fabric floors per voter agent: PMP 0, CLPC 1, GFX 2, AP 3. iBoot leaves every rail enable at 0, so no vote is honoured and DRAM and the fabric stay at their lowest states (CPU DRAM latency 171 ns instead of 70 ns). Describe the PMC with its static voters and the CLPC agent. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813.dts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 61f77d206af7f..f1378b8651878 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -339,6 +339,30 @@ #performance-domain-cells = <0>; }; + /* + * PMC voter interface (AppleT8142PMC, ADT pmc-pmgr 31 + 0x700000): + * floors at 0x382700000 + agent * 0x4000, per-agent states at + * +0x106000, rail enables at +0x108000. Agents (ADT voter-agent): + * PMP 0, CLPC 1, GFX 2, AP 3. iBoot leaves every enable at 0, so + * no vote is honoured and DRAM and fabric stay at their lowest + * states (CPU DRAM latency 171 ns instead of 70 ns). + */ + pmc: power-management@382700000 { + compatible = "apple,t8142-pmc"; + reg = <0x3 0x82700000 0x0 0x10000>, + <0x3 0x82806000 0x0 0x3000>; + reg-names = "voters", "rails"; + #address-cells = <1>; + #size-cells = <0>; + /* PMP, CLPC and AP agents */ + apple,static-voters = <0 1 3>; + + clpc: performance-controller@1 { + compatible = "apple,t8142-clpc"; + reg = <1>; + }; + }; + aic: interrupt-controller@381000000 { compatible = "apple,t8142-aic3", "apple,t8122-aic3"; interrupt-controller; From 9bb8282dfdbbd74cd13c7b42d78ff10481aac37f Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:21:50 +0100 Subject: [PATCH 016/117] arm64: dts: apple: t8142-j813: Describe the panel The 13.6" 2560x1664 panel (1600 lines below the notch) with a 500 nit peak (ADT backlight known-nits). The panel node makes the connector eDP and gives the DCP a backlight to drive. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-display.dtsi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi index 9bd549a5784cc..d9e890af7fd44 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi @@ -91,5 +91,18 @@ iommus = <&disp0_dart 16>; phandle = <&disp0_piodma>; }; + + /* + * 13.6" 2560x1664 (1600 below the notch); 500 nits peak (ADT + * backlight known-nits). Makes the connector eDP and gives the + * DCP a backlight to drive. + */ + panel: panel { + compatible = "apple,panel-j813", "apple,panel"; + width-mm = <290>; + height-mm = <188>; + adj-height-mm = <181>; + apple,max-brightness = <500>; + }; }; }; From 34ffeb034acd7b1ff8d08b7edbae769f1885c400 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:21:50 +0100 Subject: [PATCH 017/117] arm64: dts: apple: t8142-j813: QSPI NVRAM, the PMP and the trackpad reset - The pmgr block is 0x14000 long (the ISP and keyboard backlight power states sit beyond 0x4000). - The QSPI NOR flash with the NVRAM partitions (asahi-nvram, asahi-bless), its 200 MHz clock and the spi_p/qspi power states. - The PMP (DRAM and fabric DVFS coprocessor, registry-v2 firmware) with its DART, mailbox, power states and the J813 tunables. - The trackpad's AFE reset line (SMC GPIO 28, active low) instead of a firmware-managed reset. Signed-off-by: Thomas Glanzmann --- .../dts/apple/t8142-j813-peripherals.dtsi | 145 +++++++++++++++++- .../dts/apple/t8142-j813-pmp-tunables.dtsi | 70 +++++++++ 2 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-pmp-tunables.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi index a6708b1c972df..5ca6fed62568a 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi @@ -1,10 +1,20 @@ // SPDX-License-Identifier: GPL-2.0+ OR MIT +#include #include / { aliases { mtp = &mtp; keyboard = &keyboard; + nvram = &nvram; + }; + + /* J813 ADT /arm-io/qspi clock-ids [396]; rate as the T8140 QSPI. */ + clk_qspi: clock-qspi { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_qspi"; }; led-controller { @@ -24,7 +34,8 @@ &{/soc} { pmgr: power-management@380700000 { compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; - reg = <3 0x80700000 0 0x4000>; + /* As on T8140: up to the ISP17a slots at +0x4000..+0x4060. */ + reg = <3 0x80700000 0 0x14000>; #address-cells = <1>; #size-cells = <1>; @@ -35,6 +46,122 @@ #reset-cells = <0>; label = "fpwm0"; }; + + /* ADT pmgr devices: SPI_P id 94 (+0x390), QSPI id 120 (+0x460). */ + ps_spi_p: power-controller@390 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + + /* ADT pmgr devices: PMP id 159 (+0x598), PMS_SRAM id 160 (+0x5a0). */ + ps_pmp: power-controller@598 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x598 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmp"; + /* The PMP's SRAM, which the driver reads first. */ + power-domains = <&ps_pms_sram>; + }; + + ps_pms_sram: power-controller@5a0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_sram"; + /* T8140 holds it through its always-on PMP report node. */ + apple,always-on; + }; + + ps_qspi: power-controller@460 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x460 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + }; + }; + + /* + * PMP, laid out as on T8140 (ADT /arm-io/pmp, dart-pmp): it runs the + * DRAM controller (DCS) and fabric DVFS. Without it the memory stays at + * the bootloader's state (memcpy ~20 GB/s against ~120 GB/s on macOS). + */ + pmp_dart: iommu@380300000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x3 0x80300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000000000 0xbff00000>; + power-domains = <&ps_pmp>; + }; + + pmp: pmp@380500000 { + compatible = "apple,t8142-pmp-v2", "apple,t8140-pmp-v2"; + reg = <0x3 0x80500000 0x0 0xd0000>, + <0x3 0x80e00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + /* ADT /chosen board-id, dram-vendor-id */ + apple,board-id = <36>; + apple,dram-vendor-id = <1>; + /* ADT /arm-io/pmgr pmc, pmc-pmgr (registry-v2 boot arguments) */ + apple,pmc = <1>; + apple,pmc-pmgr = <31>; + }; + + pmp_mbox: mailbox@380e08000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x80e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>; + }; + + /* ADT /arm-io/qspi: 0x195118000 and its FIFO window at +0x800, IRQ 1245. */ + qspi: spi@3a5118000 { + compatible = "apple,t8142-qspi", "apple,t8140-qspi"; + reg = <0x3 0xa5118000 0x0 0x4800>; + clocks = <&clk_qspi>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_qspi>; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + /* Conservative until the input clock is confirmed. */ + spi-max-frequency = <10000000>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + /* ADT spinor/anvram reg: 0x100000 bytes at 0x700000. */ + nvram: partition@700000 { + label = "nvram"; + reg = <0x700000 0x100000>; + }; + }; + }; }; fpwm0: pwm@3a5040000 { @@ -66,6 +193,12 @@ compatible = "apple,smc-hwmon"; apple,discover-temperatures; }; + + smc_gpio: gpio { + compatible = "apple,smc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; }; mtp: mtp@394600000 { @@ -121,13 +254,20 @@ interrupt-names = "tx", "rx"; apple,fifo-size = <0x800>; apple,helper-cpu = <&mtp>; + /* + * ADT dockchannel-mtp/mtp-transport/multi-touch: + * function-afe-reset = smc-pmu pKW4('gP1c'), SMC GPIO 28. + * The MTP firmware cannot reach the AFE on its own ("No + * HINT_L"): after macOS left it running the trackpad + * worked, after a Linux warm reboot it stayed in reset. + */ + apple,afe-reset-gpios = <&smc_gpio 28 GPIO_ACTIVE_LOW>; stm { }; multi-touch { firmware-name = "apple/tpmtfw-j813.bin"; apple,power-method = <2>; - apple,firmware-managed-reset; }; /* Layout is filled by the loader using this machine's own data. */ @@ -135,3 +275,4 @@ }; }; }; +#include "t8142-j813-pmp-tunables.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8142-j813-pmp-tunables.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-pmp-tunables.dtsi new file mode 100644 index 0000000000000..fa34d4d80fe7d --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-pmp-tunables.dtsi @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J813 PMP firmware properties: the ADT /arm-io/pmp/iop-pmp-nub values the + * firmware queries by name at boot (pmp.rs serves apple,tunable-). + * Generated from the J813 ADT by tools/adtnub.py; raw ADT bytes. + */ + +&pmp { + apple,tunable-agx-dcs-bw-constraint = [00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00]; + apple,tunable-mc_quota_req_ptd_count = [08 00 00 00]; + apple,tunable-mc_cmd_ptd_count = [02 00 00 00]; + apple,tunable-ane-dcs-bw-dvfs-filter = [ff ff 1f 00]; + apple,tunable-sochot-sensors = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 15 00 00 00 16 00 00 00 17 00 00 00 18 00 00 00]; + apple,tunable-soc-ni3-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-agx-af-rd-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 35 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 34 00 00 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 00 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 00 00 00 00 00 00 3a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 39 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 2d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 2c 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 11 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3c 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 33 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 7a 00 00 00 00 00 00 00 32 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 79 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 2c 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 8a 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 89 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3e 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 6f 00 00 00 00 00 00 00 3d 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 6e 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 34 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 33 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2c 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 41 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 40 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3c 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 37 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 36 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00]; + apple,tunable-ane-af-bw-dvfs-filter = [ff ff 1f 00]; + apple,tunable-ans-af-bw-threshold = [70 3d 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 70 3d 0b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 cc 0c 1c 00 00 00 00 00]; + apple,tunable-ptd-range = [01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 50 54 44 2d 4e 55 4c 4c 00 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 01 00 00 00 0b 00 00 00 50 4d 50 2d 53 54 41 54 55 53 00 00 00 00 00 00 03 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 44 56 46 53 2d 53 54 41 54 45 00 00 00 00 00 00 04 00 00 00 06 00 00 00 04 00 00 00 00 00 00 00 50 4d 50 54 4f 4f 4c 00 00 00 00 00 00 00 00 00 05 00 00 00 98 00 00 00 08 00 00 00 00 00 00 00 50 4d 53 2d 50 4d 47 52 2d 50 4b 54 00 00 00 00 06 00 00 00 a0 00 00 00 08 00 00 00 00 00 00 00 50 4d 53 2d 44 50 45 2d 50 4b 54 00 00 00 00 00 07 00 00 00 28 02 00 00 18 00 00 00 00 00 00 00 50 4d 53 2d 44 45 56 43 2d 50 4b 54 00 00 00 00 09 00 00 00 a8 00 00 00 80 01 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 4b 54 00 00 00 00 00 0a 00 00 00 27 00 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 52 45 51 00 00 0b 00 00 00 2f 00 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 41 43 4b 00 00 0c 00 00 00 37 00 00 00 10 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 44 56 46 53 00 00 00 00 0d 00 00 00 47 00 00 00 0a 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 42 57 52 00 00 00 00 00 0e 00 00 00 13 00 00 00 0a 00 00 00 00 00 00 00 4d 43 2d 44 52 56 2d 54 4f 2d 43 54 52 4c 00 00 0f 00 00 00 51 00 00 00 01 00 00 00 00 00 00 00 44 56 46 53 2d 43 45 49 4c 00 00 00 00 00 00 00 14 00 00 00 0a 00 00 00 01 00 00 00 00 00 00 00 41 56 44 2d 4e 52 47 00 00 00 00 00 00 00 00 00 15 00 00 00 0b 00 00 00 01 00 00 00 00 00 00 00 53 4f 43 2d 41 4f 4e 2d 4e 52 47 00 00 00 00 00 16 00 00 00 0c 00 00 00 01 00 00 00 00 00 00 00 53 4f 43 2d 52 45 53 54 2d 4e 52 47 00 00 00 00 1e 00 00 00 1d 00 00 00 09 00 00 00 00 00 00 00 4d 43 2d 43 54 52 4c 2d 54 4f 2d 44 52 56 00 00 1f 00 00 00 0f 00 00 00 02 00 00 00 0c 00 00 00 46 41 53 54 2d 44 49 45 2d 43 45 00 00 00 00 00 20 00 00 00 52 00 00 00 07 00 00 00 0d 00 00 00 53 4d 43 2d 54 4f 2d 50 50 4d 00 00 00 00 00 00 21 00 00 00 0d 00 00 00 01 00 00 00 0a 00 00 00 4c 54 53 2d 56 4f 4c 54 2d 43 41 50 00 00 00 00 40 00 00 00 59 00 00 00 05 00 00 00 0e 00 00 00 50 50 4d 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 41 00 00 00 5e 00 00 00 05 00 00 00 0f 00 00 00 43 4c 50 43 2d 54 4f 2d 53 4d 43 00 00 00 00 00 43 00 00 00 11 00 00 00 02 00 00 00 00 00 00 00 46 41 53 54 2d 44 49 45 2d 54 4f 2d 53 4d 43 00 44 00 00 00 26 00 00 00 01 00 00 00 00 00 00 00 4d 45 4d 43 2d 54 4f 2d 53 4d 43 00 00 00 00 00 52 00 00 00 0e 00 00 00 01 00 00 00 00 00 00 00 4c 54 53 2d 54 4f 2d 47 46 58 00 00 00 00 00 00]; + apple,tunable-sram-index = [01 00 00 00]; + apple,tunable-no-shutdown = [01 00 00 00]; + apple,tunable-dvfs-domain = [01 00 00 00 08 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 53 4f 43 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 06 00 00 00 00 00 00 00 44 49 53 50 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-dvfs-disp-min-state = [01 00 00 00]; + apple,tunable-fast-die-ctrl-ce-map = [01 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 45 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 5f 41 4e 45 00 00 00 00 00 00 00 00]; + apple,tunable-ane-dcs-bw-constraint = [00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00]; + apple,tunable-soc-ni2-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-agx-af-wr-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00]; + apple,tunable-user-power-managed = [01 00 00 00]; + apple,tunable-ane-af-bw-threshold = [00 00 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 1c 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 09 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 06 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 16 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 41 00 00 00 00 00 00 00 15 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 40 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3f 00 00 00 00 00 00 00 12 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 3e 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3a 00 00 00 00 00 00 00 0c 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 39 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3a 00 00 00 00 00 00 00 0c 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 39 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 31 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 1e 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 0e 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 08 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 02 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 02 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00]; + apple,tunable-lts-voltage = [01 00 00 00 03 00 00 00 03 00 00 00 01 00 00 00 45 41 43 43 00 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 02 00 00 00 50 41 43 43 00 00 00 00 03 00 00 00 01 00 00 00 06 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 04 00 00 00 03 00 00 00 05 00 00 00 04 00 00 00 41 47 58 00 00 00 00 00]; + apple,tunable-ane-dcs-bw-threshold = [a3 f0 13 00 01 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 02 00 00 00 a3 f0 12 00 00 00 00 00 1e 05 3a 00 03 00 00 00 66 e6 26 00 00 00 00 00 0a 57 4a 00 04 00 00 00 1e 05 39 00 00 00 00 00 3d 0a 57 00 05 00 00 00 0a 57 49 00 00 00 00 00 3d 8a 65 00 06 00 00 00 3d 0a 56 00 00 00 00 00 5c 0f 74 00 07 00 00 00 3d 8a 64 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 00 00 00 00 a3 f0 13 00 01 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 02 00 00 00 a3 f0 12 00 00 00 00 00 1e 05 3a 00 03 00 00 00 66 e6 26 00 00 00 00 00 0a 57 4a 00 04 00 00 00 1e 05 39 00 00 00 00 00 3d 0a 57 00 05 00 00 00 0a 57 49 00 00 00 00 00 3d 8a 65 00 06 00 00 00 3d 0a 56 00 00 00 00 00 5c 0f 74 00 07 00 00 00 3d 8a 64 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 00 00 00 00 a3 f0 13 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 a3 f0 12 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 00 00 0e 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 00 00 09 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 00 00 09 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 1e 05 3a 00 00 00 00 00 00 00 0e 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 1e 05 3a 00 00 00 00 00 00 00 0e 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 04 00 06 00 00 00 0a 57 4a 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 57 4a 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 15 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 57 4a 00 00 00 00 00 00 00 14 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 00 00 06 00 00 00 3d 0a 57 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 00 00 06 00 00 00 3d 0a 57 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00]; + apple,tunable-bwr-catch-up-factor = [00 00 01 00]; + apple,tunable-controller = [01 00 00 00 1f 00 00 00 43 00 00 00 00 00 00 00 00 00 00 00 44 49 45 2d 54 45 4d 50 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 46 49 2d 54 4f 50 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 2d 52 44 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 2d 57 52 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 10 00 00 00 0e 00 00 00 1e 00 00 00 00 00 00 00 00 00 00 00 4d 45 4d 2d 43 41 43 48 45 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 11 00 00 00 52 00 00 00 21 00 00 00 00 00 00 00 00 00 00 00 4c 49 46 45 54 49 4d 45 2d 53 45 52 56 4f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-soc-device-ps-group = [00 fc ff 1f 00 00 00 00]; + apple,tunable-agx-dcs-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 27 00 00 00 00 00 00 00 12 00 00 00 00 00 00 00 39 00 00 00 00 00 00 00 26 00 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 38 00 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 11 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 27 00 00 00 00 00 00 00 10 00 06 00 00 00 00 00 39 00 00 00 00 00 00 00 26 00 06 00 00 00 00 00 4a 00 00 00 00 00 00 00 38 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 49 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 27 00 00 00 00 00 00 00 0b 00 06 00 00 00 00 00 39 00 00 00 00 00 00 00 26 00 06 00 00 00 00 00 47 00 00 00 00 00 00 00 38 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 46 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 37 00 00 00 00 00 00 00 21 00 06 00 00 00 00 00 47 00 00 00 00 00 00 00 36 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 46 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 1f 00 00 00 00 00 00 00 14 00 06 00 00 00 00 00 47 00 00 00 00 00 00 00 1e 00 06 00 00 00 00 00 57 00 00 00 00 00 00 00 46 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 56 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 07 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 57 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 65 00 00 00 00 00 00 00 56 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 64 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 17 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 43 00 00 00 00 00 00 00 16 00 06 00 00 00 00 00 65 00 00 00 00 00 00 00 42 00 06 00 00 00 00 00 74 00 00 00 00 00 00 00 64 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2d 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 50 00 00 00 00 00 00 00 2c 00 06 00 00 00 00 00 74 00 00 00 00 00 00 00 4f 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 10 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 3c 00 00 00 00 00 00 00 0f 00 06 00 00 00 00 00 74 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 1e 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00]; + apple,tunable-soc-ni1-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-mc_quota_rsp_ptd_count = [08 00 00 00]; + apple,tunable-lts-ctrl-ts = [00 00 05 00]; + apple,tunable-temp-sensor = [01 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 80 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 30 69 00 00 00 00 00 00 00 00 00 00 02 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 33 b3 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 31 69 00 00 00 00 00 00 00 00 00 00 03 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 66 e6 03 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 32 69 00 00 00 00 00 00 00 00 00 00 04 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 80 05 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 35 69 00 00 00 00 00 00 00 00 00 00 05 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 66 e6 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 36 69 00 00 00 00 00 00 00 00 00 00 06 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 cc 4c 05 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 37 69 00 00 00 00 00 00 00 00 00 00 07 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 30 69 00 00 00 00 00 00 00 00 00 00 08 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 31 69 00 00 00 00 00 00 00 00 00 00 09 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 33 69 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 34 69 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 01 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6c 69 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 00 00 01 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6d 69 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 05 00 00 00 00 00 00 00 33 33 03 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6e 69 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 06 00 00 00 00 00 00 00 66 e6 02 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6f 69 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 07 00 00 00 00 00 00 00 99 99 02 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 70 69 00 00 00 00 00 00 00 00 00 00 10 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 09 00 00 00 00 00 00 00 33 33 02 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 72 69 00 00 00 00 00 00 00 00 00 00 11 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 39 69 00 00 00 00 00 00 00 00 00 00 12 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 61 69 00 00 00 00 00 00 00 00 00 00 13 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 36 69 00 00 00 00 00 00 00 00 00 00 14 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 37 69 00 00 00 00 00 00 00 00 00 00 15 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 66 66 08 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 37 69 00 00 00 00 00 00 00 00 00 00 16 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 80 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 31 69 00 00 00 00 00 00 00 00 00 00 17 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 99 99 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 32 69 00 00 00 00 00 00 00 00 00 00 18 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 33 b3 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 33 69 00 00 00 00 00 00 00 00 00 00 19 00 00 00 01 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6d 00 00 00 6d 00 00 00 4d 54 52 5f 54 4f 50 5f 41 47 58 00 00 00 00 00]; + apple,tunable-agx-af-wr-bw-dvfs-filter = [fe 7f 00 00]; + apple,tunable-dcs-bw-threshold = [a3 f0 13 00 01 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 02 00 00 00 a3 f0 12 00 00 00 00 00 1e 05 3a 00 03 00 00 00 66 e6 26 00 00 00 00 00 0a 57 4a 00 04 00 00 00 1e 05 39 00 00 00 00 00 3d 0a 57 00 05 00 00 00 0a 57 49 00 00 00 00 00 3d 8a 65 00 06 00 00 00 3d 0a 56 00 00 00 00 00 5c 0f 74 00 07 00 00 00 3d 8a 64 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 00 00 00 00]; + apple,tunable-mc_cmd_ack_ptd_count = [01 00 00 00]; + apple,tunable-soc-ni0-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-fast-die-ctrl-loop = [01 00 00 00 00 00 00 00 00 00 00 00 66 06 00 00 47 41 00 00 00 00 6e 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 01 00 16 00 00 00 15 00 00 00 16 00 00 00 17 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 45 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 66 06 00 00 47 41 00 00 00 00 6e 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 01 00 16 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 02 00 00 00 66 06 00 00 47 41 00 00 00 00 6e 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 01 00 16 00 00 00 01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-agx-dcs-bw-dvfs-filter = [fe 7f 00 00]; + apple,tunable-agx-af-rd-bw-dvfs-filter = [fe 7f 00 00]; + apple,tunable-firmware-name = [74 38 31 34 32 70 6d 70 00]; + apple,tunable-lts-clpc-voltage = [01 00 00 00 98 03 00 00 00 00 69 00 33 33 69 00 00 00 00 00 50 41 43 43 00 00 00 00]; + apple,tunable-soc-ni5-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-energy-counter = [01 00 00 00 01 00 00 00 05 00 00 00 01 00 00 00 41 47 58 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 05 00 00 00 02 00 00 00 41 47 58 2d 53 52 41 4d 00 00 00 00 00 00 00 00 03 00 00 00 01 00 00 00 06 00 00 00 01 00 00 00 41 4e 45 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 14 00 00 00 00 00 00 00 56 44 45 43 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 53 4f 43 5f 41 4f 4e 00 00 00 00 00 00 00 00 00]; + apple,tunable-lts-config = [01 00 00 00 3b 01 00 00 a6 72 0f 00 00 00 00 00 b5 d5 0f 00 6e 00 00 00 69 00 00 00 00 00 11 00 33 b3 00 00 b0 32 47 01 02 00 00 00 47 03 00 00 3b 61 00 00 00 00 00 00 f7 dd 0f 00 6e 00 00 00 69 00 00 00 00 00 11 00 33 b3 00 00 d7 43 ef 02]; + apple,tunable-agx-af-rd-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00]; + apple,tunable-ane-af-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00]; + apple,tunable-mcache-stream = [0e 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0d 00 00 00 38 00 00 00 38 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 11 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3c 00 00 00 3c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1b 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3d 00 00 00 3d 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 13 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 40 00 00 00 40 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 24 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 41 00 00 00 41 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1c 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 42 00 00 00 42 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1e 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2b 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0e 00 00 00 0e 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 05 00 00 00 01 00 00 00 01 00 00 00 07 00 00 00 07 00 00 00 20 00 00 00 20 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 06 00 00 00 02 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 24 00 00 00 24 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0b 00 00 00 05 00 00 00 01 00 00 00 0b 00 00 00 0b 00 00 00 30 00 00 00 30 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 01 00 00 00 11 00 00 00 11 00 00 00 48 00 00 00 48 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 13 00 00 00 13 00 00 00 50 00 00 00 50 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 15 00 00 00 15 00 00 00 58 00 00 00 5a 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 10 00 00 00 04 00 00 00 01 00 00 00 1f 00 00 00 1f 00 00 00 80 00 00 00 bf 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 04 00 00 00 02 00 00 00 10 00 00 00 10 00 00 00 44 00 00 00 44 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 20 00 00 00 03 00 00 00 02 00 00 00 03 00 00 00 06 00 00 00 10 00 00 00 1f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 1f 00 00 00 03 00 00 00 02 00 00 00 1e 00 00 00 1e 00 00 00 7c 00 00 00 7c 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 12 00 00 00 05 00 00 00 02 00 00 00 09 00 00 00 09 00 00 00 28 00 00 00 28 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0a 00 00 00 05 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 34 00 00 00 34 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 15 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5d 00 00 00 5d 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 36 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5e 00 00 00 5e 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 37 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5f 00 00 00 5f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 19 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 05 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 29 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 06 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 07 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0c 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 09 00 00 00 09 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 14 00 00 00 04 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 0a 00 00 00 0a 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 27 00 00 00 04 00 00 00 03 00 00 00 1d 00 00 00 1d 00 00 00 78 00 00 00 78 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2c 00 00 00 03 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 64 00 00 00 64 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2f 00 00 00 02 00 00 00 03 00 00 00 1a 00 00 00 1a 00 00 00 6c 00 00 00 6c 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 26 00 00 00 05 00 00 00 03 00 00 00 1c 00 00 00 1c 00 00 00 74 00 00 00 74 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 23 00 00 00 00 00 00 00 03 00 00 00 14 00 00 00 14 00 00 00 54 00 00 00 54 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 28 00 00 00 00 00 00 00 03 00 00 00 1b 00 00 00 1b 00 00 00 70 00 00 00 70 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 31 00 00 00 0f 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 60 00 00 00 60 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 30 00 00 00 0e 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 61 00 00 00 61 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 33 00 00 00 11 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 62 00 00 00 62 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 34 00 00 00 12 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 63 00 00 00 63 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 35 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 68 00 00 00 68 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 1d 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 69 00 00 00 6b 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2d 00 00 00 0c 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 65 00 00 00 65 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 16 00 00 00 0b 00 00 00 03 00 00 00 ff ff 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00]; + apple,tunable-ptd-driver-version = [01 00 00 00]; + apple,tunable-afi-top-bw-threshold = [99 19 3a 00 00 00 00 00 00 00 00 00 00 00 00 00 51 78 4d 00 01 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 51 78 4c 00 00 00 00 00 00 00 00 00 00 00 00 00 33 33 73 00 00 00 00 00 00 00 00 00 00 00 00 00 b8 1e 7c 00 00 00 00 00]; + apple,tunable-region-size = [00 00 0d 00 00 00 00 00]; + apple,tunable-coredump-enable = [40 00 00 00]; + apple,tunable-soc-ni4-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-pm-ptd-ranges = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 00 00 00 00 0f 00 00 00]; + apple,tunable-fast-die-ctrl-ts-avg-temp = [00 00 64 00]; + apple,tunable-lts-ctrl-loop = [02 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 02 00 00 00 01 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 47 50 55 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-soc-device = [01 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 02 00 00 00 45 4e 4f 4e 01 00 00 00 16 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4d 43 43 00 00 00 00 17 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 00 18 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 00 19 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 00 1a 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 00 1b 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 00 1c 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 00 10 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 41 4f 4e 00 11 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 52 45 53 54 03 00 00 00 45 4e 4f 4e 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 45 41 43 43 30 00 00 00 04 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 30 00 00 00 05 00 00 00 5f 58 46 47 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 00 00 00 00 00 06 00 00 00 45 4e 4f 4e 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 30 00 00 00 00 07 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 49 53 50 00 00 00 00 00 08 00 00 00 45 4e 4f 4e 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 00 00 00 00 09 00 00 00 45 4e 4f 4e 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 30 0a 00 00 00 45 4e 4f 4e 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 31 0b 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 00 00 00 00 00 0c 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 14 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 44 00 00 00 00 00 0d 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 53 52 00 00 00 00 00 0e 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4a 50 45 47 00 00 00 00 0f 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 43 4f 44 45 43 00 00 1d 00 00 00 5f 52 54 4c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4c 54 52 00 00 00 00 00 1e 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 30 00 00 00 00 1f 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 31 00 00 00 00 20 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 32 00 00 00 00 21 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 33 00 00 00 00 22 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 00 00 00 00 00 23 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 43 49 45 00 00 00 00 24 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 42 52 00 00 00 00 00 25 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 45 50 00 00 00 00 00 26 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 49 4f 00 00 00 00 00]; + apple,tunable-fast-die-ctrl-ts = [00 00 05 00]; + apple,tunable-region-base = [00 00 50 80 03 00 00 00]; + apple,tunable-agx-af-wr-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 35 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 34 00 00 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 00 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 00 00 00 00 00 00 3a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 39 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 2d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 2c 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 11 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3c 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 33 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 7a 00 00 00 00 00 00 00 32 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 79 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 2c 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 8a 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 89 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3e 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 6f 00 00 00 00 00 00 00 3d 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 6e 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 34 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 33 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2c 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 41 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 40 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3c 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 37 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 36 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00]; + apple,tunable-uuid = [30 41 37 44 41 39 43 33 2d 45 37 46 36 2d 33 31 35 31 2d 41 43 44 45 2d 44 41 30 31 37 42 45 42 45 36 34 35 00]; + apple,tunable-segment-names = [5f 5f 54 45 58 54 3b 5f 5f 44 41 54 41 00]; + apple,tunable-segment-ranges = [00 00 50 80 03 00 00 00 00 00 00 01 00 00 00 00 00 00 50 80 03 00 00 00 00 c0 03 00 03 00 00 00 00 c0 53 80 03 00 00 00 00 c0 03 01 00 00 00 00 00 c0 53 80 03 00 00 00 00 e0 06 00 06 00 00 00]; + apple,tunable-pre-loaded = [01 00 00 00]; +}; From b31eadc7d15a0fa36a63b26f9428294234134a3c Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 018/117] arm64: dts: apple: t8142-j813: Describe the M5 GPU The M5 (T8142) GPU, laid out like the A18 Pro's: the sgx block, the gfx-asc and gfx1-asc firmware CPUs with their mailboxes, and the power domain, with addresses and interrupts from the J813 ADT. aura fills the uat-* regions from iBoot's raw-boot handoff, the gfx-data regions from the firmware DATA segments iBoot loaded, and the OPPs from sgx perf-states. apple,clpc links the GPU to the CLPC so that GPU work raises the DRAM and fabric floor. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi | 129 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 130 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi new file mode 100644 index 0000000000000..d1b8b540538bd --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * M5 (T8142) GPU, laid out like the A18 Pro's (t8140.dtsi). Addresses and + * interrupts are from the J813 ADT (sgx, gfx-asc, gfx1-asc, pmgr reg[48]). + * The bootloader fills the uat-* regions from iBoot's raw-boot handoff, the + * gfx-data regions from the firmware DATA segments iBoot loaded, and the + * OPPs from sgx perf-states. + */ + +/ { + aliases { + gpu = &gpu; + }; + + reserved-memory { + /* gfx-asc and gfx1-asc __DATA (ADT sgx gfx-data, gfx1-data) */ + gfx_data: gfx-data { + reg = <0 0 0 0>; + }; + + gfx1_data: gfx1-data { + reg = <0 0 0 0>; + }; + + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + }; + + uat_l2_pagetables: uat-l2-pagetables { + reg = <0 0 0 0>; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + }; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + }; + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* 13 entries in perf-state order, filled in by the bootloader */ + opp00 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp01 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp02 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp03 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp04 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp05 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp06 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp07 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp08 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp09 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp10 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp11 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp12 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + }; +}; + +&{/soc} { + pmgr_gpu: power-management@580e80000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <5 0x80e80000 0 0x30000>; + #address-cells = <1>; + #size-cells = <1>; + + ps_gpx: power-controller@0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gpx"; + apple,always-on; /* critical */ + }; + + ps_gfx: power-controller@110 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + power-domains = <&ps_gpx>; + }; + }; + + gpu: gpu@582600000 { + compatible = "apple,agx-t8142"; + reg = <0x5 0x82600000 0x0 0x88000>, + <0x5 0x80000000 0x0 0x4000000>; + reg-names = "asc", "sgx"; + mboxes = <&gfx_mbox>, <&gfx1_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, + <&uat_l2_pagetables>, <&gfx_data>, <&gfx1_data>; + memory-region-names = "ttbs", "pagetables", "handoff", + "shared-l2", "gfx-data", "gfx1-data"; + operating-points-v2 = <&gpu_opp>; + /* Holds the DCS and fabric floor while GPU work is in flight. */ + apple,clpc = <&clpc>; + }; + + gfx_mbox: mailbox@582600000 { + compatible = "apple,t8142-ascwrap-v6"; + reg = <0x5 0x82600000 0x0 0x88000>, + <0x5 0x82050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + }; + + gfx1_mbox: mailbox@582e00000 { + compatible = "apple,t8142-ascwrap-v6"; + reg = <0x5 0x82e00000 0x0 0x88000>, + <0x5 0x82850000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX1"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index f1378b8651878..2c3f2b35dd8c7 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -394,5 +394,6 @@ }; #include "t8142-j813-display.dtsi" +#include "t8142-j813-gpu.dtsi" #include "t8142-j813-peripherals.dtsi" #include "t8142-j813-usb.dtsi" From d2183e8dfca7370aa04038e6bd7d12a4180b84d1 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 019/117] arm64: dts: apple: t8142-j813: AVD video decoder The M5 AVD ("Hibiscus", avd-version 4, avd-fw v5 tier 2 tunables): CM3 code and SRAM, the mailbox, its DART (SIDs 0-2) and the AVD_SYS power state, from the J813 ADT. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi | 45 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 46 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi new file mode 100644 index 0000000000000..5672ebb885a51 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * M5 (T8142) AVD video decoder: a "Hibiscus" AVD (macOS CAvdMcpuHibiscus, + * avd-version 4, tunables = avd-fw v5 tier 2). From the J813 ADT: avd reg[0] + * 0x2f0000000 + 0x1404000, here at the arm-io offset 0x500000000; 0x14000 each + * of CM3 code and SRAM, mailbox at +0x10a8000 (CAvdApCommHibiscus); + * IRQs 1077/1078; dart-avd at +0x1020000, IRQ 1084, SIDs 0-2; power: PMGR + * device AVD-SYS-V -> AVD_SYS (+0x310). + */ + +&pmgr { + ps_avd_sys: power-controller@310 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "avd_sys"; + }; +}; + +&{/soc} { + avd: avd@501080000 { + compatible = "apple,t8142-avd"; + interrupt-parent = <&aic>; + interrupts = , + ; + power-domains = <&ps_avd_sys>; + resets = <&ps_avd_sys>; + reg = <0x5 0x01080000 0x0 0x14000>, + <0x5 0x01094000 0x0 0x14000>, + <0x5 0x010a8000 0x0 0x4000>, + <0x5 0x01100000 0x0 0x10000>; + reg-names = "code", "sram", "mbox", "ctrl"; + iommus = <&avd_dart 0>, <&avd_dart 1>, <&avd_dart 2>; + }; + + avd_dart: iommu@501020000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x5 0x01020000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_avd_sys>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 2c3f2b35dd8c7..ae576743895cf 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -397,3 +397,4 @@ #include "t8142-j813-gpu.dtsi" #include "t8142-j813-peripherals.dtsi" #include "t8142-j813-usb.dtsi" +#include "t8142-j813-avd.dtsi" From 48b1dd784a109ceca6c6b6714966397f06e46de3 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 020/117] arm64: dts: apple: t8142-j813: ISP (FaceTime camera) The M5 ISP is the A18 Pro's ISP17a at another base: the ISP block and its interrupts, the ISP DART with the remapped streams, the PMGR power states the firmware does not lock, and the J813 camera presets (1280x720, 1440x1080, 1080x1080 and 1920x1080), from the J813 ADT. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi | 223 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 224 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi new file mode 100644 index 0000000000000..accb5629a7e65 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * M5 (T8142) ISP: the Neo's (T8140) ISP17a at another base. From the J813 + * ADT (arm-io offset 0x210000000): isp reg 0x4a0000000 + 0x4000000, IRQs + * [425 426 427 428 438] (425 carries the IPC mailbox), sensor-type 0x71; + * dart-isp at +0x28c0000/+0x28e0000/+0x2900000, IRQ 443, vm-size 0xe0000000, + * remap streams 1 2 3 6 7 10 11 -> 0; EIC mmio-wd-isp 0x4e3b10000 + 0x8000. + * PMGR (0x380700000): ISP_SYS +0x2f8 (ADT id 75), the ISP17a slots + * +0x4000..+0x4060 (ids 162-174, the Neo's offsets), ISPSENS0/1 +0x190/+0x198. + * aura carves the firmware heap and adds it as "isp-heap"; without it, aura + * disables the node. + */ + +/ { + aliases { + isp = &isp; + }; +}; + +&pmgr { + ps_isp_sys: power-controller@2f8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sys"; + }; + + ps_ispsens0: power-controller@190 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens0"; + }; + + ps_ispsens1: power-controller@198 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens1"; + }; + + ps_isp_cpu: power-controller@4000 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cpu"; + power-domains = <&ps_isp_sys>; + }; + + ps_isp_cpu_core0: power-controller@4008 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cpu_core0"; + power-domains = <&ps_isp_cpu>; + }; + + /* + * Not here: ISP_CPU_CORE1 (+0x4010), ISP_SECURE (+0x4028), ISP_STS1 + * (+0x4038), ISP_PEARL (+0x4040), ISP_CVD (+0x4058) and ISP_CVM + * (+0x4060). On J813 their target fields ignore writes from the AP and + * read back 0. The dart-isp DAPF passes this block to the ISP. + */ + + ps_isp_fe: power-controller@4018 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + power-domains = <&ps_isp_sys>; + }; + + /* Unnamed in the ADT, as on the Neo: the firmware's ISP_DPRX. */ + ps_isp_dprx: power-controller@4020 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_dprx"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_sts0: power-controller@4030 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sts0"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_be: power-controller@4048 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_be"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_clr: power-controller@4050 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_clr"; + power-domains = <&ps_isp_be>; + }; + +}; + +&{/soc} { + isp_dart0: iommu@4a28c0000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x4 0xa28c0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + /* As on the Neo: 3-level tables, the ISP fetches at vm-base + * 0x10000000000 and the walk ignores the high bits. */ + apple,dma-range = <0x0 0x0 0x8 0x0>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + }; + + isp_dart1: iommu@4a28e0000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x4 0xa28e0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,dma-range = <0x0 0x0 0x8 0x0>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + }; + + isp_dart2: iommu@4a2900000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x4 0xa2900000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,dma-range = <0x0 0x0 0x8 0x0>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + }; + + /* + * As on the Neo: "coproc" stops short of the secure route table at + * +0x1804000 (a read takes a sticky SError) and of the IRQ block + * (+0x2920000, "mbox") and the GPIO words (+0x2921000). + */ + isp: isp@4a0000000 { + compatible = "apple,t8142-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "wdt"; + reg = <0x4 0xa0000000 0x0 0x1801000>, + <0x4 0xa2920000 0x0 0x1000>, + <0x4 0xa2921000 0x0 0x100>, + <0x4 0xe3b10000 0x0 0x8000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, + <&ps_isp_cpu_core0>, <&ps_isp_fe>, + <&ps_isp_dprx>, <&ps_isp_sts0>, + <&ps_isp_be>, <&ps_isp_clr>; + apple,dart-vm-size = <0x0 0xe0000000>; + apple,platform-id = <0x71>; + apple,temporal-filter = <1>; + }; +}; + +/* + * The J813 IMX958 (sensor version 0x958) as its firmware lists it: 13 configs, + * among them 3040x2880 (config 7) and a binned 1520x1440 (config 10); half + * of them report doubled heights and are left out. Its setfile is + * apple/isp_2224_01XX.dat (from macOS appleh16camerad). + */ +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <1>; + + sensor-presets { + /* 1280x720 */ + preset0 { + apple,config-index = <10>; + apple,input-size = <1520 1440>; + apple,output-size = <1280 720>; + apple,crop = <0 292 1520 856>; + }; + /* 1440x1080 (4:3) */ + preset1 { + apple,config-index = <10>; + apple,input-size = <1520 1440>; + apple,output-size = <1440 1080>; + apple,crop = <40 180 1440 1080>; + }; + /* 1080x1080 (1:1) */ + preset2 { + apple,config-index = <10>; + apple,input-size = <1520 1440>; + apple,output-size = <1080 1080>; + apple,crop = <220 180 1080 1080>; + }; + /* 1920x1080 */ + preset3 { + apple,config-index = <7>; + apple,input-size = <3040 2880>; + apple,output-size = <1920 1080>; + apple,crop = <0 584 3040 1710>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index ae576743895cf..4732e92f7f270 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -398,3 +398,4 @@ #include "t8142-j813-peripherals.dtsi" #include "t8142-j813-usb.dtsi" #include "t8142-j813-avd.dtsi" +#include "t8142-j813-isp.dtsi" From f3e66db7471f2800437b3aea0ce0f710a290fbac Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 021/117] arm64: dts: apple: t8142-j813: CPU idle through PSCI aura runs the OS at EL1 under its monitor and adds a /psci node with enable-method psci for every CPU. Its CPU_SUSPEND is a WFI at EL2 that preserves the registers WFI loses on these cores, so one retention state for both clusters replaces idle=nop. Without a /psci node (raw aura, spin-table) cpuidle-psci does not probe. Signed-off-by: Thomas Glanzmann --- .../boot/dts/apple/t8142-j813-psci-idle.dtsi | 62 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 63 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-psci-idle.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-psci-idle.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-psci-idle.dtsi new file mode 100644 index 0000000000000..9c9cf88a0267b --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-psci-idle.dtsi @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * CPU idle through PSCI CPU_SUSPEND when aura runs the OS at EL1 under its + * monitor (aura's /psci node, "method = smc", enable-method = "psci"). The + * monitor's CPU_SUSPEND is a WFI at EL2 that preserves the registers WFI + * wipes on these cores, so this one retention state replaces idle=nop. + * Without a /psci node (raw aura, spin-table) cpuidle-psci does not probe. + */ + +&{/cpus} { + idle-states { + entry-method = "psci"; + + cpu_retention: cpu-retention { + compatible = "arm,idle-state"; + arm,psci-suspend-param = <0x0>; + entry-latency-us = <10>; + exit-latency-us = <10>; + min-residency-us = <100>; + }; + }; +}; + +&cpu0 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu1 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu2 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu3 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu4 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu5 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu6 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu7 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu8 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu9 { + cpu-idle-states = <&cpu_retention>; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 4732e92f7f270..4e19d8c3cd052 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -399,3 +399,4 @@ #include "t8142-j813-usb.dtsi" #include "t8142-j813-avd.dtsi" #include "t8142-j813-isp.dtsi" +#include "t8142-j813-psci-idle.dtsi" From db6599daaadf3a8999252a18a645917773d0cc76 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 022/117] arm64: dts: apple: t8142-j813: Internal NVMe ANS2 with its mailbox, the coastguard SART (sart-version 3) and the apcie and ANS power states, from the J813 ADT. As on T8140 the NVMe registers are used through the non-secure alias and the NVMMU through the 0x60000 window; aura leaves ANS running for Linux to adopt. Signed-off-by: Thomas Glanzmann --- .../arm64/boot/dts/apple/t8142-j813-nvme.dtsi | 105 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 106 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-nvme.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-nvme.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-nvme.dtsi new file mode 100644 index 0000000000000..82f2bce059d17 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-nvme.dtsi @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (13-inch, M5) J813 with the internal ANS2 NVMe SSD. + * + * Addresses from the J813 ADT (/arm-io/ans, /arm-io/sart-ans, arm-io offset + * 0x210000000). ANS is an ascwrap-v6 ASC with its mailbox at +0x8000, like the + * SMC; the SART is coastguard sart-version 3. As on T8140, the NVMe registers + * are used through the non-secure alias (ADT reg[9]) and the NVMMU through + * the 0x60000 window (ADT reg[3]). iBoot leaves ANS running with the + * controller disabled, so the driver wakes it with a fresh RTKit handshake. + */ + + +&pmgr { + ps_apcie_gp: power-controller@2f0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_gp"; + }; + + ps_ans: power-controller@300 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ans"; + apple,always-on; /* handed over running, cannot be reset */ + }; + + ps_apcie_sys_gp: power-controller@408 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_gp"; + power-domains = <&ps_apcie_gp>; + }; + + ps_apcie_st: power-controller@410 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x410 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_st"; + power-domains = <&ps_ans>; + }; + + ps_apcie_sys_st: power-controller@520 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x520 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_st"; + power-domains = <&ps_ans>, <&ps_apcie_st>; + }; + + ps_apcie_phy_sw: power-controller@560 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x560 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_phy_sw"; + power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + }; +}; + +&{/soc} { + ans_mbox: mailbox@481608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x81608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_ans>; + #mbox-cells = <0>; + }; + + sart: iommu@485c50000 { + compatible = "apple,t8142-sart", "apple,t6000-sart"; + reg = <0x4 0x85c50000 0x0 0xc000>; + power-domains = <&ps_ans>; + }; + + nvme: nvme@4c5cc0000 { + compatible = "apple,t8142-nvme-ans2", "apple,t8140-nvme-ans2"; + reg = <0x4 0xc5cc0000 0x0 0x30000>, + <0x4 0x81600000 0x0 0x4000>, + <0x4 0x85cc0000 0x0 0x60000>; + reg-names = "nvme", "ans", "nvmmu"; + interrupt-parent = <&aic>; + interrupts = ; + mboxes = <&ans_mbox>; + apple,sart = <&sart>; + power-domains = <&ps_ans>, <&ps_apcie_sys_st>, + <&ps_apcie_phy_sw>; + power-domain-names = "ans", "apcie0", "apcie_phy"; + resets = <&ps_ans>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 4e19d8c3cd052..581eb10f876f4 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -400,3 +400,4 @@ #include "t8142-j813-avd.dtsi" #include "t8142-j813-isp.dtsi" #include "t8142-j813-psci-idle.dtsi" +#include "t8142-j813-nvme.dtsi" From 3101b9b7e451fc5ed12254c3da3a7c81ee301d83 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 023/117] arm64: dts: apple: t8142-j813: N1 Wi-Fi and Bluetooth on apcie port 0 aura trains the apcie port 0 link to the N1 (AUS5 PHY, refclk buffer, N1 reset). Linux uses the root port as a generic ECAM host with port 0's 32 MSIs (apple,t8142-pcie-msi) and owns the DART and the endpoint. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi | 47 ++++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 48 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi new file mode 100644 index 0000000000000..43a87da373a91 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * N1 Wi-Fi/Bluetooth on apcie port 0. The bootloader trains the link (AUS5 + * PHY, refclk buffer, N1 reset, port; see the m5air FINDINGS); Linux uses the + * root port as a generic ECAM host and owns the DART and the endpoint. + * Ranges and the DART IRQ from the J813 ADT, as in t8142-j813-wifi.dts. + */ +&{/} { + pcie0: pcie@1cb0000000 { + compatible = "pci-host-ecam-generic"; + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + bus-range = <0 1>; + reg = <0x1c 0xb0000000 0 0x200000>; + ranges = <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0 0x20000000>, + <0x02000000 0 0x80000000 0xb 0x80000000 0 0x40000000>; + dma-coherent; + msi-parent = <&pcie0_msi>; + /* N1's three ADT mapper nodes use stream zero. */ + iommu-map = <0x100 &dart_apcie0 0 1>, + <0x101 &dart_apcie0 0 1>, + <0x102 &dart_apcie0 0 1>; + }; +}; + +&{/soc} { + /* + * Port 0's MSI map (ADT apcie reg[8]). ADT msi-address 0xfffff000, + * msi-vector-offset 1741, #msi-vectors 96 for three ports: port 0 + * owns the first 32. + */ + pcie0_msi: msi-controller@488028000 { + compatible = "apple,t8142-pcie-msi"; + reg = <4 0x88028000 0 0x8000>; + msi-controller; + msi-ranges = <&aic AIC_IRQ 1741 IRQ_TYPE_EDGE_RISING 32>; + }; + + dart_apcie0: iommu@488000000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x88000000 0 0x20000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 581eb10f876f4..cba25ff51274c 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -401,3 +401,4 @@ #include "t8142-j813-isp.dtsi" #include "t8142-j813-psci-idle.dtsi" #include "t8142-j813-nvme.dtsi" +#include "t8142-j813-n1.dtsi" From ef0810261ec84014febdcd5b15ab1661e945b1c0 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 024/117] arm64: dts: apple: t8142-j813: SMC power off and restart, the watchdog The watchdog for restart (and as the SMC's fallback), the PMU on SPMI with the scratchpad cells the SMC uses for power off, and the SMC reboot node. Signed-off-by: Thomas Glanzmann --- .../boot/dts/apple/t8142-j813-power.dtsi | 72 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 73 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-power.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi new file mode 100644 index 0000000000000..fa080be75e8fa --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (13-inch, M5) J813: SMC power off / restart, the PMU on + * SPMI and the watchdog. + */ + +&{/soc} { + /* ADT /arm-io/wdt (wdt,t8142): restart (and the SMC's fallback). */ + wdt: watchdog@3882b0000 { + compatible = "apple,t8142-wdt", "apple,t8103-wdt"; + reg = <0x3 0x882b0000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; +}; + +&{/soc} { + /* + * ADT /arm-io/nub-spmi0 (gen 4) and its primary PMU, pmu-main + * ("abbey", slave 0xe). The boot flags live in the legacy scratchpad + * (info-leg_scrpad = 0xf800), laid out as on T8103 and J873. + */ + nub_spmi0: spmi@388714000 { + compatible = "apple,t8142-spmi"; + reg = <0x3 0x88714000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + + pmic@e { + compatible = "apple,abbey-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + boot_stage: boot-stage@f801 { + reg = <0xf801 0x1>; + }; + + boot_error_count: boot-error-count@f802,0 { + reg = <0xf802 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@f802,4 { + reg = <0xf802 0x1>; + bits = <4 4>; + }; + + /* Set before off1, or the machine powers back on. */ + shutdown_flag: shutdown-flag@f80f,3 { + reg = <0xf80f 0x1>; + bits = <3 1>; + }; + }; + }; + }; +}; + +&smc { + /* Power off (MBSE = off1) and restart (MBSE = phra). */ + reboot { + compatible = "apple,smc-reboot"; + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index cba25ff51274c..b9c71fd75dacf 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -402,3 +402,4 @@ #include "t8142-j813-psci-idle.dtsi" #include "t8142-j813-nvme.dtsi" #include "t8142-j813-n1.dtsi" +#include "t8142-j813-power.dtsi" From 2f7f8fbed3003a9ea010bbf171518c1e45b25377 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 025/117] arm64: dts: apple: t8142-j813: Sound through the AOP As on J873 the AOP's audio service owns the audio fabric and runs the MCA serializers: the CS42L84 headphone jack on admac-base-ns TX2/RX2, the four SN012776 speaker amplifiers on TX0 with their V/I sense on RX0. The AOP is laid out like T8140's (ascwrap-v6, asc-mailbox-v4, setup port). Addresses are from the J813 ADT. Signed-off-by: Thomas Glanzmann --- .../boot/dts/apple/t8142-j813-audio.dtsi | 417 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 418 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-audio.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-audio.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-audio.dtsi new file mode 100644 index 0000000000000..892155da27ee0 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-audio.dtsi @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (13-inch, M5) J813: sound through the AOP firmware. + * + * As on J873 (T8152), the AOP's AOPAudioService owns the audio fabric and + * runs the MCA serializers: "cout"/"cin" (CS42L84 headphone jack) on + * admac-base-ns TX2/RX2 and "spkr" (four SN012776) on TX0, their V/I sense + * on RX0 (ADT iop-audio-controller; RX1 is the firmware's speaker tap). + * The AOP itself is laid out like T8140's (ascwrap-v6, asc-mailbox-v4), + * including the setup port. + * Addresses are from the J813 ADT (arm-io offset 0x210000000). + */ + +#include + +&pmgr { + ps_i2c1: power-controller@350 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c1"; + }; + + ps_i2c3: power-controller@360 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c3"; + }; +}; + +&{/soc} { + pinctrl_ap: pinctrl@39a000000 { + compatible = "apple,t8142-pinctrl", "apple,t8140-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x9a000000 0x0 0x100000>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 224>; + apple,npins = <224>; + /* + * Not every pin is the AP's: reading pin 78's register raises + * an SError. Expose only the pins sound uses: 100 (jack IRQ), + * 150 (amp IRQ) and 151 (amp shutdown). + */ + gpio-reserved-ranges = <0 100>, <101 49>, <152 72>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pinctrl_nub: pinctrl@3881f0000 { + compatible = "apple,t8142-pinctrl", "apple,t8140-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x881f0000 0x0 0x4000>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_nub 0 0 40>; + apple,npins = <40>; + /* Only pin 18 (jack codec reset), as on pinctrl_ap. */ + gpio-reserved-ranges = <0 18>, <19 21>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pmgr_aop: power-management@388284000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x3 0x88284000 0x0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + + /* + * Only the MCA serializer leaves (ADT AUDIO_MCA0_M..AUDIO_MCA5_M) + * are the AP's, as on J873. Their parents AUDIO_A, AUDIO_FR, + * AUDIO_P and AUDIO_BASE_NS_A (0x80, 0xb8, 0xd8, 0xf8) are the AOP + * firmware's: iBoot leaves them off, the firmware's audio init + * switches them off and polls for that (an AP-held auto-enable + * made it time out with 0xe00002d6, so "aop-audio" never + * registered and the firmware panicked in systemReady), and a + * service's pw0 request raises them again. + * + * Likewise the microphone array's LEAP leaves (AUDIO_LEAP_MCA, + * AUDIO_LEAP_S_A, AUDIO_LEAP_RX0) without their parents + * AUDIO_LEAP_C and AUDIO_LEAP_A (0x138, 0xe0); hpai's pw0 raises + * those. + */ + ps_audio_leap_mca: power-controller@100 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_mca"; + }; + + ps_audio_mca0_m: power-controller@108 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca0_m"; + }; + + ps_audio_mca1_m: power-controller@110 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca1_m"; + }; + + ps_audio_mca2_m: power-controller@118 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca2_m"; + }; + + ps_audio_mca3_m: power-controller@120 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca3_m"; + }; + + ps_audio_mca4_m: power-controller@128 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca4_m"; + }; + + ps_audio_mca5_m: power-controller@130 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca5_m"; + }; + + ps_audio_leap_s_a: power-controller@140 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_s_a"; + }; + + ps_audio_leap_rx0: power-controller@150 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_rx0"; + }; + }; + + aop_mbox: mailbox@390608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x90608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + /* + * The ADT's aop-exclave-mailbox, flagged exclave-assigned as on T8140, + * is the AOP setup port the host drives. Its ADT reg (0x3d067c000) is + * not behind the arm-io translation: AOP ASC + 0x4007c000, as on + * T8140. ADT interrupts [497, 494], T8140's +100. + */ + aop_setup_mbox: mailbox@3d067c000 { + compatible = "apple,t8142-aop-setup-mailbox", "apple,t8140-aop-setup-mailbox"; + reg = <0x3 0xd067c000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + /* dart-aop stream 1: ADT vm-base-1 0x20000000000, as on T8140. */ + iommus = <&aop_dart 1>; + apple,dma-range = <0x200 0x0 0x10 0x0>; + #mbox-cells = <0>; + }; + + /* + * dart-aop: streams 0 (AOP), 3, 8 (admac-base-ns), 10 and 16 are the + * AP's; 1, 9 and 11 are exclave streams, of which the host drives 1 + * (setup port) and 11 (admac-leap-s) on the raw boot. As on T8140, the + * firmware's own mappings own the first slot, host buffers go above it. + */ + aop_dart: iommu@390fc0000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x3 0x90fc0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + admac_base_ns: dma-controller@393800000 { + compatible = "apple,t8140-admac"; + reg = <0x3 0x93800000 0x0 0x34000>; + dma-channels = <13>; + interrupts-extended = <&aic AIC_IRQ 549 IRQ_TYPE_LEVEL_HIGH>, + <0>, <0>, <0>; + #dma-cells = <1>; + iommus = <&aop_dart 8>; + apple,dma-range = <0x110 0x00000000 0x0 0x10000000>; + }; + + /* + * admac-leap-s, for the microphone array (ADT audio-hp-mic -> + * leap-s-adma-rx-0), through its admac-leap-s-rx-0-proxy aperture as + * on T8140: the direct window (0x393200000) faults on per-channel + * registers. dart-aop stream 11 with the ADT's vm-base-11/vm-size-11. + * No power domain: the channel registers answer only with the LEAP + * leaves up, which the audio driver raises (after hpai's pw0) before + * it requests the channel; the controller touches no register at + * probe. No resets: the firmware owns the LEAP. + */ + admac_leap_s: dma-controller@3d3200000 { + compatible = "apple,t8140-admac"; + reg = <0x3 0xd3200000 0x0 0x34000>; + dma-channels = <9>; + interrupts-extended = <&aic AIC_IRQ 551 IRQ_TYPE_LEVEL_HIGH>, + <0>, <0>, <0>; + #dma-cells = <1>; + iommus = <&aop_dart 11>; + apple,dma-range = <0x100 0x40000000 0x0 0xffff0000>; + }; + + i2c1: i2c@3a5014000 { + compatible = "apple,t8142-i2c", "apple,t8103-i2c"; + reg = <0x3 0xa5014000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c1>; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 18 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 100 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + sound-name-prefix = "Jack"; + }; + }; + + /* + * Four SN012776: left woofer (0x38, carries the shared shutdown and + * interrupt lines), left tweeter, right woofer, right tweeter. The ADT + * speaker-config lists them as 0x38, then its speaker1..3 (right + * woofer, left tweeter, right tweeter): playback slots 0-3 and V/I + * sense in 8-bit slots 0/2, 4/6, 8/10, 12/14 in that order. The sense + * returns on two lines, the left pair and the right pair; the woofers + * drive the other pair's slots to zero (ADT amp-tx-zd-config, see the + * speaker dai-link). The firmware's speaker frame is 250 BCLKs of + * 12 MHz (ADT audio-speaker stream), which the amplifiers' ratio + * detection rejects. + */ + i2c3: i2c@3a501c000 { + compatible = "apple,t8142-i2c", "apple,t8103-i2c"; + reg = <0x3 0xa501c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c3>; + + speaker_left_woofer: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 150 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer"; + ti,imon-slot-no = <2>; + ti,vmon-slot-no = <0>; + ti,bclk-ratio = <250>; + }; + + speaker_left_tweeter: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + ti,imon-slot-no = <10>; + ti,vmon-slot-no = <8>; + ti,bclk-ratio = <250>; + }; + + speaker_right_woofer: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer"; + ti,imon-slot-no = <6>; + ti,vmon-slot-no = <4>; + ti,bclk-ratio = <250>; + }; + + speaker_right_tweeter: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + ti,imon-slot-no = <14>; + ti,vmon-slot-no = <12>; + ti,bclk-ratio = <250>; + }; + }; + + aop: aop@390c00000 { + compatible = "apple,t8142-aop", "apple,t8152-aop"; + reg = <0x3 0x90c00000 0x0 0x1e0000>, + <0x3 0x90600000 0x0 0x4000>; + reg-names = "mmio", "asc"; + mboxes = <&aop_mbox>, <&aop_setup_mbox>; + mbox-names = "mbox", "setup"; + iommus = <&aop_dart 0>; + apple,dma-range = <0x110 0x0 0x10 0x0>; + + aop_audio: audio { + compatible = "apple,t8142-aop-audio", "apple,t8152-aop-audio"; + /* The microphone card's name: AppleJ813HPAI. */ + apple,chassis-name = "J813"; + apple,machine-kind = "MacBook Air"; + /* hpai: leap-s RX0 (Linux channel 1, odd = DEV_TO_MEM). */ + dmas = <&admac_base_ns 4>, <&admac_base_ns 5>, + <&admac_base_ns 0>, <&admac_base_ns 1>, + <&admac_leap_s 1>; + dma-names = "cout", "cin", "spkr", "sense", "hpai"; + power-domains = <&ps_audio_mca0_m>, <&ps_audio_mca1_m>, + <&ps_audio_mca2_m>, <&ps_audio_mca3_m>, + <&ps_audio_mca4_m>, <&ps_audio_mca5_m>, + <&ps_audio_leap_s_a>, <&ps_audio_leap_rx0>, + <&ps_audio_leap_mca>; + power-domain-names = "mca0", "mca1", "mca2", + "mca3", "mca4", "mca5", + "leap_s_a", "leap_rx0", "leap_mca"; + #sound-dai-cells = <1>; + }; + }; +}; + +/ { + sound { + compatible = "apple,j813-macaudio", "apple,macaudio"; + model = "MacBook Air J813"; + #address-cells = <1>; + #size-cells = <0>; + + dai-link@0 { + reg = <0>; + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&aop_audio 0>; + }; + + codec { + sound-dai = <&jack_codec>; + }; + }; + + dai-link@1 { + reg = <1>; + link-name = "Speaker"; + /* + * Codecs in ADT speaker-config order, so that macaudio + * gives them playback slots 0-3 as macOS does (left + * channels in slots 0 and 2, right in 1 and 3). + */ + dai-tdm-idle-mode-0 = "zero"; + dai-tdm-slot-tx-idle-mask-0 = <0xfff0f0f0>; + dai-tdm-idle-mode-1 = "zero"; + dai-tdm-slot-tx-idle-mask-1 = <0xff0f0f0f>; + + cpu { + sound-dai = <&aop_audio 3>; + }; + + codec { + sound-dai = <&speaker_left_woofer>, <&speaker_right_woofer>, + <&speaker_left_tweeter>, <&speaker_right_tweeter>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index b9c71fd75dacf..dd1526e6caad1 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -403,3 +403,4 @@ #include "t8142-j813-nvme.dtsi" #include "t8142-j813-n1.dtsi" #include "t8142-j813-power.dtsi" +#include "t8142-j813-audio.dtsi" From 3d2aca53b9b40745b02c204b34bae00dd357cdd4 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 13:22:56 +0100 Subject: [PATCH 026/117] arm64: dts: apple: t8142-j813: Run the camera presets from the full sensor mode The 1280x720, 1440x1080 and 1080x1080 presets used the IMX958's binned mode (config 10, 1520x1440). In it the ISP firmware's AWB never leaves the setfile's default white point and the picture is magenta (mean U/V 160/168 on a grey room). In the full-resolution mode (config 7, 3040x2880), which 1920x1080 already used, it converges. Scale all presets from config 7, the crops doubled. With the firmware's AWB alone the J813 streams a neutral picture at all four sizes (mean U/V 127/127, two streams each), at 30 fps. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi index accb5629a7e65..f3850c4161925 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi @@ -190,27 +190,33 @@ &isp { apple,temporal-filter = <1>; + /* + * All presets scale from the full-resolution sensor mode (config 7). + * In the binned mode (config 10, 1520x1440) the firmware's AWB keeps + * the setfile's default white point and the picture is magenta; in + * config 7 it converges. + */ sensor-presets { /* 1280x720 */ preset0 { - apple,config-index = <10>; - apple,input-size = <1520 1440>; + apple,config-index = <7>; + apple,input-size = <3040 2880>; apple,output-size = <1280 720>; - apple,crop = <0 292 1520 856>; + apple,crop = <0 584 3040 1710>; }; /* 1440x1080 (4:3) */ preset1 { - apple,config-index = <10>; - apple,input-size = <1520 1440>; + apple,config-index = <7>; + apple,input-size = <3040 2880>; apple,output-size = <1440 1080>; - apple,crop = <40 180 1440 1080>; + apple,crop = <80 360 2880 2160>; }; /* 1080x1080 (1:1) */ preset2 { - apple,config-index = <10>; - apple,input-size = <1520 1440>; + apple,config-index = <7>; + apple,input-size = <3040 2880>; apple,output-size = <1080 1080>; - apple,crop = <220 180 1080 1080>; + apple,crop = <440 360 2160 2160>; }; /* 1920x1080 */ preset3 { From 64632f46881f9df3bb9bd4176e1584d6f887e326 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 06:32:18 +0100 Subject: [PATCH 027/117] drm/apple: Backlight control on the H17G (M5) panel On H17G the system endpoint announces powerlog-service on channel 1 with only ep-has-desc-mgr and interface-id, so parse_epic_service_init() returned -ENOENT and the service carrying the mNits reports never bound. Match such announces by their name, as props-less announces are (only for the H17G method profile, so H17P is unchanged). H17G publishes no NITS property, so register the backlight device from the first powerlog report instead (140 nits at boot on the J813). H17G firmware ignores the DAC words (bl_unk/bl_value/bl_power at swap+0x32f) and takes the H17P brightness encoding in the same swap bytes (activation at +0x354, binary64 nits at +0x35e, ceiling pair at +0x36e/+0x376), which the unserialized 26.6 request already spans. Factor the H17P encoder to take the swap record and use it from the flush on a brightness update. Tested on the MacBook Air M5 (J813): apple-panel-bl 0..500 nits; writes of 100/300/140 give DCP "send_brightness" at those levels and matching actual_brightness, and the panel follows. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/apple/afk.c | 14 +++++++++-- drivers/gpu/drm/apple/iomfb_h17p.c | 35 +++++++++++++++----------- drivers/gpu/drm/apple/iomfb_h17p.h | 2 ++ drivers/gpu/drm/apple/iomfb_template.c | 16 ++++++++++++ drivers/gpu/drm/apple/systemep.c | 4 +++ 5 files changed, 55 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index f4452c1005834..c988e566c5344 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1491,13 +1491,23 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, } ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit, ep->dcp->fw_compat == DCP_FIRMWARE_H17P); - if (ret) { + if (ret == -ENOENT && + ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) { + /* + * M5 (H17G) announces powerlog-service with only + * ep-has-desc-mgr and interface-id; match it by the + * announced name, as for props-less announces. + */ + epic_name = NULL; + epic_class = NULL; + } else if (ret) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: failed to extract init props: %d\n", ep->endpoint, ret); return; + } else { + service_name = epic_class; } - service_name = epic_class; } else { service_name = name; } diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index abca2a106223e..bd8ef5756286f 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -168,30 +168,37 @@ static u64 iomfb_nits_binary64_h17p(u32 nits) return value; } -void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, - u32 maximum, bool update) +/* @swap is the swap record of a present; H17G sends it unserialized. */ +void iomfb_encode_backlight_swap_h17p(u8 *swap, u32 nits, u32 maximum, + bool update) { u64 ceiling = iomfb_nits_binary64_h17p(maximum); u64 unity = iomfb_nits_binary64_h17p(1); /* This profile does not use the older DAC/power words. */ - memset(wire->swap + 0x32f, 0, 0xd); - memset(wire->swap + 0x354, 0, 0x9a); + memset(swap + 0x32f, 0, 0xd); + memset(swap + 0x354, 0, 0x9a); /* The ceiling pair must remain valid even on a brightness-idle frame. */ - put_unaligned_le64(ceiling, wire->swap + 0x36e); - put_unaligned_le64(ceiling, wire->swap + 0x376); + put_unaligned_le64(ceiling, swap + 0x36e); + put_unaligned_le64(ceiling, swap + 0x376); if (!update) return; /* Linux brightness changes and soft DPMS use the same activation bytes. */ - put_unaligned_le32(1, wire->swap + 0x354); - wire->swap[0x358] = 1; - wire->swap[0x359] = 1; - wire->swap[0x35a] = 1; - put_unaligned_le16(1, wire->swap + 0x35c); - put_unaligned_le64(iomfb_nits_binary64_h17p(nits), wire->swap + 0x35e); - put_unaligned_le64(unity, wire->swap + 0x37e); - put_unaligned_le64(unity, wire->swap + 0x3e6); + put_unaligned_le32(1, swap + 0x354); + swap[0x358] = 1; + swap[0x359] = 1; + swap[0x35a] = 1; + put_unaligned_le16(1, swap + 0x35c); + put_unaligned_le64(iomfb_nits_binary64_h17p(nits), swap + 0x35e); + put_unaligned_le64(unity, swap + 0x37e); + put_unaligned_le64(unity, swap + 0x3e6); +} + +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, + u32 maximum, bool update) +{ + iomfb_encode_backlight_swap_h17p(wire->swap, nits, maximum, update); } static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 9880e82078178..0bb74295c382f 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -158,6 +158,8 @@ static inline struct dcp_apply_property_h17p dcp_opaque_x_property_h17p(void) }; } +void iomfb_encode_backlight_swap_h17p(u8 *swap, u32 nits, u32 maximum, + bool update); void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, u32 maximum, bool update); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 337096ed2a9b2..5413cabfd08e7 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -3848,6 +3848,21 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; #endif if (update_brightness) { +#if DCP_FW_H17(DCP_FW_VER) + /* + * H17G firmware ignores the DAC words and takes nits in the + * same swap bytes as H17P. Without a requested or reported + * level, send nothing rather than 0 nits. + */ + int nits = dcp->brightness.bl_dev ? + dcp->brightness.bl_dev->props.brightness : + dcp->brightness.nits; + + if (nits > 0) + iomfb_encode_backlight_swap_h17p((u8 *)&req->swap, nits, + dcp->brightness.maximum, + true); +#else #if DCP_FW_LAYOUT_14_7(DCP_FW_VER) req->swap.bl_update = 1; #else @@ -3855,6 +3870,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru #endif req->swap.bl_value = dcp->brightness.dac; req->swap.bl_power = 0x40; +#endif dcp->brightness.update = false; } #endif diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index 5360aaf3e5804..38f667e114514 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -120,6 +120,10 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype } dcp->brightness.nits = mnits.millinits / 1000; + /* H17G publishes no NITS property; its first report registers the device. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && + dcp_has_panel(dcp) && !dcp->brightness.bl_dev && dcp->brightness.maximum > 0) + schedule_work(&dcp->bl_register_wq); return 0; } From e56c4660312c5517e3acec8f04a59eff576d2943 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 19:06:09 +0100 Subject: [PATCH 028/117] drm: apple: iomfb_h17p: answer H17G CoreAnalyticsSendEvent with an empty dict The M5's DCP (H17G) panicked with "app NMI" one hour after every boot, taking the display with it. Its crash log shows "OSDictionary deserialization failed": D114 CoreAnalyticsSendEvent was answered with trampoline_zero on H17G, so the reply starts with 0 instead of an OSDictionary. H17P already returns an empty dictionary ('d', 0x1004 bytes); D114's sizes are the same on H17G (0x1044/0x1004), so use that reply there too. DJ identified the analytics callback as the one-hour trigger. Tested on J813 (M5 MacBook Air): two boots run past one hour (3799 s, 4269 s+) without a DCP crash; before, every boot crashed at about 3600 s. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/apple/iomfb_h17p.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index bd8ef5756286f..ad8307f602ea7 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -315,9 +315,12 @@ static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return trampoline_zero(dcp, tag, out, in); - + /* + * H17G (M5) needs the same empty dictionary. A zeroed reply fails the + * firmware's OSDictionary deserialization ("OSDictionary + * deserialization failed" in the DCP crash log) and the DCP panics + * with "app NMI" one hour after boot. + */ trace_iomfb_callback(dcp, tag, __func__); /* The dispatcher has validated the D114 reply size. */ memset(out, 0, 0x1004); From b6674cb5f4f6d0cae86a72e48c14fae74bf6c28a Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 13:11:47 +0100 Subject: [PATCH 029/117] drm/apple: Blank the H17G internal panel instead of stopping the pipe On the M5 (H17G) internal panel, DPMS off stopped the DCP pipe: setPowerState(0) times out and the following setPowerState(1) is never answered ("wait for power timed out, connector will be broken"), so the panel stays frozen after a lid close or a compositor's output power off. Keep the pipe running instead, as H17P soft DPMS does: DPMS off submits a black, surface-free clear swap that also takes the backlight to 0 nits; DPMS on only requests a brightness update, and the next frame restores scanout and level. Tested on J813: fbdev blank/unblank and sway output power off/on cycles and a lid close/open; the firmware reports brightness 0 and back to the previous level, no timeouts. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/apple/dcp.c | 23 ++++++++++++++++ drivers/gpu/drm/apple/iomfb_h17p.h | 1 + drivers/gpu/drm/apple/iomfb_template.c | 38 ++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c5d05af2f976f..72e0eabca5b18 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4086,6 +4086,18 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) #define DCP_BL_FALLBACK_DELAY msecs_to_jiffies(2000) +/* + * H17G (M5) internal panel: a pipe stop never comes back (see + * iomfb_blank_h17p()), so DPMS blanks the panel and keeps the pipe running. + */ +static bool dcp_h17g_panel(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && + dcp->connector_type == DRM_MODE_CONNECTOR_eDP && + dcp_has_panel(dcp); +} + static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) { /* The 14.7 panel has its own known-level blank and restore hooks. */ @@ -4176,6 +4188,12 @@ void dcp_poweron(struct platform_device *pdev) return; } + if (dcp_h17g_panel(dcp)) { + /* The pipe never stopped; the next swap restores the level. */ + dcp->brightness.update = true; + return; + } + if (dcp_is_typec_output(dcp)) { /* This CRTC now owns the current Type-C display session. */ scoped_guard(mutex, &dcp->hpd_mutex) @@ -4303,6 +4321,11 @@ void dcp_poweroff(struct platform_device *pdev) return; } + if (dcp_h17g_panel(dcp)) { + iomfb_blank_h17p(dcp); + return; + } + if (dcp->avep) av_service_disconnect(dcp); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 0bb74295c382f..112a4e002138d 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -158,6 +158,7 @@ static inline struct dcp_apply_property_h17p dcp_opaque_x_property_h17p(void) }; } +void iomfb_blank_h17p(struct apple_dcp *dcp); void iomfb_encode_backlight_swap_h17p(u8 *swap, u32 nits, u32 maximum, bool update); void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 5413cabfd08e7..7da349b978c22 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2471,6 +2471,44 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) dev_info(dcp->dev, "dcp_poweroff() done\n"); } +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER && DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) +/* + * H17G (M5) internal panel DPMS off. A pipe stop does not come back: + * setPowerState(0) times out and the next setPowerState(1) is never + * answered. Blank instead and keep the pipe running: a black, + * surface-free swap that also takes the backlight to 0 nits. The next CRTC + * enable presents the scanout and the brightness again. + */ +void DCP_FW_NAME(iomfb_blank)(struct apple_dcp *dcp) +{ + struct DCP_FW_NAME(dcp_swap_start_req) swap_req = { 0 }; + typeof(DCP_FW_UNION(dcp->swap)) *swap = &DCP_FW_UNION(dcp->swap); + struct dcp_swap_cookie *cookie; + unsigned long ret; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp_prepare_clear_swap(dcp, swap); + if (dcp_has_panel(dcp)) + iomfb_encode_backlight_swap_h17p((u8 *)&swap->swap, 0, + dcp->brightness.maximum, true); + + dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); + if (!ret || READ_ONCE(cookie->status)) + dev_warn(dcp->dev, "%s: blank swap %s\n", __func__, + ret ? "failed" : "timed out"); + kref_put(&cookie->refcount, release_swap_cookie); +} +#endif + static void last_client_closed_sleep(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_DESKTOP_26(DCP_FW_VER) From 5e98b68eb5ecb154eb0a270b19e57d8535592390 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 07:24:43 +0100 Subject: [PATCH 030/117] drm/apple: Offer only alpha formats on the H17G primary plane The H17G (M5) DCP firmware blends the fourth byte of an XRGB surface as alpha, and it ignores the opaque-X property as H17P does. A compositor that picks XRGB8888 leaves that byte to whatever its renderer stored: the GPU-rendered sway output on J813 showed blue and cyan text as black and grey while the pixels in memory were right. Offer the overlay formats, which all carry alpha, on the H17G primary plane, so that compositors write the byte. Treat the fbdev console the same way as on H17P: an XRGB console with X = 0 stays black. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/apple/apple_drv.c | 11 ++++++++--- drivers/gpu/drm/apple/plane.c | 12 +++++++++++- drivers/gpu/drm/apple/plane.h | 1 + 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index cc709dac66626..9001a1fe20c3c 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -170,9 +170,12 @@ static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, if (t8132_ext_crtc(crtc)) continue; dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); - if (dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - /* H17P applies the stored fourth byte as straight alpha. */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + /* + * H17P applies the stored fourth byte as straight alpha, + * the H17G (M5) profile too: an XRGB console with X = 0 + * stays black. ARGB makes fbcon write it opaque. + */ sizes->surface_bpp = 32; sizes->surface_depth = 32; break; @@ -915,6 +918,8 @@ static int apple_probe_per_dcp(struct device *dev, planes[zpos] = apple_plane_init(drm, 1U << num, surf, supports_l10r, supports_xrgb2101010, + dcp_data->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17G, plane_type, interchange, supports_premultiplied); } diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index f2b57a700ffb7..a8c37e29ed65a 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -743,6 +743,7 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, u32 iomfb_surf, bool supports_l10r, bool supports_xrgb2101010, + bool xrgb_is_alpha, enum drm_plane_type type, bool interchange, bool premultiplied) @@ -755,7 +756,16 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, switch (type) { case DRM_PLANE_TYPE_PRIMARY: - if (supports_l10r && supports_xrgb2101010) { + if (xrgb_is_alpha) { + /* + * The firmware blends the X byte as alpha and ignores the + * opaque-X property: offer only formats with alpha, so that + * compositors write it instead of leaving whatever the + * renderer stored there. + */ + fmts = dcp_overlay_formats; + num_fmts = ARRAY_SIZE(dcp_overlay_formats); + } else if (supports_l10r && supports_xrgb2101010) { fmts = dcp_primary_formats; num_fmts = ARRAY_SIZE(dcp_primary_formats); } else if (supports_l10r) { diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index 69229ad6a5775..32d7da10a6a4f 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -37,6 +37,7 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, u32 iomfb_surf, bool supports_l10r, bool supports_xrgb2101010, + bool xrgb_is_alpha, enum drm_plane_type type, bool interchange, bool premultiplied); From 848cb14c09cd4bca83ae587b1ebf7a6be45d2625 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 25 Oct 2026 14:33:36 +0100 Subject: [PATCH 031/117] arm64: cpufeature: treat WFxT as absent on Apple M5 and M6 cores Apple T8142 (M5) and T8152 (M6) cores clear architectural state across WFIT, the FP/SIMD registers included. apple_m5_cpu_wfit() preserves the general-purpose registers around the WFIT in __delay(), but not FP/SIMD, and __delay() runs from any context: apple-soc-cpufreq polls the DVFS command register with readq_poll_timeout_atomic() on every transition, which schedutil and ondemand do from the scheduler or a kworker on a CPU whose current task still has live FP/SIMD state. On a J873 (Mac mini M6), 12 workers hashing a 128 MiB tmpfs file with sha256sum, with short sleeps in between, got 25-56% wrong results under schedutil or ondemand, each one different; fio verify found 64-byte runs of zeros in reads, and systemd and Python crashed. The performance, powersave and userspace governors were clean, as was the same load with arm64.nowfxt (0 of 288 under schedutil twice and under ondemand, with ~1200/550 transitions per run). Do what arm64.nowfxt does for these MIDRs: ARM64_HAS_WFXT is not detected, so __delay() falls back to the event-stream WFE loop. The userspace HWCAP2_WFXT still follows ID_AA64ISAR2_EL1. Only T8152 was measured; T8142 is included because apple_m5_needs_wait_save() already groups its cores with T8152 for this state loss. Signed-off-by: Thomas Glanzmann --- arch/arm64/kernel/cpufeature.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 1d12b124fd7dc..bd53d897b765a 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2625,6 +2625,31 @@ test_has_gicv5_legacy(const struct arm64_cpu_capabilities *entry, int scope) return !!(read_sysreg_s(SYS_ICC_IDR0_EL1) & ICC_IDR0_EL1_GCIE_LEGACY); } +/* + * Apple T8152 (M6) cores clear architectural state across WFIT, the FP/SIMD + * registers included. __delay() would execute it from any context, including + * the cpufreq transition polling that load-driven governors run on a CPU + * whose current task still has live FP/SIMD state: on J873 (M6) 25-56% of + * SHA-256 runs gave wrong results under schedutil. Treat WFxT as absent + * there, as arm64.nowfxt does, so __delay() uses WFE. T8142 (M5) was not + * measured; it is included because apple_m5_needs_wait_save() groups its + * cores with T8152 for this state loss. + */ +static bool has_wfxt(const struct arm64_cpu_capabilities *entry, int scope) +{ + static const struct midr_range apple_wfxt_loses_state[] = { + MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x062)), + MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x063)), + MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x072)), + MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0x073)), + {} + }; + + if (is_midr_in_range_list(apple_wfxt_loses_state)) + return false; + return has_cpuid_feature(entry, scope); +} + static const struct arm64_cpu_capabilities arm64_features[] = { #ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL { @@ -3129,7 +3154,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .desc = "WFx with timeout", .capability = ARM64_HAS_WFXT, .type = ARM64_CPUCAP_SYSTEM_FEATURE, - .matches = has_cpuid_feature, + .matches = has_wfxt, ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, WFxT, IMP) }, { From 5c9cbb25dba8fa4e7a2ed2406a0443968198273b Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:19:28 +0100 Subject: [PATCH 032/117] arm64: idle: T8142 idles in WFI even with idle=nop aura, the J813 boot loader, passes idle=nop on M5 for kernels that do not preserve registers across WFI. This kernel does (apple_wfi_quirk_init), so on T8142 keep WFI as the default idle: spinning keeps all ten cores busy, cpuidle's own WFI state spins too, and the SoC sits near 90 C and throttles. Signed-off-by: Thomas Glanzmann --- arch/arm64/kernel/idle.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index f4ad1c3e95968..267677adc220a 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -53,10 +53,22 @@ static bool apple_wfi_loses_regs __ro_after_init; void __init apple_wfi_quirk_init(void) { + u32 model = read_cpuid_id() & MIDR_CPU_MODEL_MASK; + if (midr_is_apple_m4_or_later(read_cpuid_id())) { apple_wfi_loses_regs = true; pr_info("Apple: preserving registers across WFI\n"); } + /* + * aura passes idle=nop on M5 for kernels without this quirk. Spinning + * idle keeps all ten cores busy (cpuidle's WFI state spins too), and + * the SoC sits near 90 C and throttles; WFI is safe here. + */ + if (idle == ARM64_IDLE_NOP && + (model == MIDR_APPLE_T8142_E || model == MIDR_APPLE_T8142_P)) { + idle = ARM64_IDLE_WFI; + pr_info("Apple: T8142 idles in WFI, ignoring idle=nop\n"); + } } static __always_inline bool apple_wfi_needs_save(void) From 10527e3b09f9a5b09fc9b9a0ab6522d77c438f6d Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:23:03 +0100 Subject: [PATCH 033/117] soc: apple: mailbox: Add the T8142 (M5) GPU ASCWrap v6 mailboxes The M5's two GPU firmware CPUs (GFX and GFX1) sit at T8140's wrapper and IOP VBAR windows moved up by 0x100000000. Add their role table, the T8142 provider and the compatible; the GPU deferred forever without them. Signed-off-by: Thomas Glanzmann --- .../bindings/mailbox/apple,ascwrap-v6.yaml | 1 + drivers/soc/apple/mailbox.c | 35 ++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml index f060194de4b9d..289c29ba752fb 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -21,6 +21,7 @@ properties: enum: - apple,t6040-ascwrap-v6 - apple,t8140-ascwrap-v6 + - apple,t8142-ascwrap-v6 - apple,t6050-apif-ascwrap-v6 - apple,t6050-ascwrap-v6 - apple,t6050-x1n1-ascwrap-v6 diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 6f0d1f7957ec0..7c7c1e0e84531 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -66,6 +66,8 @@ #define APPLE_T6040_GFX_WRAPPER_BASE 0x28a600000ULL #define APPLE_T6040_GFX_IOP_VBAR_BASE 0x28a050000ULL #define APPLE_T6050_ASCWRAP_BASE_DELTA 0x380000000ULL +/* T8142 (M5) moves T8140's 0x4... GPU ASC bank to 0x5.... */ +#define APPLE_T8142_ASCWRAP_BASE_DELTA 0x100000000ULL static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); @@ -825,7 +827,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->ascwrap_v6.wrapper_size = resource_size(res); mbox->ascwrap_v6.iop_vbar_start = iop_vbar_res->start; mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar_res); - /* T8140 and T6040 implement both the CPU start and the CPU stop. */ + /* T8140, T8142 and T6040 implement both the CPU start and stop. */ mbox->ascwrap_v6.missing = 0; mbox->wrapper_regs = @@ -978,6 +980,15 @@ static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t8140_ascwrap_v6_ APPLE_T8140_GFX1_WRAPPER_BASE, APPLE_T8140_GFX1_IOP_VBAR_BASE }, }; +static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t8142_ascwrap_v6_roles[] = { + { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_T8140_GFX_WRAPPER_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA, + APPLE_T8140_GFX_IOP_VBAR_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA }, + { "GFX1", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, + APPLE_T8140_GFX1_WRAPPER_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA, + APPLE_T8140_GFX1_IOP_VBAR_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA }, +}; + /* The T6040 runs a single GPU firmware instance. */ static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6040_ascwrap_v6_roles[] = { { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, @@ -1004,6 +1015,27 @@ static const struct apple_mbox_hw apple_mbox_t8140_ascwrap_v6_hw = { .has_irq_controls = false, }; +/* T8142 (M5): T8140's two GPU firmware CPUs at the 0x5... bank. */ +static const struct apple_mbox_hw apple_mbox_t8142_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, + .ascwrap_v6_roles = apple_mbox_t8142_ascwrap_v6_roles, + .num_ascwrap_v6_roles = ARRAY_SIZE(apple_mbox_t8142_ascwrap_v6_roles), + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_t6040_ascwrap_v6_hw = { .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, .is_ascwrap_v6 = true, @@ -1117,6 +1149,7 @@ static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t8122-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t6040-ascwrap-v6", .data = &apple_mbox_t6040_ascwrap_v6_hw }, { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_t8140_ascwrap_v6_hw }, + { .compatible = "apple,t8142-ascwrap-v6", .data = &apple_mbox_t8142_ascwrap_v6_hw }, { .compatible = "apple,t6050-apif-ascwrap-v6", .data = &apple_mbox_t6050_apif_ascwrap_v6_hw }, { .compatible = "apple,t6050-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, { .compatible = "apple,t6050-x1n1-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, From 761ab15abc07bfde07045229e81793f0f50d7d61 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:24:16 +0100 Subject: [PATCH 034/117] dt-bindings: power: apple,pmc: Add T8142 PMC and CLPC compatibles Signed-off-by: Thomas Glanzmann --- Documentation/devicetree/bindings/power/apple,pmc.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 4c1d9aefc2519..477fc330d253f 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -28,6 +28,7 @@ properties: - apple,t6041-pmc - apple,t8132-pmc - apple,t8140-pmc + - apple,t8142-pmc reg: minItems: 2 @@ -189,6 +190,7 @@ patternProperties: - apple,t6041-clpc - apple,t8132-clpc - apple,t8140-clpc + - apple,t8142-clpc reg: maxItems: 1 From 2555ad8723184f0ebc4dc470ae7bd593615ced8b Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:24:16 +0100 Subject: [PATCH 035/117] pmdomain: apple: pmc: Add T8142 (M5) The AppleT8142PMC accessors put the floors at agent << 14, the per-agent states at 0x106000 and the static voter enables at 0x108000, as on T8140. It has three rails, DCS, SOC and DISP (ADT voter-rail-data), and clamps a floor to DCS 7 and SOC 4. iBoot leaves every rail enable at 0, so without the PMC no vote is honoured and DRAM and the fabric stay at their lowest states (CPU DRAM latency 171 ns instead of 70 ns). Signed-off-by: Thomas Glanzmann --- drivers/pmdomain/apple/pmc.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 6b1e4c472e399..00bebde94a82c 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -1124,6 +1124,23 @@ static const struct apple_pmc_hw apple_pmc_hw_t8140 = { .ap_agent = 3, }; +/* + * T8142: the AppleT8142PMC accessors put the floors at agent << 14, the + * per-agent states at 0x106000 and the static voter enables at 0x108000; + * rails DCS, SOC and DISP (ADT voter-rail-data). The PMC clamps a floor to + * DCS 7 and SOC 4. + */ +static const struct apple_pmc_hw apple_pmc_hw_t8142 = { + .agent_stride = 0x4000, + .agents = 4, + .rails = 3, + .agent_state = 0x0, + .rail_enable = 0x2000, + .dcs_rail = 0, + .dcs_max = 7, + .ap_agent = 3, +}; + /* * T6040 (M4 Pro): the T8140 layout, with DCS states 0-5 and fabric states * 0-2. The ADT's voter-agent rail mask is 0x7 (DCS, SOC and DISP_INT). @@ -1146,6 +1163,7 @@ static const struct apple_pmc_hw apple_pmc_hw_t6040 = { static const struct of_device_id apple_pmc_of_match[] = { { .compatible = "apple,t8132-pmc", .data = &apple_pmc_hw_t8132 }, { .compatible = "apple,t8140-pmc", .data = &apple_pmc_hw_t8140 }, + { .compatible = "apple,t8142-pmc", .data = &apple_pmc_hw_t8142 }, { .compatible = "apple,t6041-pmc", .data = &apple_pmc_hw_t6040 }, {} }; From ee4f4957971fff13569866ed610abda83a968585 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:24:29 +0100 Subject: [PATCH 036/117] soc: apple: clpc: Add T8142 (M5) with the GPU input T8142's CLPC drives DCS states 0-7 on rail 0 and fabric states 0-4 on rail 1; the map is the one measured on J813 (four P cores copy at ~40 GB/s with the fabric at 1 and ~110 GB/s at 4). The M5 GPU driver reports its submissions in flight through apple_clpc_gpu_get(), as on T6040, but the CLPC allocated that input only for the t6041 compatible or a gpu_dense table, so on T8142 the GPU waited for it forever. Make it a per-SoC flag, gpu_input, set for T6040 and T8142, and let apple_clpc_gpu_get() accept the T8142 CLPC. Signed-off-by: Thomas Glanzmann --- drivers/soc/apple/clpc.c | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index ae0d6d48bac7f..b31a46b050358 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -78,6 +78,9 @@ enum apple_clpc_rail { * @pmp_agent, limits the DCS, see apple_clpc_gpu_limited() * @gpu_dense: submissions per sampling period that make light GPU work * dense, see apple_clpc_gpu_dense(); 0 for none + * @gpu_input: the GPU driver reports its work in flight through + * apple_clpc_gpu_get(); while there is some, both rails go to the top + * of @map */ struct apple_clpc_hw { unsigned int rail[APPLE_CLPC_RAILS]; @@ -89,6 +92,7 @@ struct apple_clpc_hw { bool pmp_backoff; unsigned int pmp_agent; unsigned int gpu_dense; + bool gpu_input; }; struct apple_clpc_cluster { @@ -228,7 +232,8 @@ int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) return of_find_property(dev->of_node, "apple,clpc", NULL) ? -EINVAL : 0; if (!of_device_is_compatible(np, "apple,t6041-clpc") && !of_device_is_compatible(np, "apple,t8132-clpc") && - !of_device_is_compatible(np, "apple,t8140-clpc")) { + !of_device_is_compatible(np, "apple,t8140-clpc") && + !of_device_is_compatible(np, "apple,t8142-clpc")) { of_node_put(np); return -EINVAL; } @@ -744,13 +749,11 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->pmc = dev->parent; clpc->hw = of_device_get_match_data(dev); /* T6040 firmware keeps GFX powered for its whole lifetime. A separate - * host publication/idle input, not that supplier state, drives demand. - * On T8140 the same input counts submissions for dense light work. The - * T8132 GPU driver reports its work through it as well. + * host publication/idle input, not that supplier state, drives demand; + * the T8132 and M5 (T8142) GPU drivers report through the same input. + * On T8140 the same input counts submissions for dense light work. */ - if (of_device_is_compatible(dev->of_node, "apple,t6041-clpc") || - of_device_is_compatible(dev->of_node, "apple,t8132-clpc") || - clpc->hw->gpu_dense) { + if (clpc->hw->gpu_input || clpc->hw->gpu_dense) { clpc->gpu = kzalloc_obj(*clpc->gpu); if (!clpc->gpu) return -ENOMEM; @@ -882,6 +885,7 @@ static const struct apple_clpc_hw apple_clpc_hw_t8132 = { [APPLE_CLPC_SOC] = { 3, 3, 3, 3 }, }, .gpu_demand = false, + .gpu_input = true, }; /* @@ -948,6 +952,26 @@ static const struct apple_clpc_hw apple_clpc_hw_t6040 = { }, .sustain = 3, .gpu_demand = false, + .gpu_input = true, +}; + +/* + * T8142: DCS states 0-7 on rail 0, fabric states 0-4 on rail 1. The fabric + * state bounds CPU memory bandwidth: four P cores copy at ~40 GB/s with the + * fabric at 1 and at ~110 GB/s at 4 (macOS ~124). + * + * The M5 GPU driver reports its submissions in flight. Once the PMP runs, its + * vote (agent 0) drops from iBoot's DCS 7 / fabric 1 to 0 and nothing else + * raises the floor for the GPU: a 4K RGBA16F fill then takes 5.1 ms instead + * of 0.9 ms with both rails at the top of the map. + */ +static const struct apple_clpc_hw apple_clpc_hw_t8142 = { + .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, + .map = { + [APPLE_CLPC_DCS] = { 2, 4, 6, 7 }, + [APPLE_CLPC_SOC] = { 1, 2, 3, 4 }, + }, + .gpu_input = true, }; #if IS_BUILTIN(CONFIG_KUNIT) && !defined(MODULE) @@ -1120,6 +1144,7 @@ kunit_test_suite(apple_clpc_suite); static const struct of_device_id apple_clpc_of_match[] = { { .compatible = "apple,t8132-clpc", .data = &apple_clpc_hw_t8132 }, { .compatible = "apple,t8140-clpc", .data = &apple_clpc_hw_t8140 }, + { .compatible = "apple,t8142-clpc", .data = &apple_clpc_hw_t8142 }, { .compatible = "apple,t6041-clpc", .data = &apple_clpc_hw_t6040 }, {} }; From 2325ffbb9f5eae44d9eab3368afd4fd45a32d831 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:25:03 +0100 Subject: [PATCH 037/117] soc: apple: pmp: Add T8142 (M5) The J813 PMP runs the same registry-v2 firmware as T8140. Its DART window starts at 1 TiB, so it takes T8140's 42-bit DMA mask, and like T6041 it starts endpoint 7 (CoreAnalytics) and takes its 8 KiB buffer before it acknowledges the IOP power state. Signed-off-by: Thomas Glanzmann --- drivers/soc/apple/pmp.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index c239fae558cee..2212509efd82c 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -692,6 +692,15 @@ const HW_CFG_T6041: PmpHwConfig = PmpHwConfig { coreanalytics: Ep7::Always, }; +/// T8142: the DART window starts at 1 TiB as on T8140. Its registry-v2 +/// firmware starts endpoint 7 and takes its 8 KiB buffer before it +/// acknowledges the IOP power state. +const HW_CFG_T8142: PmpHwConfig = PmpHwConfig { + dma_mask: Some(DmaMask::new::<42>()), + check_bootargs_uuid: false, + coreanalytics: Ep7::Always, +}; + const HW_CFG_T8132: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, @@ -711,6 +720,7 @@ kernel::of_device_table!( [ (of::DeviceId::new(c"apple,t8132-pmp-v2"), &HW_CFG_T8132), (of::DeviceId::new(c"apple,t8140-pmp-v2"), &HW_CFG_T8140), + (of::DeviceId::new(c"apple,t8142-pmp-v2"), &HW_CFG_T8142), (of::DeviceId::new(c"apple,t6041-pmp-v2"), &HW_CFG_T6041), (of::DeviceId::new(c"apple,t6000-pmp-v2"), &HW_CFG_DEFAULT), ] From efa93c24e38da07318b5eb79c8a7ac562125effc Mon Sep 17 00:00:00 2001 From: Acelogic Date: Wed, 9 Sep 2026 15:28:21 -0400 Subject: [PATCH 038/117] wifi: apple: add hardware-qualified N1 driver and J813 workbench [Thomas: the driver part of 9f4a5246489a on the J813 line of linux-aurora] Signed-off-by: Thomas Glanzmann --- drivers/net/wireless/apple/Kconfig | 20 +- drivers/net/wireless/apple/centauri/Makefile | 3 +- .../net/wireless/apple/centauri/centauri.h | 57 +- .../wireless/apple/centauri/centauri_ipc.c | 478 ++++ .../wireless/apple/centauri/centauri_pci.c | 278 ++ .../wireless/apple/centauri/centauri_port.c | 192 ++ .../wireless/apple/centauri/centauri_wifi.c | 2312 +++++++++++++++++ 7 files changed, 3282 insertions(+), 58 deletions(-) create mode 100644 drivers/net/wireless/apple/centauri/centauri_ipc.c create mode 100644 drivers/net/wireless/apple/centauri/centauri_pci.c create mode 100644 drivers/net/wireless/apple/centauri/centauri_port.c create mode 100644 drivers/net/wireless/apple/centauri/centauri_wifi.c diff --git a/drivers/net/wireless/apple/Kconfig b/drivers/net/wireless/apple/Kconfig index e40d5d9c5fd53..d3ead4d9259e7 100644 --- a/drivers/net/wireless/apple/Kconfig +++ b/drivers/net/wireless/apple/Kconfig @@ -2,18 +2,24 @@ config WLAN_VENDOR_APPLE bool "Apple wireless devices" default y + help + Support for Apple's wireless network devices. if WLAN_VENDOR_APPLE config APPLE_CENTAURI - tristate "Apple N1 Centauri firmware and IPC transport" + tristate "Apple N1 Centauri experimental J813 Wi-Fi" depends on ARCH_APPLE || COMPILE_TEST - depends on PCI && CFG80211 && PCIE_APPLE_APIF + depends on PCI && CFG80211 && DEBUG_FS select FW_LOADER help - Boot the shared N1 firmware and establish its PCI DMA contexts. - Requires a preinitialized APIF host with the supported reset layout - and DT-supplied personalized firmware name and platform identifier. - The endpoint must start in the stopped ROM personality. Firmware - and per-device calibration are external inputs and are not included. + Experimental Apple N1 PCIe and ACIPC Wi-Fi support for the J813 + MacBook Air. Firmware boot, ring setup and connection setup currently + use debugfs and the tools/aurora/j813 workbench. The tested single-link + WPA2 setup completes DHCP and bidirectional traffic. Suspend/resume, + roaming and automatic startup are not yet qualified. + + The firmware and per-device calibration are private external inputs. + This driver does not include Bluetooth HCI support. + To build as a module, choose M; the module is apple_centauri. endif diff --git a/drivers/net/wireless/apple/centauri/Makefile b/drivers/net/wireless/apple/centauri/Makefile index ddd2501998887..1a17e5283e227 100644 --- a/drivers/net/wireless/apple/centauri/Makefile +++ b/drivers/net/wireless/apple/centauri/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_CENTAURI) += apple_centauri.o -apple_centauri-y := centauri_pci.o centauri_ipc.o -apple_centauri-$(CONFIG_APPLE_CENTAURI_WIFI) += centauri_wifi.o +apple_centauri-y := centauri_pci.o centauri_port.o centauri_ipc.o centauri_wifi.o diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h index 55106d4ef0794..f7fdf4105a3b3 100644 --- a/drivers/net/wireless/apple/centauri/centauri.h +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -1,87 +1,46 @@ - +/* SPDX-License-Identifier: GPL-2.0-only */ #ifndef AURORA_CENTAURI_H #define AURORA_CENTAURI_H #include -#include #include #include #include -#include -#define CEN_ALPHA_WINDOW_SIZE SZ_32M #define CEN_ALPHA_SCRATCH_OFFSET SZ_8M #define CEN_ALPHA_SCRATCH_SIZE (5 * SZ_1M) struct centauri_ipc; struct centauri_wifi; -struct centauri_irqs { - struct pci_dev *pdev; - void *data; - unsigned int count; -}; - struct cen_alpha_resources { - struct pci_dev *pdev; void *context, *messages, *window; dma_addr_t context_dma, messages_dma, window_dma; void __iomem *bar; u32 index[4], mtr, mcr; }; -enum centauri_boot_phase { - CEN_BOOT_ROM, - CEN_BOOT_CYCLE, - CEN_BOOT_IPC, - CEN_BOOT_READY, - CEN_BOOT_FAILED, - CEN_BOOT_STOPPED, -}; - struct centauri { struct pci_dev *pdev; void __iomem *bar; resource_size_t size; - u32 platform_id, protocol; - const char *firmware_name; + u32 stage_offset; struct dentry *debug; struct mutex boot_lock; - struct delayed_work boot_work; - enum centauri_boot_phase boot_phase; - bool removing, ready; - void *firmware, *descriptor; - struct centauri_irqs irqs, alpha_irqs; - dma_addr_t firmware_dma, descriptor_dma; + void *firmware, *descriptor, *msi_page; + dma_addr_t firmware_dma, descriptor_dma, msi_dma; size_t firmware_size; u32 image_response; + u16 msi_original; + int msi_cap; bool boot_attempted; bool link_ready; struct centauri_ipc *ipc; struct centauri_wifi *wifi; }; -int centauri_request_irqs(struct centauri_irqs *irqs, struct pci_dev *pdev, - unsigned int count, irq_handler_t handler, void *data); -void centauri_free_irqs(struct centauri_irqs *irqs); - +int centauri_j813_port_cycle(struct pci_dev *pdev); int centauri_start_ipc(struct centauri *c); void centauri_free_ipc(struct centauri *c); int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *resources); -bool centauri_stop_dma(struct pci_dev *pdev); - -#if IS_ENABLED(CONFIG_APPLE_CENTAURI_WIFI) -irqreturn_t centauri_alpha_irq(int irq, void *data); +int centauri_alpha_log_buffer(struct centauri *c, void **cpu, dma_addr_t *dma, size_t *size); int centauri_wifi_register(struct centauri *c); void centauri_wifi_unregister(struct centauri *c); -#else -static inline irqreturn_t centauri_alpha_irq(int irq, void *data) -{ - return IRQ_HANDLED; -} -static inline int centauri_wifi_register(struct centauri *c) -{ - return 0; -} -static inline void centauri_wifi_unregister(struct centauri *c) -{ -} -#endif #endif diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c new file mode 100644 index 0000000000000..6eec46d831888 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Native DMA implementation of the measured J813 Control/Alpha bootstrap. + * Wire layout source: AppleCentauriCommon/scripts/alpha_wifi_scan.py v259. + * Polling and bounded bootstrap rings are deliberate bring-up restrictions. + */ +#include +#include +#include +#include +#include +#include +#include +#include "centauri.h" + +struct cen_buffer { + struct list_head list; + void *cpu; + dma_addr_t dma; + size_t size; +}; + +struct cen_context { + struct cen_buffer *buffer, *messages; + u32 index[4], mtr, mcr; + u16 tr_count, cr_count, message_head; +}; + +struct centauri_ipc { + struct list_head buffers; + struct pci_dev *function[3]; + void __iomem *bar[3]; + struct cen_context control, alpha; + struct cen_buffer *secondary, *window, *cchi_rx, *cchi_tx, *debug_rx; + struct cen_buffer *alpha_log; + u16 cchi_head, cchi_tail; +}; + +static struct cen_buffer *buffer_alloc(struct centauri *c, size_t size) +{ + struct cen_buffer *buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + + if (!buffer) + return NULL; + buffer->size = PAGE_ALIGN(size); + buffer->cpu = dma_alloc_coherent(&c->pdev->dev, buffer->size, &buffer->dma, GFP_KERNEL); + if (!buffer->cpu) { + kfree(buffer); + return NULL; + } + memset(buffer->cpu, 0, buffer->size); + list_add_tail(&buffer->list, &c->ipc->buffers); + return buffer; +} + +static int ftab_entry(const void *image, size_t size, const char tag[4], + u32 *offset, u32 *length, bool sparse) +{ + u32 count, i, off, len; + const u8 *table = image; + + if (size < 0x30 || memcmp(table + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(table + 0x28); + if (!count || count > 256 || 0x30 + count * 16 > size) + return -EINVAL; + for (i = 0; i < count; i++) { + const u8 *entry = table + 0x30 + 16 * i; + + if (memcmp(entry, tag, 4)) + continue; + off = get_unaligned_le32(entry + 4); + len = get_unaligned_le32(entry + 8); + if (off < 0x30 + count * 16 || !len || (u64)off + len > (sparse ? SZ_64M : size)) + return -EINVAL; + *offset = off; + *length = len; + return 0; + } + return -ENOENT; +} + +static int secondary_init(struct centauri *c) +{ + u32 offset, length, count, i, required; + const u8 *seed; + int ret; + + ret = ftab_entry(c->firmware, c->firmware_size, "2ftb", &offset, &length, false); + if (ret) + return ret; + seed = c->firmware + offset; + if (length < 0x30 || memcmp(seed + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(seed + 0x28); + if (!count || count > 256 || 0x30 + 16 * count != length) + return -EINVAL; + required = length; + for (i = 0; i < count; i++) { + u64 end = (u64)get_unaligned_le32(seed + 0x34 + i * 16) + + get_unaligned_le32(seed + 0x38 + i * 16); + + if (end > SZ_64M) + return -EINVAL; + required = max_t(u32, required, end); + } + c->ipc->secondary = buffer_alloc(c, required); + if (!c->ipc->secondary) + return -ENOMEM; + memcpy(c->ipc->secondary->cpu, seed, length); + dma_wmb(); + pci_write_config_dword(c->pdev, 0xf88, lower_32_bits(c->ipc->secondary->dma)); + pci_write_config_dword(c->pdev, 0xf8c, upper_32_bits(c->ipc->secondary->dma)); + pci_write_config_dword(c->pdev, 0xf90, required); + dev_info(&c->pdev->dev, "secondary memory: seed=%u required=%u\n", length, required); + return 0; +} + +static int context_init(struct centauri *c, struct cen_context *ctx, + u16 tr, u16 cr, dma_addr_t scratch, size_t scratch_size) +{ + u8 *header; + dma_addr_t dma; + int i; + + ctx->buffer = buffer_alloc(c, SZ_32K); + ctx->messages = buffer_alloc(c, SZ_16K); + if (!ctx->buffer || !ctx->messages) + return -ENOMEM; + ctx->tr_count = tr; + ctx->cr_count = cr; + ctx->index[0] = 0x78; + ctx->index[1] = ctx->index[0] + cr * 2; + ctx->index[2] = ctx->index[1] + tr * 2; + ctx->index[3] = ctx->index[2] + cr * 2; + ctx->mtr = ALIGN(ctx->index[3] + tr * 2, tr == 192 ? 16 : 2); + ctx->mcr = ALIGN(ctx->mtr + 256, 4); + header = ctx->buffer->cpu; + dma = ctx->buffer->dma; + put_unaligned_le16(768, header); + put_unaligned_le16(0x68, header + 2); + put_unaligned_le64(dma + 0x68, header + 8); + for (i = 0; i < 4; i++) + put_unaligned_le64(dma + ctx->index[i], header + 0x10 + 8 * i); + put_unaligned_le16(cr, header + 0x30); + put_unaligned_le16(tr, header + 0x32); + put_unaligned_le64(dma + ctx->mcr, header + 0x34); + put_unaligned_le64(dma + ctx->mtr, header + 0x3c); + put_unaligned_le16(16, header + 0x44); + put_unaligned_le16(16, header + 0x46); + put_unaligned_le64(scratch ?: dma + ALIGN(ctx->mcr + 256, 8), header + 0x58); + put_unaligned_le64(scratch_size, header + 0x60); + dma_wmb(); + return 0; +} + +static int handshake(void __iomem *bar, struct cen_context *ctx, + u32 status, u32 address, u32 control) +{ + u32 value; + int ret; + + writel(1, bar + control); + ret = readl_poll_timeout(bar + status, value, value == 1, 100, 2000000); + if (ret) + return ret; + writel(lower_32_bits(ctx->buffer->dma), bar + address); + writel(upper_32_bits(ctx->buffer->dma), bar + address + 4); + dma_wmb(); + writel(2, bar + control); + return readl_poll_timeout(bar + status, value, value == 2, 100, 2000000); +} + +static int message(struct cen_context *ctx, void __iomem *doorbell, + const void *payload, size_t length) +{ + u16 head = ctx->message_head; + u8 *descriptor; + u16 tail; + int ret; + + if (head >= 15 || !length || length > 512) + return -ENOSPC; + memcpy(ctx->messages->cpu + 512 * (head + 1), payload, length); + descriptor = ctx->buffer->cpu + ctx->mtr + head * 16; + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(ctx->messages->dma + 512 * (head + 1), descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + dma_wmb(); + writel(head + 1, doorbell); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) == head + 1, + 100, 1200000, false, *(__le16 *)(ctx->buffer->cpu + ctx->index[1])); + dma_rmb(); + ctx->message_head++; + return ret; +} + +static int open_transfer(struct centauri *c, u16 id, struct cen_buffer *buffer, + u16 footer) +{ + u8 msg[0x34] = { 1 }; + u32 words = footer >> 2, shift = min_t(u32, __ffs(words), 4); + + msg[2] = words >> shift; + msg[3] = shift << 4; + put_unaligned_le16(id, msg + 4); + put_unaligned_le64(buffer->dma, msg + 8); + put_unaligned_le64(~0ULL, msg + 0x10); + put_unaligned_le16(16, msg + 0x18); + put_unaligned_le16(0xffff, msg + 0x1a); + put_unaligned_le16(id, msg + 0x1c); + put_unaligned_le16(0x40, msg + 0x1e); + return message(&c->ipc->control, c->bar + 0x9000, msg, sizeof(msg)); +} + +static int cchi_init(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + u8 config[4] = {9, 0, 0xff, 0xff}, *entry; + u16 tail; + int i, ret; + + ipc->cchi_rx = buffer_alloc(c, SZ_32K); + ipc->cchi_tx = buffer_alloc(c, SZ_32K); + ipc->debug_rx = buffer_alloc(c, SZ_16K); + if (!ipc->cchi_rx || !ipc->cchi_tx || !ipc->debug_rx) + return -ENOMEM; + ret = message(&ipc->control, c->bar + 0x9000, config, sizeof(config)); + if (ret) + return ret; + ret = open_transfer(c, 4, ipc->cchi_rx, 1024); + if (!ret) + ret = open_transfer(c, 3, ipc->cchi_tx, 1024); + if (!ret) + ret = open_transfer(c, 5, ipc->debug_rx, 256); + if (ret) + return ret; + for (i = 0; i < 15; i++) { + entry = ipc->cchi_rx->cpu + i * 1040; + put_unaligned_le32(1 | (1024 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + entry = ipc->debug_rx->cpu + i * 272; + put_unaligned_le32(1 | (256 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + } + dma_wmb(); + writel(15, c->bar + 0x9010); + writel(15, c->bar + 0x9014); + entry = ipc->cchi_tx->cpu; + put_unaligned_le32(1 | (64 << 8), entry); + put_unaligned_le32(1, entry + 12); + put_unaligned_le16(64, entry + 16); /* CCHI gid 0, oid 0, seq 0 hello. */ + dma_wmb(); + writel(1, c->bar + 0x900c); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) != 0, + 100, 2000000, false, + *(__le16 *)(ipc->control.buffer->cpu + ipc->control.index[1] + 4 * 2)); + dma_rmb(); + if (ret) + return ret; + entry = ipc->cchi_rx->cpu; + if ((get_unaligned_le32(entry) >> 8) < 8 || entry[18] || entry[19]) + return -EPROTO; + ipc->cchi_head = ipc->cchi_tail = 1; + dev_info(&c->pdev->dev, "Control CCHI hello response received\n"); + return 0; +} + +static int memswap_handoff(struct centauri *c, unsigned int function, const char tag[4]) +{ + struct cen_buffer *secondary = c->ipc->secondary, *working; + u32 offset, size; + int ret; + + ret = ftab_entry(secondary->cpu, secondary->size, tag, &offset, &size, false); + if (ret) + return ret; + working = buffer_alloc(c, size); + if (!working) + return -ENOMEM; + dma_rmb(); + memcpy(working->cpu, secondary->cpu + offset, size); + dma_wmb(); + pci_write_config_dword(c->ipc->function[function], 0xf88, lower_32_bits(working->dma)); + pci_write_config_dword(c->ipc->function[function], 0xf8c, upper_32_bits(working->dma)); + pci_write_config_dword(c->ipc->function[function], 0xf90, size); + dev_info(&c->pdev->dev, "memswap function %u: %.4s %u bytes\n", function, tag, size); + return 0; +} + +static int enumerate_functions(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + u16 control; + int i, cap; + + pci_lock_rescan_remove(); + /* The ROM personality was single-function; boot exposes Alpha and Beta. */ + c->pdev->multifunction = 1; + pci_rescan_bus(c->pdev->bus); + pci_unlock_rescan_remove(); + for (i = 0; i < 3; i++) { + ipc->function[i] = pci_get_domain_bus_and_slot(pci_domain_nr(c->pdev->bus), + c->pdev->bus->number, PCI_DEVFN(0, i)); + if (!ipc->function[i] || ipc->function[i]->vendor != PCI_VENDOR_ID_APPLE || + (i && ipc->function[i]->device != 0x1901 + i)) + return -ENODEV; + if (!(pci_resource_flags(ipc->function[i], 0) & IORESOURCE_MEM)) + return -ENODEV; + ipc->bar[i] = pci_iomap(ipc->function[i], 0, 0); + if (!ipc->bar[i]) + return -ENOMEM; + cap = pci_find_capability(ipc->function[i], PCI_CAP_ID_MSI); + if (!cap) + return -ENODEV; + pci_read_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, &control); + if (!(control & PCI_MSI_FLAGS_64BIT)) + return -EOPNOTSUPP; + pci_write_config_dword(ipc->function[i], cap + PCI_MSI_ADDRESS_LO, + lower_32_bits(c->msi_dma + 0x400 * i)); + pci_write_config_dword(ipc->function[i], cap + PCI_MSI_ADDRESS_HI, + upper_32_bits(c->msi_dma + 0x400 * i)); + pci_write_config_word(ipc->function[i], cap + PCI_MSI_DATA_64, 0xa5a0 + 0x100 * i); + pci_write_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, + (control & ~PCI_MSI_FLAGS_QSIZE) | (5 << 4) | PCI_MSI_FLAGS_ENABLE); + pci_set_master(ipc->function[i]); + } + return 0; +} + +int centauri_start_ipc(struct centauri *c) +{ + struct centauri_ipc *ipc; + u32 stage; + int ret; + + if (!c->firmware || !c->link_ready || c->ipc || readl(c->bar + 0x8000) != 1) + return -EBUSY; + ipc = kzalloc(sizeof(*ipc), GFP_KERNEL); + if (!ipc) + return -ENOMEM; + INIT_LIST_HEAD(&ipc->buffers); + c->ipc = ipc; + ret = enumerate_functions(c); + if (ret) + goto failed; + ret = secondary_init(c); + if (ret) + goto failed; + ret = context_init(c, &ipc->control, 13, 1, 0, 0); + if (ret) + goto failed; + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + writel(0x62746632, c->bar + 0x8074); /* 2ftb */ + dma_wmb(); + writel(1, c->bar + 0x9070); + ret = readl_poll_timeout(c->bar + 0x8000, stage, stage == 2 || stage == 3, 100, 3000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + dev_info(&c->pdev->dev, "Control firmware reached OS stage\n"); + ret = handshake(c->bar, &ipc->control, 0x8050, 0x8054, 0x907c); + if (ret) + goto failed; + dev_info(&c->pdev->dev, "Control IPC handshake complete\n"); + ret = memswap_handoff(c, 0, "mswc"); + if (!ret) + ret = memswap_handoff(c, 2, "mswb"); + if (!ret) + ret = memswap_handoff(c, 1, "msww"); + if (ret) + goto failed; + ret = cchi_init(c); + if (ret) + goto failed; + ret = readl_poll_timeout(ipc->bar[1] + 8, stage, stage == 2 || stage == 3, 100, 2000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + ipc->window = buffer_alloc(c, SZ_16M); + if (!ipc->window) { + ret = -ENOMEM; + goto failed; + } + ret = context_init(c, &ipc->alpha, 192, 234, + ipc->window->dma + CEN_ALPHA_SCRATCH_OFFSET, CEN_ALPHA_SCRATCH_SIZE); + if (ret) + goto failed; + writel(lower_32_bits(ipc->window->dma), ipc->bar[1] + 0x18); + writel(upper_32_bits(ipc->window->dma), ipc->bar[1] + 0x1c); + writel(SZ_16M, ipc->bar[1] + 0x20); + ret = handshake(ipc->bar[1], &ipc->alpha, 0xc, 0x10, 0x1028); + if (ret) + goto failed; + dev_info(&c->pdev->dev, "Alpha Wi-Fi firmware running; IPC handshake complete\n"); + return centauri_wifi_register(c); +failed: + dev_err(&c->pdev->dev, "IPC bootstrap failed: %d (Control stage=%#x)\n", + ret, readl(c->bar + 0x8000)); + /* Published DMA stays pinned until every function is stopped at removal. */ + return ret; +} + +void centauri_free_ipc(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + struct cen_buffer *buffer, *next; + int i, cap; + u16 control; + + if (!ipc) + return; + centauri_wifi_unregister(c); + for (i = 0; i < 3; i++) { + if (!ipc->function[i]) + continue; + cap = pci_find_capability(ipc->function[i], PCI_CAP_ID_MSI); + if (cap) { + pci_read_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, &control); + pci_write_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, + control & ~PCI_MSI_FLAGS_ENABLE); + } + pci_clear_master(ipc->function[i]); + } + list_for_each_entry_safe(buffer, next, &ipc->buffers, list) { + dma_free_coherent(&c->pdev->dev, buffer->size, buffer->cpu, buffer->dma); + list_del(&buffer->list); + kfree(buffer); + } + for (i = 0; i < 3; i++) { + if (ipc->bar[i]) + pci_iounmap(ipc->function[i], ipc->bar[i]); + pci_dev_put(ipc->function[i]); + } + kfree(ipc); + c->ipc = NULL; +} + +int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *resources) +{ + struct centauri_ipc *ipc = c->ipc; + + if (!ipc || !ipc->window || !ipc->alpha.buffer || !ipc->alpha.messages) + return -ENODEV; + resources->context = ipc->alpha.buffer->cpu; + resources->context_dma = ipc->alpha.buffer->dma; + resources->messages = ipc->alpha.messages->cpu; + resources->messages_dma = ipc->alpha.messages->dma; + resources->window = ipc->window->cpu; + resources->window_dma = ipc->window->dma; + resources->bar = ipc->bar[1]; + memcpy(resources->index, ipc->alpha.index, sizeof(resources->index)); + resources->mtr = ipc->alpha.mtr; + resources->mcr = ipc->alpha.mcr; + return 0; +} + +/* Called once under the Wi-Fi lock. Published logging memory stays owned by + * IPC until every function has bus mastering disabled in centauri_free_ipc. + */ +int centauri_alpha_log_buffer(struct centauri *c, void **cpu, dma_addr_t *dma, size_t *size) +{ + struct centauri_ipc *ipc = c->ipc; + + if (!ipc || !ipc->window) + return -ENODEV; + if (!ipc->alpha_log) + ipc->alpha_log = buffer_alloc(c, SZ_4M); + if (!ipc->alpha_log) + return -ENOMEM; + *cpu = ipc->alpha_log->cpu; + *dma = ipc->alpha_log->dma; + *size = ipc->alpha_log->size; + return 0; +} diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c new file mode 100644 index 0000000000000..f3f7c6413f0a3 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Initial real-hardware N1 PCI/ACIPC backend for J813. + * Register offsets are from AppleCentauriCommon/PROTOCOL.md and its measured + * J813 prototype. Firmware execution and radio operations are separate gates. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "centauri.h" + +static struct dentry *centauri_debug; + +static int centauri_status_show(struct seq_file *s, void *unused) +{ + struct centauri *c = s->private; + + seq_printf(s, "pci=%s vendor=%04x device=%04x\n", + pci_name(c->pdev), c->pdev->vendor, c->pdev->device); + if (c->link_ready) + seq_printf(s, "bar0_size=%pa exec_stage=%#x\n", &c->size, + readl(c->bar + c->stage_offset)); + else + seq_puts(s, "link unavailable; endpoint reads disabled\n"); + seq_printf(s, "transport=hardware-pci firmware_boot=%s image_response=%#x wifi=%s\n", + c->boot_attempted ? "attempted" : "not-started", c->image_response, + READ_ONCE(c->wifi) ? "registered" : "not-owned-by-this-function"); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(centauri_status); + +static void centauri_free_boot(struct centauri *c) +{ + struct device *dev = &c->pdev->dev; + + if (!c->link_ready && c->boot_attempted) { + dev_warn(dev, "retaining DMA buffers after failed link transition; reboot required\n"); + return; + } + /* Stop device writes before any DMA storage is released. */ + centauri_free_ipc(c); + if (c->msi_cap) + pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, + c->msi_original & ~PCI_MSI_FLAGS_ENABLE); + pci_clear_master(c->pdev); + if (c->firmware) + dma_free_coherent(dev, c->firmware_size, c->firmware, c->firmware_dma); + if (c->descriptor) + dma_free_coherent(dev, SZ_16K, c->descriptor, c->descriptor_dma); + if (c->msi_page) + dma_free_coherent(dev, SZ_16K, c->msi_page, c->msi_dma); + c->firmware = c->descriptor = c->msi_page = NULL; +} + +static int centauri_rom_boot(struct centauri *c) +{ + const struct firmware *fw; + struct device *dev = &c->pdev->dev; + u32 stage; + int ret; + + if ((c->pdev->device != 0x1900 && c->pdev->device != 0x1901) || + !of_machine_is_compatible("apple,j813")) + return -ENODEV; + if (!c->link_ready || c->boot_attempted) + return -EBUSY; + stage = readl(c->bar + 0x8000); + if (stage != 0 && stage != 4 && stage != 1) + return -EBUSY; + ret = request_firmware(&fw, "apple/centauri/j813-ftab.bin", dev); + if (ret) + return ret; + if (fw->size < 0x30 || fw->size > SZ_64M || + memcmp(fw->data + 0x20, "rkosftab", 8)) { + ret = -EINVAL; + goto release; + } + c->firmware_size = PAGE_ALIGN(fw->size); + c->firmware = dma_alloc_coherent(dev, c->firmware_size, &c->firmware_dma, GFP_KERNEL); + c->descriptor = dma_alloc_coherent(dev, SZ_16K, &c->descriptor_dma, GFP_KERNEL); + c->msi_page = dma_alloc_coherent(dev, SZ_16K, &c->msi_dma, GFP_KERNEL); + if (!c->firmware || !c->descriptor || !c->msi_page) { + ret = -ENOMEM; + goto free_buffers; + } + memset(c->firmware, 0, c->firmware_size); + memcpy(c->firmware, fw->data, fw->size); + memset(c->descriptor, 0, SZ_16K); + memset(c->msi_page, 0, SZ_16K); + /* Measured ROM scatter entry: u64 DMA, u32 length, u32 flags, u64 x2 zero. */ + put_unaligned_le64(c->firmware_dma, c->descriptor); + put_unaligned_le32(ALIGN(fw->size, SZ_4K), c->descriptor + 8); + c->msi_cap = pci_find_capability(c->pdev, PCI_CAP_ID_MSI); + if (!c->msi_cap) { + ret = -ENODEV; + goto free_buffers; + } + pci_read_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, &c->msi_original); + if (!(c->msi_original & PCI_MSI_FLAGS_64BIT)) { + ret = -EOPNOTSUPP; + goto free_buffers; + } + /* Poll the measured DMA-backed MSI mailbox until native IRQ support exists. */ + pci_write_config_dword(c->pdev, c->msi_cap + PCI_MSI_ADDRESS_LO, lower_32_bits(c->msi_dma)); + pci_write_config_dword(c->pdev, c->msi_cap + PCI_MSI_ADDRESS_HI, upper_32_bits(c->msi_dma)); + pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_DATA_64, 0xa5a0); + pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, + (c->msi_original & ~PCI_MSI_FLAGS_QSIZE) | PCI_MSI_FLAGS_ENABLE); + pci_set_master(c->pdev); + writel(0x00248142, c->bar + 0x807c); /* J813 board 0x24, T8142 chip. */ + writel(0, c->bar + 0x803c); + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + dma_wmb(); + c->boot_attempted = true; + if (stage == 4 || stage == 1) { + c->image_response = readl(c->bar + 0x8040); + dev_info(dev, "adopted firmware stage %u; native DMA buffers ready\n", stage); + ret = 0; + goto release; + } + writel(1, c->bar + 0x9000); + ret = readl_poll_timeout(c->bar + 0x8040, c->image_response, + c->image_response != 0, 20000, 20000000); + if (!ret) + ret = readl_poll_timeout(c->bar + 0x8000, stage, stage != 0, 20000, 20000000); + else + stage = readl(c->bar + 0x8000); + dma_rmb(); + dev_info(dev, "ROM firmware result: response=%#x stage=%#x mailbox=%#x poll=%d\n", + c->image_response, stage, READ_ONCE(*(u32 *)c->msi_page), ret); + /* Keep DMA buffers owned until device removal, even on timeout. */ + if (!ret && stage != 4) + ret = -EIO; + goto release; +free_buffers: + centauri_free_boot(c); +release: + release_firmware(fw); + return ret; +} + +static ssize_t centauri_boot_write(struct file *file, const char __user *buffer, + size_t count, loff_t *pos) +{ + struct centauri *c = file->private_data; + u32 value; + int ret = kstrtou32_from_user(buffer, count, 0, &value); + + if (ret || value < 1 || value > 3) + return ret ?: -EINVAL; + mutex_lock(&c->boot_lock); + if (value == 1) { + ret = centauri_rom_boot(c); + } else if (value == 3) { + ret = centauri_start_ipc(c); + } else if (!c->firmware || !c->link_ready || readl(c->bar + 0x8000) != 4) { + ret = -EBUSY; + } else { + c->link_ready = false; + ret = centauri_j813_port_cycle(c->pdev); + if (!ret) { + c->link_ready = true; + dev_info(&c->pdev->dev, "post-reset execution stage=%#x\n", + readl(c->bar + 0x8000)); + } + } + mutex_unlock(&c->boot_lock); + return ret ?: count; +} + +static const struct file_operations centauri_boot_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = centauri_boot_write, +}; + +static int centauri_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct centauri *c; + dma_addr_t dma; + void *page; + int ret; + + c = devm_kzalloc(&pdev->dev, sizeof(*c), GFP_KERNEL); + if (!c) + return -ENOMEM; + c->pdev = pdev; + c->link_ready = true; + mutex_init(&c->boot_lock); + c->stage_offset = id->driver_data; + ret = pcim_enable_device(pdev); + if (ret) + return ret; + c->size = pci_resource_len(pdev, 0); + if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || + c->size < c->stage_offset + sizeof(u32)) + return dev_err_probe(&pdev->dev, -ENODEV, "invalid ACIPC BAR0\n"); + ret = pcim_iomap_regions(pdev, BIT(0), "apple_centauri"); + if (ret) + return ret; + c->bar = pcim_iomap_table(pdev)[0]; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return dev_err_probe(&pdev->dev, ret, "40-bit DMA unavailable\n"); + /* Test Linux's DART allocation without publishing it to the device. */ + page = dma_alloc_coherent(&pdev->dev, SZ_16K, &dma, GFP_KERNEL); + if (!page) + return dev_err_probe(&pdev->dev, -ENOMEM, "DART DMA allocation failed\n"); + memset(page, 0, SZ_16K); + dev_info(&pdev->dev, "ACIPC endpoint reachable: stage=%#x, 40-bit DMA mapping=%pad\n", + readl(c->bar + c->stage_offset), &dma); + dma_free_coherent(&pdev->dev, SZ_16K, page, dma); + /* Bus mastering remains disabled until the firmware transport owns buffers. */ + pci_clear_master(pdev); + pci_set_drvdata(pdev, c); + c->debug = debugfs_create_dir(pci_name(pdev), centauri_debug); + debugfs_create_file("status", 0400, c->debug, c, ¢auri_status_fops); + debugfs_create_file("rom_boot", 0200, c->debug, c, ¢auri_boot_fops); + return 0; +} + +static void centauri_remove(struct pci_dev *pdev) +{ + struct centauri *c = pci_get_drvdata(pdev); + + centauri_wifi_unregister(c); + debugfs_remove_recursive(c->debug); + centauri_free_boot(c); +} + +static const struct pci_device_id centauri_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1900), .driver_data = 0x8000 }, + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1901), .driver_data = 0x8000 }, + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1902), .driver_data = 0x0008 }, + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903), .driver_data = 0x0018 }, + { } +}; +MODULE_DEVICE_TABLE(pci, centauri_ids); + +static struct pci_driver centauri_driver = { + .name = "apple_centauri", + .id_table = centauri_ids, + .probe = centauri_probe, + .remove = centauri_remove, +}; + +static int __init centauri_init(void) +{ + int ret; + + centauri_debug = debugfs_create_dir("apple_centauri", NULL); + ret = pci_register_driver(¢auri_driver); + if (ret) + debugfs_remove_recursive(centauri_debug); + return ret; +} + +static void __exit centauri_exit(void) +{ + pci_unregister_driver(¢auri_driver); + debugfs_remove_recursive(centauri_debug); +} +module_init(centauri_init); +module_exit(centauri_exit); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 Centauri native PCI and ACIPC bring-up backend"); +MODULE_FIRMWARE("apple/centauri/j813-ftab.bin"); diff --git a/drivers/net/wireless/apple/centauri/centauri_port.c b/drivers/net/wireless/apple/centauri/centauri_port.c new file mode 100644 index 0000000000000..eea0a1ba11e0e --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_port.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* J813 preboot -> boot port cycle, transcribed from the measured proxy + * alpha_wifi_scan.py pcie_cycle/apple_teardown/apple_bringup sequence. + * These apertures were verified from the live ADT before this Linux boot. + * This temporary board helper belongs in the host-controller driver once + * T8142 platform/clock/GPIO bindings are available. + */ +#include +#include +#include +#include +#include +#include "centauri.h" + +static void update(void __iomem *base, u32 offset, u32 clear, u32 set) +{ + writel((readl(base + offset) & ~clear) | set, base + offset); +} + +static bool wait_bits(void __iomem *address, u32 mask, u32 wanted, u32 us) +{ + u32 value; + + return !readl_poll_timeout(address, value, (value & mask) == wanted, 100, us); +} + +int centauri_j813_port_cycle(struct pci_dev *pdev) +{ + static const u32 reset[][2] = { + {0x88, 0x110}, {0x100, ~0U}, {0x148, ~0U}, {0x210, ~0U}, + {0x80, 0}, {0x84, 0}, {0x104, 0xfffffff0}, {0x124, 0x100}, + {0x16c, 0}, {0x13c, 0}, {0x800, 0x00100100}, + {0x808, 0x001001ff}, {0x82c, 0x00010000}, + }; + static const u32 tune[][2] = { + {0x130, 0x03020000}, {0x140, 0x10}, {0x144, 0x00253770}, + {0x21c, 0}, {0x834, 0}, {0x83c, 0}, + }; + struct pci_dev *bridge = pdev->bus->self; + void __iomem *port, *phy, *pad; + u32 value, gpio, identity; + u16 link; + unsigned int lanes, i; + int ret = -ETIMEDOUT, pm; + bool ack, l2; + + if (!of_machine_is_compatible("apple,j813") || !bridge || + bridge->vendor != PCI_VENDOR_ID_APPLE || bridge->device != 0x100c) + return -ENODEV; + port = ioremap(0x488028000ULL, 0x4000); + phy = ioremap(0x48f010000ULL, 0x1000); + pad = ioremap(0x39a000300ULL, sizeof(u32)); + if (!port || !phy || !pad) { + ret = -ENOMEM; + goto out; + } + gpio = readl(pad); + if (gpio & BIT(21)) { + ret = -EPERM; + goto out; + } + ret = pci_save_state(bridge); + if (ret) + goto out; + ret = pci_save_state(pdev); + if (ret) + goto out; + pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &link); + lanes = (link & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT; + lanes = lanes >= 1 && lanes <= 16 ? BIT(lanes) - 1 : 1; + pm = pci_find_capability(pdev, PCI_CAP_ID_PM); + if (pm) { + pci_write_config_word(pdev, pm + PCI_PM_CTRL, 0x103); + msleep(10); + } + update(port, 0x800, 0, 0x100); + writel(0x11, port + 0x88); + ack = wait_bits(port + 0x88, 1, 0, 500000); + l2 = wait_bits(port + 0x208, BIT(6), BIT(6), 500000); + dev_info(&pdev->dev, "port preboot transition: PME ack=%d L2=%d\n", ack, l2); + /* Endpoint reads cease here until the root LTSSM is back in L0. */ + pci_write_config_word(bridge, PCI_COMMAND, 0); + pci_read_config_dword(bridge, PCI_PRIMARY_BUS, &value); + pci_write_config_dword(bridge, PCI_PRIMARY_BUS, value & 0xff0000ff); + update(phy, 0, BIT(31), 0); + writel((gpio & ~0xfU) | 2, pad); + if (readl(pad) != ((gpio & ~0xfU) | 2)) { + ret = -EIO; + goto out; + } + wait_bits(port + 0x804, BIT(2), 0, 50000); + wait_bits(port + 0x820, 0x1ff, 0, 100000); + update(port, 0x13c, 0, 0x100); + for (i = 0; i < 200; i++) { + pci_read_config_dword(bridge, 0x728, &value); + if ((value & 0x3f) == 0x14) + break; + usleep_range(1000, 1200); + } + wait_bits(port + 0x208, BIT(12), 0, 200000); + readl(phy); + wait_bits(port + 0xa8, 3, 0, 10000); + update(port, 0x82c, 0, BIT(24)); + wait_bits(port + 0xa8, lanes << 16, lanes << 16, 20000); + update(port, 0x82c, 0, BIT(16)); + udelay(10); + update(port, 0x82c, 1, 0); + udelay(10); + update(port, 0x82c, BIT(24), 0); + update(port, 0x13c, 0x100, 0); + update(phy, 0, BIT(31), 0); + update(phy, 0, 0x400, 0); + update(phy, 0, 0x200, 0); + update(phy, 0, 0, 0x10); + update(phy, 0, 1, 0); + update(phy, 0, 2, 0); + writel(~0U, port + 0x100); + update(port, 0x800, 1, 0); + wait_bits(port + 0x804, 1, 0, 200000); + update(phy, 0, BIT(31), 0); + for (i = 0; i < ARRAY_SIZE(reset); i++) + writel(reset[i][1], port + reset[i][0]); + for (i = 0; i < 16; i++) + writel(0, port + 0x3000 + 4 * i); + for (i = 0; i < 256; i++) + writel(0, port + 0x3800 + 4 * i); + for (i = 0; i < ARRAY_SIZE(tune); i++) + writel(tune[i][1], port + tune[i][0]); + update(port, 0x800, 0, 1); + udelay(10); + update(phy, 0, 0, 1); + if (!wait_bits(phy, 4, 4, 50000)) + goto timed_out; + update(phy, 0, 0, 2); + if (!wait_bits(phy, 8, 8, 50000)) + goto timed_out; + update(phy, 0, 0x10, 0); + udelay(100); + update(phy, 0, 0, 0x200); + update(phy, 0, 0, 0x400); + udelay(100); + update(port, 0x82c, 0, 1); + writel((gpio & ~0xfU) | 3, pad); + udelay(100); + readl(phy); + wait_bits(port + 0xa8, 3, 0, 20000); + update(port, 0x82c, BIT(16), 0); + if (!wait_bits(port + 0x804, 1, 1, 200000)) + goto timed_out; + update(port, 0x800, 0x100, 0); + pci_read_config_dword(bridge, 0x8bc, &value); + pci_write_config_dword(bridge, 0x8bc, value ^ 1); + pci_read_config_dword(bridge, 0x80, &value); + pci_write_config_dword(bridge, 0x80, value | 0x400); + update(phy, 0, 0, BIT(31)); + update(port, 0x80, 0, 1); + for (i = 0; i < 2000; i++) { + pci_read_config_dword(bridge, 0x728, &value); + if ((value & 0x3f) == 0x11) + break; + usleep_range(1000, 1200); + } + if (i == 2000) + goto timed_out; + msleep(100); + pci_restore_state(bridge); + /* Bus numbers/windows now restored; endpoint config may be read again. */ + pci_read_config_dword(pdev, PCI_VENDOR_ID, &identity); + if (identity != 0x1901106b && identity != 0x1900106b) { + ret = -ENODEV; + goto out; + } + pci_restore_state(pdev); + if (pm) { + pci_write_config_word(pdev, pm + PCI_PM_CTRL, 0); + msleep(10); + } + dev_info(&pdev->dev, "port transition complete: endpoint=%#x LTSSM=%#x\n", + identity, value & 0x3f); + ret = 0; + goto out; +timed_out: + ret = -ETIMEDOUT; +out: + if (pad) + iounmap(pad); + if (phy) + iounmap(phy); + if (port) + iounmap(port); + return ret; +} diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c new file mode 100644 index 0000000000000..21201241f49ec --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -0,0 +1,2312 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* cfg80211 connection to the measured Alpha command/management rings. + * Control queues recycle across hardware-validated wraps. Packet data uses + * a separate protocol; never report transmit success before implementing it. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "centauri.h" + +#define CR_BASE 0x100000 +#define TR_BASE 0x400000 +#define RING_SPAN 0x20000 +#define TX_BUFFER 0x600000 +#define BUFFER_SIZE 2048 +#define DATA_TX_BUFFER 0xd00000 +#define DATA_RX_BUFFER 0xe00000 +#define DATA_TX_SLOTS 128 +#define DATA_TX_QUEUE_LIMIT 64 + +/* The firmware owns window +8 MiB through +13 MiB as scratch memory. */ +static_assert(DATA_TX_BUFFER >= CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); +static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= DATA_RX_BUFFER); + +static bool verify_pmk; +module_param(verify_pmk, bool, 0400); +MODULE_PARM_DESC(verify_pmk, "Compare firmware PMK with the requested key without exposing either"); + +static bool eapol_after_assoc; +module_param(eapol_after_assoc, bool, 0644); +MODULE_PARM_DESC(eapol_after_assoc, "One EAPOL-Start probe at the next successful association event"); + +static bool link_diagnostics; +module_param(link_diagnostics, bool, 0644); +MODULE_PARM_DESC(link_diagnostics, "Query non-secret firmware link status during association"); + +static bool eapol_control_profile; +module_param(eapol_control_profile, bool, 0644); +MODULE_PARM_DESC(eapol_control_profile, "Use the measured control TX profile for the EAPOL-Start probe"); + +static bool eapol_unicast; +module_param(eapol_unicast, bool, 0644); +MODULE_PARM_DESC(eapol_unicast, "Address the bounded EAPOL-Start probe to the associated AP"); + +static bool eapol_relative_lifetime; +module_param(eapol_relative_lifetime, bool, 0644); +MODULE_PARM_DESC(eapol_relative_lifetime, "Use a measured two-second relative expiration for the probe"); + +static bool data_tx_broadcast; +module_param(data_tx_broadcast, bool, 0400); +MODULE_PARM_DESC(data_tx_broadcast, "Diagnostic: open TR 25 with measured broadcast metadata; transmit only broadcast Ethernet frames"); + +static bool data_tx_control_profile; +module_param(data_tx_control_profile, bool, 0600); +MODULE_PARM_DESC(data_tx_control_profile, "Diagnostic: select configured TX policy 1 for normal data; apply tx_config action 2 first"); + +static bool data_tx_native_prepare = true; +module_param(data_tx_native_prepare, bool, 0600); +MODULE_PARM_DESC(data_tx_native_prepare, "Use native packet preparation value 4; disable only to reproduce the legacy data TX failure"); + +static bool station_link_addresses; +module_param(station_link_addresses, bool, 0400); +MODULE_PARM_DESC(station_link_addresses, "Diagnostic: use the netdev address for each firmware link in this single-link WPA2 client"); + +struct centauri_rx_snapshot { + __le32 version, count; + struct { + u8 completion[32]; + u8 packet[4096]; + } sample[8]; +}; + +struct centauri_wifi { + struct centauri *control; + struct cen_alpha_resources a; + struct wiphy *wiphy; + struct wireless_dev wdev; + struct net_device *netdev; + struct work_struct scan_work; + struct work_struct connect_work; + struct delayed_work link_work; + struct cfg80211_scan_request *scan_request; + struct mutex lock; + /* The logger has its own coherent allocation. Reads/acknowledgements + * must remain possible while scan/connect hold the command mutex. + * Initialization takes lock then fwlog_lock; readers take only the latter. + */ + struct mutex fwlog_lock; + u16 main_head, command_head, sequence; + u32 rx_seen[5]; + u16 rx_next[5], rx_head[5]; + u16 reply_sequence, reply_status; + u32 reply_command, channels_seen, scans_done, beacons_seen; + bool reply_valid, registered, configured; + bool stopping; + bool connecting, connected, connect_event, disconnect_event; + bool associated, pmk_seen, pmk_matches; + u8 expected_pmk[32]; + u16 connect_status; + u8 bssid[ETH_ALEN], connect_payload[1024], request_ie[256]; + size_t connect_size, request_ie_size; + u8 passphrase[64]; + size_t passphrase_size; + struct dentry *passphrase_file; + struct dentry *data_probe_file; + struct dentry *data_status_file; + struct dentry *data_rx_snapshot_file; + struct centauri_rx_snapshot rx_snapshot; + u32 data_rx_received, data_rx_reposted, data_rx_bad; + struct sk_buff_head data_tx_queue; + u16 data_tx_length[DATA_TX_SLOTS]; + bool data_tx_busy[DATA_TX_SLOTS]; + u32 data_tx_submitted, data_tx_completed, data_tx_failed; + u32 data_tx_coalesced, data_tx_bad; + struct dentry *station_stats_file; + struct dentry *eapol_start_file; + struct dentry *country_file; + struct dentry *link_status_file; + struct dentry *tx_config_file; + struct dentry *host_wakeup_file; + struct dentry *fwlog_start_file, *fwlog_status_file, *fwlog_buffer_file, *fwlog_ack_file; + void *fwlog_buffer; + dma_addr_t fwlog_dma; + size_t fwlog_size; + bool fwlog_attempted, fwlog_ready; + u8 tx_profile[10], tx_defaults_applied; + bool tx_profile_valid; + u8 link_count, link_prime[3], link_channel[3], link_band[3], emlsr_mode; + bool link_status_valid, emlsr_valid; + u16 link_status_size; + u16 link_tags[12], link_lengths[12]; + u8 link_tlv_count; + u8 country[3]; + bool country_valid; + u8 station_stats[0x378]; + bool station_stats_valid; + u8 rx_crypto_stats[64]; + bool rx_crypto_stats_valid; + struct dentry *rx_crypto_stats_file; + u8 dot11_mib_stats[192]; + bool dot11_mib_stats_valid; + struct dentry *dot11_mib_stats_file; + u8 tx_power_report[280]; + bool tx_power_report_valid; + struct dentry *tx_power_report_file; + u16 stats_reply_size, stats_tag, stats_length; + u32 data_probe_done; + struct ieee80211_supported_band bands[2]; + struct ieee80211_channel channels[39]; +}; + +/* These helpers require wifi->lock, including from the connect worker. */ +static int transmit_eapol_start(struct centauri_wifi *wifi); +static int query_station_stats(struct centauri_wifi *wifi); +static int query_link_status(struct centauri_wifi *wifi); + +static const u32 rx_buffer[5] = { 0, 0, 0x620000, 0x680000, 0x700000 }; +static const u16 ring_size[5] = { 0, 64, 192, 96, 192 }; +static const u16 five_ghz[] = { + 36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, + 116, 120, 124, 128, 132, 136, 140, 144, 149, 153, 157, 161, 165, +}; +static struct ieee80211_rate rates[] = { + { .bitrate = 10 }, { .bitrate = 20 }, { .bitrate = 55 }, { .bitrate = 110 }, + { .bitrate = 60 }, { .bitrate = 90 }, { .bitrate = 120 }, { .bitrate = 180 }, + { .bitrate = 240 }, { .bitrate = 360 }, { .bitrate = 480 }, { .bitrate = 540 }, +}; +static const u32 cipher_suites[] = { + WLAN_CIPHER_SUITE_CCMP, WLAN_CIPHER_SUITE_AES_CMAC, +}; + +static u16 index_read(struct centauri_wifi *wifi, unsigned int array, unsigned int index) +{ + return le16_to_cpu(READ_ONCE(*(__le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index))); +} + +static void index_write(struct centauri_wifi *wifi, unsigned int array, + unsigned int index, u16 value) +{ + WRITE_ONCE(*(__le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index), cpu_to_le16(value)); +} + +static void completions(struct centauri_wifi *wifi) +{ + unsigned int vector; + + for (vector = 0; vector <= 4; vector++) + index_write(wifi, 2, vector, index_read(wifi, 0, vector)); + dma_wmb(); +} + +static int alpha_message(struct centauri_wifi *wifi, const void *message, size_t size) +{ + u16 head = wifi->main_head; + u8 *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(2000); + + if (head >= 15 || size > 512) + return -ENOSPC; + memcpy(wifi->a.messages + (head + 1) * 512, message, size); + descriptor = wifi->a.context + wifi->a.mtr + 16 * head; + put_unaligned_le32(1 | (size << 8), descriptor); + put_unaligned_le64(wifi->a.messages_dma + (head + 1) * 512, descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + index_write(wifi, 3, 0, head + 1); + if (head) + index_write(wifi, 3, 8, head + 1); + dma_wmb(); + wifi->main_head++; + writel(1, wifi->a.bar + 0x1004); + do { + if (readl(wifi->a.bar + 8) != 2) + return -EIO; + if (index_read(wifi, 1, 0) >= head + 1) { + dma_rmb(); + completions(wifi); + return 0; + } + usleep_range(100, 200); + } while (time_before(jiffies, deadline)); + return -ETIMEDOUT; +} + +static int open_pair(struct centauri_wifi *wifi, u16 id) +{ + u8 cr[0x2c] = { 2 }, tr[0x34] = { 1 }; + int ret; + + put_unaligned_le32(id | (id << 16), cr + 4); + put_unaligned_le64(wifi->a.window_dma + CR_BASE + (id - 1) * RING_SPAN, cr + 8); + put_unaligned_le16(1024, cr + 0x10); + put_unaligned_le16(0xffff, cr + 0x12); + ret = alpha_message(wifi, cr, sizeof(cr)); + if (ret) + return ret; + put_unaligned_le32(id | (id << 16), tr + 4); + put_unaligned_le64(wifi->a.window_dma + TR_BASE + (id - 1) * RING_SPAN, tr + 8); + put_unaligned_le64(~0ULL, tr + 0x10); + put_unaligned_le16(ring_size[id], tr + 0x18); + put_unaligned_le16(id, tr + 0x1a); + put_unaligned_le16(id, tr + 0x1c); + put_unaligned_le16(0x40, tr + 0x1e); + put_unaligned_le16(2, tr + 0x24); + return alpha_message(wifi, tr, sizeof(tr)); +} + +static void report_beacon(struct centauri_wifi *wifi, const u8 *payload, size_t size) +{ + const u8 *frame, *ie, *end; + struct ieee80211_channel *channel; + struct cfg80211_bss *bss; + u16 frame_size; + int number = 0, frequency, signal; + enum nl80211_band band; + + if (!wifi->registered || size < 0x60 + 36) + return; + /* The ACIPC packet length is authoritative; the vendor descriptor's + * separate 802.11 length can include a stripped FCS. The measured parser + * consumes the frame bytes that were actually returned. + */ + frame_size = size - 0x60; + frame = payload + 0x60; + if (!ieee80211_is_beacon(*(__le16 *)frame) && !ieee80211_is_probe_resp(*(__le16 *)frame)) + return; + end = frame + frame_size; + for (ie = frame + 36; ie + 2 <= end && ie + 2 + ie[1] <= end; ie += 2 + ie[1]) { + if ((ie[0] == WLAN_EID_DS_PARAMS || ie[0] == WLAN_EID_HT_OPERATION) && ie[1]) + number = ie[2]; + } + if (!number) + return; + band = number <= 14 ? NL80211_BAND_2GHZ : NL80211_BAND_5GHZ; + frequency = ieee80211_channel_to_frequency(number, band); + channel = ieee80211_get_channel(wifi->wiphy, frequency); + if (!channel) + return; + signal = max((s8)payload[0x16], (s8)payload[0x17]); + if (signal >= 0 || signal < -127) + signal = -95; + bss = cfg80211_inform_bss_frame(wifi->wiphy, channel, + (struct ieee80211_mgmt *)frame, frame_size, signal * 100, GFP_KERNEL); + if (bss) { + wifi->beacons_seen++; + cfg80211_put_bss(wifi->wiphy, bss); + } +} + +static void post_receive(struct centauri_wifi *wifi, unsigned int ring, u16 slot) +{ + u8 *descriptor = wifi->a.window + TR_BASE + (ring - 1) * RING_SPAN + slot * 16; + u32 offset = rx_buffer[ring] + slot * BUFFER_SIZE; + + memset(wifi->a.window + offset, 0, BUFFER_SIZE); + put_unaligned_le32(1 | (BUFFER_SIZE << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + offset, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); +} + +/* Host-owned RX buffer cookies and completion length are the ACIPC fields + * validated in probe 28. Probe 29 validates the footer's prefix length and + * Ethernet framing with IPv4 header checksums and IPv6 payload lengths. + * Called with wifi->lock, including during connect/command polling. + */ +static void drain_data_rx(struct centauri_wifi *wifi) +{ + u16 head, tail, post; + unsigned int processed = 0; + + if (!(wifi->data_probe_done & BIT(4)) || readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 207); + tail = index_read(wifi, 2, 207); + post = index_read(wifi, 3, 12); + if (head >= 128 || tail >= 128 || post >= 128) { + wifi->data_rx_bad++; + return; + } + dma_rmb(); + while (tail != head && processed < 127) { + u8 *completion = wifi->a.window + 0x240000 + tail * 32; + u32 header = get_unaligned_le32(completion); + u32 info = get_unaligned_le32(completion + 4); + u16 cookie = info & 0xffff, length = info >> 16; + u16 next = (post + 1) % 128; + u32 count = le32_to_cpu(wifi->rx_snapshot.count); + u8 *packet, *descriptor; + u32 buffer; + + if ((header >> 16) != 12 || !cookie || cookie > 127 || + length > 4096 || get_unaligned_le32(completion + 8)) { + wifi->data_rx_bad++; + break; + } + /* Do not overwrite a descriptor the receive engine still owns. */ + if (next == index_read(wifi, 1, 12)) + break; + buffer = DATA_RX_BUFFER + (cookie - 1) * 4096; + packet = wifi->a.window + buffer; + if (count < ARRAY_SIZE(wifi->rx_snapshot.sample)) { + memcpy(wifi->rx_snapshot.sample[count].completion, completion, 32); + memcpy(wifi->rx_snapshot.sample[count].packet, packet, length); + wifi->rx_snapshot.version = cpu_to_le32(1); + wifi->rx_snapshot.count = cpu_to_le32(count + 1); + } + wifi->data_rx_received++; + if (wifi->connected && netif_running(wifi->netdev)) { + u32 footer = get_unaligned_le32(completion + 12); + unsigned int padding = (footer >> 12) & 15; + struct sk_buff *skb = NULL; + + /* Extended prefix headers are not decoded yet. */ + if (!(footer & BIT(24)) && length >= padding + ETH_HLEN) + skb = netdev_alloc_skb_ip_align(wifi->netdev, length - padding); + if (skb) { + skb_put_data(skb, packet + padding, length - padding); + skb->protocol = eth_type_trans(skb, wifi->netdev); + skb->ip_summed = CHECKSUM_NONE; + wifi->netdev->stats.rx_packets++; + wifi->netdev->stats.rx_bytes += length - padding; + netif_rx(skb); + } else { + wifi->netdev->stats.rx_dropped++; + } + } else { + wifi->netdev->stats.rx_dropped++; + } + descriptor = wifi->a.window + 0x520000 + post * 16; + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le32(cookie, descriptor + 12); + wifi->data_rx_reposted++; + tail = (tail + 1) % 128; + post = next; + processed++; + } + if (processed) { + dma_wmb(); + index_write(wifi, 2, 207, tail); + index_write(wifi, 3, 12, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + } +} + +/* Serialized with the firmware poller. Payload slots remain owned until a + * CR completion, even if the device has already consumed the TR descriptor. + */ +static void drain_data_tx(struct centauri_wifi *wifi) +{ + u16 head, tail, post; + unsigned int budget = 256; + bool posted = false, completed = false; + + if (!(wifi->data_probe_done & BIT(5)) || readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 192); + tail = index_read(wifi, 2, 192); + post = index_read(wifi, 3, 25); + if (head >= 2048 || tail >= 2048 || post >= DATA_TX_SLOTS) + return; + dma_rmb(); + while (tail != head && budget--) { + u8 *entry = wifi->a.window + 0x200000 + tail * 32; + u16 ring = get_unaligned_le32(entry) >> 16; + u16 cookie = get_unaligned_le16(entry + 4); + u16 count = get_unaligned_le16(entry + 9); + u8 status = entry[12]; + + if (ring == 25) { + unsigned int first, i; + + /* Airship cr_controller::sync maps CD bytes 9:10 to the + * transfer count. With the multicomp flag (CD byte 1 bit 0), + * the cookie names the LAST transfer; prior transfers share + * its completion status. We submit one contiguous DMA buffer + * per slot, so those preceding cookies must all still be busy. + */ + if (!cookie || cookie > DATA_TX_SLOTS || !count || + count >= DATA_TX_SLOTS || !(entry[1] & 1)) { + wifi->data_tx_bad++; + break; + } + first = (cookie + DATA_TX_SLOTS - count) % DATA_TX_SLOTS; + for (i = 0; i < count; i++) + if (!wifi->data_tx_busy[(first + i) % DATA_TX_SLOTS]) + break; + if (i != count) { + wifi->data_tx_bad++; + break; + } + if (count > 1) + wifi->data_tx_coalesced++; + for (i = 0; i < count; i++) { + unsigned int slot = (first + i) % DATA_TX_SLOTS; + + wifi->data_tx_completed++; + if (status == 0 || status == 0x83) { + wifi->netdev->stats.tx_packets++; + wifi->netdev->stats.tx_bytes += wifi->data_tx_length[slot]; + } else { + wifi->data_tx_failed++; + wifi->netdev->stats.tx_errors++; + } + wifi->data_tx_busy[slot] = false; + } + } + tail = (tail + 1) % 2048; + completed = true; + } + if (completed) + index_write(wifi, 2, 192, tail); + budget = DATA_TX_SLOTS - 1; + while (wifi->connected && !READ_ONCE(wifi->stopping) && + netif_running(wifi->netdev) && budget--) { + u16 next = (post + 1) % DATA_TX_SLOTS; + u32 buffer = DATA_TX_BUFFER + post * BUFFER_SIZE; + u8 *descriptor = wifi->a.window + 0x540000 + post * 32; + struct sk_buff *skb; + u8 destination[ETH_ALEN]; + u64 metadata = 6ULL << 27; + + if (wifi->data_tx_busy[post] || next == index_read(wifi, 1, 25)) + break; + skb = skb_dequeue(&wifi->data_tx_queue); + if (!skb) + break; + /* This diagnostic ring is only valid for its broadcast destination. + * Do not silently send unicast or IPv6 multicast through that flow. + */ + if (data_tx_broadcast && + (skb_copy_bits(skb, 0, destination, ETH_ALEN) || + !is_broadcast_ether_addr(destination))) { + wifi->netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + continue; + } + if (skb_copy_bits(skb, 0, wifi->a.window + buffer, skb->len)) { + wifi->netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + continue; + } + memset(descriptor, 0, 32); + put_unaligned_le32(3 | (skb->len << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le16(post + 1, descriptor + 12); + /* AirshipDataPacket type 1 supplies this 16-byte TX header inline. + * setTxPolicyId(1), 0x1809ed27c, sets validity bit 60 and ID bit 48. + * Keep normal policy zero by default; the switch permits a bounded + * comparison with the existing, read-back-verified control profile. + */ + if (READ_ONCE(data_tx_control_profile) && + (wifi->tx_defaults_applied & BIT(1))) + metadata |= BIT_ULL(60) | BIT_ULL(48); + /* Native 25G76 AirshipDataPacket::prepare (0x18090faf8) writes + * (argument & 7) to bits 17:15. Guest37's captured DHCP Discover + * has value 4 (0x20000). The same-connection Linux A/B changes + * zero DHCP offers into three offers by setting only this field. + * The opt-out retains the old value for diagnostic reproduction. + */ + if (READ_ONCE(data_tx_native_prepare)) + metadata |= 4ULL << 15; + put_unaligned_le64(metadata, descriptor + 16); + put_unaligned_le64(2000000, descriptor + 24); + wifi->data_tx_length[post] = skb->len; + wifi->data_tx_busy[post] = true; + wifi->data_tx_submitted++; + dev_kfree_skb_any(skb); + post = next; + posted = true; + } + if (posted || completed) { + dma_wmb(); + if (posted) + index_write(wifi, 3, 25, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1030); + } + if (wifi->connected && !READ_ONCE(wifi->stopping) && netif_running(wifi->netdev) && + skb_queue_len(&wifi->data_tx_queue) < DATA_TX_QUEUE_LIMIT) + netif_wake_queue(wifi->netdev); +} + +static void drain(struct centauri_wifi *wifi) +{ + unsigned int ring; + bool posted = false; + + for (ring = 2; ring <= 4; ring++) { + unsigned int budget = ring_size[ring]; + + while (budget--) { + u16 slot = wifi->rx_next[ring], size, status, sequence; + u32 command; + u8 *descriptor = wifi->a.window + TR_BASE + (ring - 1) * RING_SPAN + slot * 16; + u8 *packet = wifi->a.window + rx_buffer[ring] + slot * BUFFER_SIZE; + + if (!(READ_ONCE(descriptor[15]) & 4)) + break; + dma_rmb(); + size = get_unaligned_le16(packet + 0x10); + status = get_unaligned_le16(packet + 0xe); + sequence = get_unaligned_le16(packet + 0x12); + command = get_unaligned_le32(packet + 0x14) & ~BIT(30); + wifi->rx_seen[ring]++; + wifi->rx_next[ring] = (slot + 1) % ring_size[ring]; + if (size < 24 || size > BUFFER_SIZE) + goto refill; + if (ring == 4) { + report_beacon(wifi, packet + 24, size - 24); + } else if (command & BIT(31)) { + if (command == 0x80000030) + wifi->channels_seen++; + if (command == 0x80000005) + wifi->scans_done++; + if (command == 0x80000000 && wifi->connecting) { + wifi->connect_event = true; + wifi->connect_status = status; + } + if (command == 0x80000001) { + wifi->associated = status == 0; + dev_info(&wifi->control->pdev->dev, "Wi-Fi association event: status=%u\n", status); + if (eapol_after_assoc && wifi->associated && wifi->connecting && + !wifi->connect_event) { + int probe_ret = transmit_eapol_start(wifi); + + eapol_after_assoc = false; + dev_info(&wifi->control->pdev->dev, + "association-event EAPOL-Start: rc=%d\n", probe_ret); + } + } + /* ACI event 0x50 carries fixed, non-secret supplicant + * diagnostics: result, phase and handshake timestamps. + * Do not dump the event or any EAPOL/key material. + */ + if (command == 0x80000050 && size >= 64) + dev_info(&wifi->control->pdev->dev, + "Wi-Fi supplicant: result=%u phase=%u times=%u/%u/%u/%u/%u\n", + get_unaligned_le32(packet + 0x18), packet[0x1c], + get_unaligned_le32(packet + 0x24), + get_unaligned_le32(packet + 0x28), + get_unaligned_le32(packet + 0x2c), + get_unaligned_le32(packet + 0x30), + get_unaligned_le32(packet + 0x34)); + if (command == 0x80000003 || command == 0x80000004) + wifi->disconnect_event = true; + if (wifi->connecting && command != 0x80000000 && + command != 0x80000001 && command != 0x80000030 && + command != 0x80000005 && command != 0x80000050) + dev_info_ratelimited(&wifi->control->pdev->dev, + "Wi-Fi event: id=%#x status=%u size=%u\n", + command, status, size); + } else { + wifi->reply_valid = true; + wifi->reply_sequence = sequence; + wifi->reply_status = status; + wifi->reply_command = command; + if (command == 10) { + wifi->country_valid = !status && size >= 27; + if (wifi->country_valid) + memcpy(wifi->country, packet + 24, 3); + } + if (command == 56) { + wifi->tx_profile_valid = !status && size >= 34; + if (wifi->tx_profile_valid) + memcpy(wifi->tx_profile, packet + 24, sizeof(wifi->tx_profile)); + } + if (command == 20 && !status) { + /* getInterfaceStatus skips its 32-byte fixed response. */ + unsigned int pos = 32; + + wifi->link_status_size = size; + while (pos + 4 <= size) { + u16 tag = get_unaligned_le16(packet + pos); + u16 length = get_unaligned_le16(packet + pos + 2); + u8 *value = packet + pos + 4; + unsigned int i; + + if (length > size - pos - 4) + break; + if (wifi->link_tlv_count < ARRAY_SIZE(wifi->link_tags)) { + wifi->link_tags[wifi->link_tlv_count] = tag; + wifi->link_lengths[wifi->link_tlv_count++] = length; + } + /* ACIDebugAdapter::dumpMloStatus, TLVs 377/378. + * Keep only link count/channel/role. Do not retain MACs. + */ + if (tag == 377 && length >= 64 && value[0] <= 3) { + wifi->link_count = value[0]; + wifi->link_status_valid = true; + for (i = 0; i < value[0]; i++) { + u16 chanspec = get_unaligned_le16(value + 26 + 17 * i); + + wifi->link_prime[i] = value[25 + 17 * i]; + wifi->link_channel[i] = chanspec & 0xff; + wifi->link_band[i] = (chanspec >> 14) & 3; + } + } + if (tag == 378 && length >= 9) { + wifi->emlsr_valid = true; + wifi->emlsr_mode = value[0]; + } + pos += 4 + length; + } + } + if (command == 0xb0001) { + wifi->stats_reply_size = size; + wifi->stats_tag = size >= 28 ? get_unaligned_le16(packet + 24) : 0; + wifi->stats_length = size >= 28 ? get_unaligned_le16(packet + 26) : 0; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf928 && + (wifi->stats_length == 752 || wifi->stats_length == 888) && + size >= 28 + wifi->stats_length) { + /* Native firmware returns the older 752-byte counter record. */ + memset(wifi->station_stats, 0, sizeof(wifi->station_stats)); + memcpy(wifi->station_stats, packet + 28, wifi->stats_length); + wifi->station_stats_valid = true; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf921 && + wifi->stats_length == sizeof(wifi->rx_crypto_stats) && + size >= 28 + sizeof(wifi->rx_crypto_stats)) { + memcpy(wifi->rx_crypto_stats, packet + 28, sizeof(wifi->rx_crypto_stats)); + wifi->rx_crypto_stats_valid = true; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf91b && + wifi->stats_length == sizeof(wifi->dot11_mib_stats) && + size >= 28 + sizeof(wifi->dot11_mib_stats)) { + memcpy(wifi->dot11_mib_stats, packet + 28, sizeof(wifi->dot11_mib_stats)); + wifi->dot11_mib_stats_valid = true; + } + if (command == 0xb0001 && !status && wifi->stats_tag == 0xf920 && + wifi->stats_length == sizeof(wifi->tx_power_report) && + size >= 28 + sizeof(wifi->tx_power_report)) { + memcpy(wifi->tx_power_report, packet + 28, sizeof(wifi->tx_power_report)); + wifi->tx_power_report_valid = true; + } + if (command == 0xf0025) { + wifi->pmk_seen = !status && size >= 60; + wifi->pmk_matches = wifi->pmk_seen && + get_unaligned_le16(packet + 26) == 32 && + !memcmp(packet + 28, wifi->expected_pmk, 32); + memzero_explicit(packet + 24, size - 24); + } + } + refill: + post_receive(wifi, ring, wifi->rx_head[ring]); + wifi->rx_head[ring] = (wifi->rx_head[ring] + 1) % ring_size[ring]; + dma_wmb(); + index_write(wifi, 3, ring, wifi->rx_head[ring]); + posted = true; + } + } + completions(wifi); + if (posted) + writel(1, wifi->a.bar + 0x1004); + drain_data_rx(wifi); + drain_data_tx(wifi); +} + +static int command(struct centauri_wifi *wifi, u32 id, const void *payload, size_t size) +{ + u16 slot = wifi->command_head, sequence = ++wifi->sequence; + u8 *packet, *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(1500); + + if (size + 24 > BUFFER_SIZE) + return -EMSGSIZE; + if ((slot + 1) % ring_size[1] == index_read(wifi, 1, 1)) + return -ENOSPC; + packet = wifi->a.window + TX_BUFFER + slot * BUFFER_SIZE; + memset(packet, 0, 24 + size); + put_unaligned_le16(24 + size, packet + 0x10); + put_unaligned_le16(sequence, packet + 0x12); + put_unaligned_le32(id, packet + 0x14); + if (size) + memcpy(packet + 24, payload, size); + descriptor = wifi->a.window + TR_BASE + slot * 16; + put_unaligned_le32(1 | ((24 + size) << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + TX_BUFFER + slot * BUFFER_SIZE, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + dma_wmb(); + wifi->command_head = (slot + 1) % ring_size[1]; + index_write(wifi, 3, 1, wifi->command_head); + wifi->reply_valid = false; + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + do { + drain(wifi); + if (readl(wifi->a.bar + 8) != 2) + return -EIO; + /* Bit 30 is more-data-pending, not part of the command identity. + * Scan/connect replies set it before their later completion event. + */ + if (wifi->reply_valid && wifi->reply_sequence == sequence && + (wifi->reply_command & ~BIT(30)) == id) { + if (wifi->reply_status) + dev_info(&wifi->control->pdev->dev, + "Wi-Fi command %#x rejected: firmware_status=%u\n", + id, wifi->reply_status); + return wifi->reply_status ? -EREMOTEIO : 0; + } + usleep_range(500, 1000); + } while (time_before(jiffies, deadline)); + dev_err(&wifi->control->pdev->dev, "Wi-Fi command %#x timed out (rx=%u)\n", id, wifi->rx_seen[2]); + return -ETIMEDOUT; +} + +static int radio_start(struct centauri_wifi *wifi) +{ + u8 start[5 + 4 + 3 * ETH_ALEN] = { 0, 0x0b, 0, 0, 0 }; + size_t start_size = 5; + unsigned int ring, slot; + int ret; + + /* Stock 25G76 startInterface (0x1808e26ec) supplies TLV 385, + * length 18. Firmware 0x5736c0 otherwise generates three unrelated + * random addresses. This diagnostic keeps the sole active link equal + * to the netdev identity; it does not enable multi-link association. + */ + if (station_link_addresses) { + put_unaligned_le16(385, start + 5); + put_unaligned_le16(3 * ETH_ALEN, start + 7); + for (slot = 0; slot < 3; slot++) + ether_addr_copy(start + 9 + slot * ETH_ALEN, wifi->netdev->dev_addr); + start_size = sizeof(start); + } + for (ring = 1; ring <= 4; ring++) { + ret = open_pair(wifi, ring); + if (ret) + return ret; + } + for (ring = 2; ring <= 4; ring++) { + for (slot = 0; slot < ring_size[ring] - 1; slot++) + post_receive(wifi, ring, slot); + wifi->rx_head[ring] = ring_size[ring] - 1; + dma_wmb(); + index_write(wifi, 3, ring, ring_size[ring] - 1); + } + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + msleep(400); + ret = command(wifi, 0, NULL, 0); + if (!ret) + ret = command(wifi, 1, wifi->netdev->dev_addr, ETH_ALEN); + if (!ret) + ret = command(wifi, 2, start, start_size); + if (ret) + return ret; + msleep(1000); + wifi->configured = true; + dev_info(&wifi->control->pdev->dev, "Wi-Fi command rings ready; interface started\n"); + return 0; +} + +static void scan_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = container_of(work, struct centauri_wifi, scan_work); + struct cfg80211_scan_info info = {}; + struct cfg80211_scan_request *request; + u8 scan[14] = { 1, 0, 0, 0x80, 0, 2, 0, 0x10, 110, 0, 110, 0, 0, 0 }; + unsigned int chunk; + int ret = 0; + + mutex_lock(&wifi->lock); + for (chunk = 0; chunk < 3 && !ret; chunk++) { + u32 channels = wifi->channels_seen, done = wifi->scans_done; + bool stopped = false; + unsigned long deadline; + + scan[0] = chunk + 1; + ret = command(wifi, 65551, scan, sizeof(scan)); + if (ret) + break; + deadline = jiffies + msecs_to_jiffies(8000); + do { + drain(wifi); + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + break; + } + if (!stopped && wifi->channels_seen - channels >= 8) { + stopped = true; + ret = command(wifi, 65537, NULL, 0); + if (ret) + break; + } + if (stopped && wifi->scans_done > done) + break; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + if (!ret && wifi->scans_done == done) + ret = -ETIMEDOUT; + } + info.aborted = !!ret; + request = wifi->scan_request; + wifi->scan_request = NULL; + mutex_unlock(&wifi->lock); + dev_info(&wifi->control->pdev->dev, "Wi-Fi scan finished: rc=%d channels=%u beacons=%u\n", + ret, wifi->channels_seen, wifi->beacons_seen); + if (request) + cfg80211_scan_done(request, &info); +} + +static int wifi_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (!wifi->configured) + ret = -ENETDOWN; + else if (wifi->scan_request || wifi->connecting) + ret = -EBUSY; + else { + wifi->scan_request = request; + schedule_work(&wifi->scan_work); + } + mutex_unlock(&wifi->lock); + return ret; +} + +static int rsn_caps_offset(const u8 *body, size_t size) +{ + size_t offset = 6; + u16 count; + unsigned int i; + + for (i = 0; i < 2; i++) { + if (offset + 2 > size) + return -EINVAL; + count = get_unaligned_le16(body + offset); + offset += 2 + 4 * count; + if (offset > size) + return -EINVAL; + } + return offset + 2 <= size ? offset : -EINVAL; +} + +static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, u16 length) +{ + put_unaligned_le16(tag, payload + *size); + put_unaligned_le16(length, payload + *size + 2); + memcpy(payload + *size + 4, data, length); + *size += 4 + length; +} + +static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, + struct cfg80211_connect_params *params) +{ + /* Stock 25G76 ACIConnectionAdapter::connectAp (0x1808e94cc) emits a + * 12-byte TLV 376. The newer SDK 27 adapter uses 14 bytes. Preserve the + * unspecified-link masks; all-zero is not the non-MLO default. + */ + static const u8 mld_defaults[12] = { + 1, 0, 1, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 3, + }; + /* Firmware connection-context defaults (aciw 0x511ee8). TLV 0xf913 + * replaces all five values. Sending zeros arms Supp_AddTimer with a + * zero duration and aborts WPA before message 2 can be transmitted. + */ + static const __le32 connection_timeouts[] = { + cpu_to_le32(260), cpu_to_le32(6000), cpu_to_le32(2000), + cpu_to_le32(15000), cpu_to_le32(300), + }; + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + struct cfg80211_bss *bss; + const struct cfg80211_bss_ies *ies; + const u8 *rsn, *ap_rsn; + u8 rsn_body[255], ssid[35], key[66], zero[20] = {}, pmf; + u8 *payload = wifi->connect_payload; + size_t size = 130, rsn_size; + u16 caps = 0; + int ret = 0, offset; + + if (!params->ssid_len || params->ssid_len > 32 || !params->crypto.psk || + params->crypto.n_akm_suites != 1 || params->crypto.akm_suites[0] != WLAN_AKM_SUITE_PSK || + params->crypto.n_ciphers_pairwise != 1 || params->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || + params->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || params->ie_len > sizeof(wifi->request_ie)) + return -EOPNOTSUPP; + rsn = cfg80211_find_ie(WLAN_EID_RSN, params->ie, params->ie_len); + if (!rsn || rsn_caps_offset(rsn + 2, rsn[1]) < 0) + return -EINVAL; + mutex_lock(&wifi->lock); + if (!wifi->configured || wifi->connecting || wifi->connected || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + if (!wifi->passphrase_size) { + ret = -ENOKEY; + goto out; + } + bss = cfg80211_get_bss(wiphy, params->channel, params->bssid, + params->ssid, params->ssid_len, IEEE80211_BSS_TYPE_ESS, IEEE80211_PRIVACY_ON); + if (!bss) { + ret = -ENOENT; + goto out; + } + /* Match the measured successful J813 association shape. The firmware + * expects the AP RSN capability bits even with the legal single-AKM STA IE. + */ + rcu_read_lock(); + ies = rcu_dereference(bss->ies); + ap_rsn = ies ? cfg80211_find_ie(WLAN_EID_RSN, ies->data, ies->len) : NULL; + if (ap_rsn) { + offset = rsn_caps_offset(ap_rsn + 2, ap_rsn[1]); + if (offset >= 0) + caps = get_unaligned_le16(ap_rsn + 2 + offset); + } + rcu_read_unlock(); + rsn_size = rsn[1]; + memcpy(rsn_body, rsn + 2, rsn_size); + offset = rsn_caps_offset(rsn_body, rsn_size); + put_unaligned_le16(caps, rsn_body + offset); + pmf = !!(caps & BIT(7)); + memset(payload, 0, sizeof(wifi->connect_payload)); + payload[0] = 1; + memcpy(payload + 1, bss->bssid, ETH_ALEN); + memcpy(wifi->bssid, bss->bssid, ETH_ALEN); + payload[7] = ieee80211_frequency_to_channel(bss->channel->center_freq); + /* AppleChanSpecToACI maps chanspec 0xc000 (5 GHz) to ACI 1; + * 0x4000 (6 GHz) maps to ACI 2. Earlier RE notes swapped these. + */ + payload[8] = bss->channel->band == NL80211_BAND_2GHZ ? 0 : 1; + payload[0x47] = 4; + /* ACIConnectionAdapter::mapRSNCipherSuiteToACIWsec maps CCMP to 4. + * The RSN IE does not replace this fixed cipher field. + */ + put_unaligned_le16(4, payload + 0x48); + put_unaligned_le16(32, payload + 0x4a); + memcpy(payload + 0x4c, params->crypto.psk, 32); + if (verify_pmk) + memcpy(wifi->expected_pmk, params->crypto.psk, 32); + append_tlv(payload, &size, 304, zero, 1); + append_tlv(payload, &size, 63763, connection_timeouts, sizeof(connection_timeouts)); + ssid[0] = 1; + ssid[1] = 4; + ssid[2] = params->ssid_len; + memcpy(ssid + 3, params->ssid, params->ssid_len); + append_tlv(payload, &size, 342, ssid, 3 + params->ssid_len); + append_tlv(payload, &size, WLAN_EID_RSN, rsn_body, rsn_size); + /* The fixed field carries the PMK; TLV 313 carries the passphrase. */ + put_unaligned_le16(wifi->passphrase_size, key); + memcpy(key + 2, wifi->passphrase, wifi->passphrase_size); + append_tlv(payload, &size, 313, key, 2 + wifi->passphrase_size); + memzero_explicit(key, sizeof(key)); + append_tlv(payload, &size, 316, &pmf, 1); + append_tlv(payload, &size, 376, mld_defaults, sizeof(mld_defaults)); + wifi->connect_size = size; + memcpy(wifi->request_ie, params->ie, params->ie_len); + wifi->request_ie_size = params->ie_len; + wifi->connecting = true; + wifi->connect_status = 0; + wifi->associated = wifi->pmk_seen = wifi->pmk_matches = false; + wifi->connect_event = wifi->disconnect_event = false; + cfg80211_put_bss(wiphy, bss); + schedule_work(&wifi->connect_work); +out: + mutex_unlock(&wifi->lock); + return ret; +} + +static void connect_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = container_of(work, struct centauri_wifi, connect_work); + unsigned long deadline; + u16 slot, status, command_status; + bool checked_pmk = false, probed_eapol = false, queried_link = false; + int ret; + + mutex_lock(&wifi->lock); + slot = wifi->command_head; + ret = command(wifi, 131072, wifi->connect_payload, wifi->connect_size); + command_status = wifi->reply_status; + memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); + deadline = jiffies + msecs_to_jiffies(15000); + while (!ret && !wifi->connect_event && time_before(jiffies, deadline)) { + drain(wifi); + if (link_diagnostics && wifi->associated && !queried_link) { + unsigned int i; + int link_ret; + + queried_link = true; + link_ret = query_link_status(wifi); + dev_info(&wifi->control->pdev->dev, + "connection link status: rc=%d complete=%d valid=%d count=%u emlsr=%d/%u\n", + link_ret, wifi->connect_event, wifi->link_status_valid, wifi->link_count, + wifi->emlsr_valid, wifi->emlsr_mode); + for (i = 0; i < wifi->link_tlv_count; i++) + dev_info(&wifi->control->pdev->dev, "connection link TLV: tag=%u length=%u\n", + wifi->link_tags[i], wifi->link_lengths[i]); + if (wifi->link_status_valid) + for (i = 0; i < wifi->link_count; i++) + dev_info(&wifi->control->pdev->dev, + "connection link %u: prime=%u channel=%u band=%u\n", + i, wifi->link_prime[i], wifi->link_channel[i], wifi->link_band[i]); + } + if (eapol_after_assoc && wifi->associated && !probed_eapol) { + unsigned int i; + int probe_ret; + + probed_eapol = true; + probe_ret = query_station_stats(wifi); + dev_info(&wifi->control->pdev->dev, + "connection counters: complete=%d rc=%d valid=%d length=%u\n", + wifi->connect_event, probe_ret, wifi->station_stats_valid, wifi->stats_length); + if (!probe_ret && wifi->station_stats_valid) + for (i = 0; i < wifi->stats_length; i += 4) { + u32 value = get_unaligned_le32(wifi->station_stats + i); + + if (value) + dev_info(&wifi->control->pdev->dev, + "station counter %03x=%u\n", i, value); + } + } + if (verify_pmk && wifi->associated && !checked_pmk) { + int key_ret = command(wifi, 0xf0025, wifi->bssid, ETH_ALEN); + + checked_pmk = true; + dev_info(&wifi->control->pdev->dev, + "Wi-Fi firmware key check: rc=%d available=%d matches=%d\n", + key_ret, wifi->pmk_seen, wifi->pmk_matches); + } + if (readl(wifi->a.bar + 8) != 2) + ret = -EIO; + usleep_range(1000, 2000); + } + if (!ret && !wifi->connect_event) + ret = -ETIMEDOUT; + status = !ret && wifi->connect_status == 0 ? WLAN_STATUS_SUCCESS : WLAN_STATUS_UNSPECIFIED_FAILURE; + wifi->connecting = false; + wifi->connected = status == WLAN_STATUS_SUCCESS; + memzero_explicit(wifi->expected_pmk, sizeof(wifi->expected_pmk)); + /* The command and connection completion have consumed the PMK buffer. */ + if (slot < ring_size[1]) + memzero_explicit(wifi->a.window + TX_BUFFER + slot * BUFFER_SIZE, BUFFER_SIZE); + mutex_unlock(&wifi->lock); + dev_info(&wifi->control->pdev->dev, + "Wi-Fi authentication result: rc=%d command_status=%u event_status=%u complete_event=%d\n", + ret, command_status, wifi->connect_status, wifi->connect_event); + cfg80211_connect_result(wifi->netdev, wifi->bssid, + wifi->request_ie, wifi->request_ie_size, NULL, 0, status, GFP_KERNEL); + if (status == WLAN_STATUS_SUCCESS && !READ_ONCE(wifi->stopping)) { + netif_carrier_on(wifi->netdev); + if (wifi->data_probe_done & BIT(5)) + netif_wake_queue(wifi->netdev); + schedule_delayed_work(&wifi->link_work, msecs_to_jiffies(20)); + } +} + +static void link_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = container_of(to_delayed_work(work), struct centauri_wifi, link_work); + bool disconnected; + + mutex_lock(&wifi->lock); + drain(wifi); + disconnected = wifi->disconnect_event || READ_ONCE(wifi->stopping) || + readl(wifi->a.bar + 8) != 2; + if (disconnected) + wifi->connected = false; + mutex_unlock(&wifi->lock); + if (disconnected) { + netif_stop_queue(wifi->netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(wifi->netdev); + cfg80211_disconnected(wifi->netdev, WLAN_REASON_UNSPECIFIED, NULL, 0, false, GFP_KERNEL); + } else { + schedule_delayed_work(&wifi->link_work, msecs_to_jiffies(20)); + } +} + +static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 payload[16] = {}; + bool connected; + int ret; + + mutex_lock(&wifi->lock); + if (wifi->connecting) { + mutex_unlock(&wifi->lock); + return -EBUSY; + } + put_unaligned_le16(reason, payload); + put_unaligned_le16(328, payload + 8); + put_unaligned_le16(4, payload + 10); + ret = command(wifi, 131073, payload, sizeof(payload)); + connected = wifi->connected; + wifi->connected = false; + mutex_unlock(&wifi->lock); + cancel_delayed_work_sync(&wifi->link_work); + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(netdev); + if (connected) + cfg80211_disconnected(netdev, reason, NULL, 0, true, GFP_KERNEL); + return ret == -EREMOTEIO && wifi->reply_status == 326 ? 0 : ret; +} + +static int wifi_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, + bool enabled, int timeout) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 mode = enabled ? 0 : 3; + int ret; + + /* ACIIO80211PowerAdapter::setPowerSaveMode, command 7. */ + mutex_lock(&wifi->lock); + ret = wifi->configured ? command(wifi, 7, &mode, sizeof(mode)) : -ENETDOWN; + mutex_unlock(&wifi->lock); + return ret; +} + +static void wifi_regulatory(struct wiphy *wiphy, struct regulatory_request *request) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 country[3] = { request->alpha2[0], request->alpha2[1], 1 }; + int ret; + + /* ACIRegulatoryAdapter::setCountryCode, command 9. Leave world-domain + * handling to cfg80211; do not invent a device country or bypass SAR. + */ + if (country[0] < 'A' || country[0] > 'Z' || country[1] < 'A' || country[1] > 'Z') + return; + mutex_lock(&wifi->lock); + if (wifi->configured) { + ret = command(wifi, 9, country, sizeof(country)); + dev_info(&wifi->control->pdev->dev, + "Wi-Fi country %c%c: rc=%d\n", country[0], country[1], ret); + } + mutex_unlock(&wifi->lock); +} + +static const struct cfg80211_ops wifi_ops = { + .scan = wifi_scan, + .connect = wifi_connect, + .disconnect = wifi_disconnect, + .set_power_mgmt = wifi_power_mgmt, +}; + +static int net_open(struct net_device *netdev) +{ + netif_carrier_off(netdev); + netif_stop_queue(netdev); + return 0; +} + +static int net_stop(struct net_device *netdev) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)netdev_priv(netdev); + + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + return 0; +} + +static netdev_tx_t net_transmit(struct sk_buff *skb, struct net_device *netdev) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)netdev_priv(netdev); + + if (READ_ONCE(wifi->stopping) || !READ_ONCE(wifi->connected) || + skb->len < ETH_HLEN || skb->len > BUFFER_SIZE) { + netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) { + netif_stop_queue(netdev); + return NETDEV_TX_BUSY; + } + skb_queue_tail(&wifi->data_tx_queue, skb); + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) + netif_stop_queue(netdev); + mod_delayed_work(system_wq, &wifi->link_work, 0); + return NETDEV_TX_OK; +} + +static const struct net_device_ops net_ops = { + .ndo_open = net_open, + .ndo_stop = net_stop, + .ndo_start_xmit = net_transmit, + .ndo_validate_addr = eth_validate_addr, +}; + +static ssize_t passphrase_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + u8 value[64]; + size_t length = count; + int ret = 0; + + if (count < 8 || count > sizeof(value)) + return -EINVAL; + if (copy_from_user(value, input, count)) { + ret = -EFAULT; + goto out; + } + if (value[length - 1] == '\n') + length--; + if (length < 8 || length > 63) { + ret = -EINVAL; + goto out; + } + mutex_lock(&wifi->lock); + if (wifi->connecting) { + ret = -EBUSY; + } else { + memzero_explicit(wifi->passphrase, sizeof(wifi->passphrase)); + memcpy(wifi->passphrase, value, length); + wifi->passphrase_size = length; + } + mutex_unlock(&wifi->lock); +out: + memzero_explicit(value, sizeof(value)); + return ret ?: count; +} + +static const struct file_operations passphrase_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = passphrase_write, +}; + +/* Airship direct_memregion::map (0x1802864e0) sends opcode 8, a region ID, + * byte size and a 64-bit DVA in a 24-byte header. Alpha's region-debug is ID 1. + * Firmware 0x175340 passes those fields to DramPhyService_handle_buffer; + * InitDramLoggerMgr writes a 0x158-byte control header to the supplied buffer. + * This is distinct from the fw_debug transfer ring. Never print log payloads: + * firmware diagnostics may contain private connection material. + */ +static ssize_t fwlog_start_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + u8 message[24] = { 8, 0, 1, 0 }; + unsigned int action; + unsigned long deadline; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &action); + if (ret || action != 1) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + mutex_lock(&wifi->fwlog_lock); + if (!wifi->configured || wifi->connecting || wifi->connected || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + if (wifi->fwlog_attempted) { + ret = -EALREADY; + goto out; + } + ret = centauri_alpha_log_buffer(wifi->control, &wifi->fwlog_buffer, + &wifi->fwlog_dma, &wifi->fwlog_size); + if (ret) + goto out; + put_unaligned_le32(wifi->fwlog_size, message + 4); + put_unaligned_le64(wifi->fwlog_dma, message + 8); + wifi->fwlog_attempted = true; + ret = alpha_message(wifi, message, sizeof(message)); + if (ret) + goto report; + deadline = jiffies + msecs_to_jiffies(2000); + do { + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + goto report; + } + dma_rmb(); + if (get_unaligned_le16(wifi->fwlog_buffer) == 0x201) { + wifi->fwlog_ready = true; + goto report; + } + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + ret = -ETIMEDOUT; +report: + dev_info(&wifi->control->pdev->dev, + "Wi-Fi DRAM logger: rc=%d initialized=%u stage=%u\n", + ret, wifi->fwlog_ready, readl(wifi->a.bar + 8)); +out: + mutex_unlock(&wifi->fwlog_lock); + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations fwlog_start_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = fwlog_start_write, +}; + +static ssize_t fwlog_status_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + char text[320]; + u8 header[0xb4] = { 0 }; + int length; + + mutex_lock(&wifi->fwlog_lock); + if (wifi->fwlog_buffer) { + dma_rmb(); + memcpy(header, wifi->fwlog_buffer, sizeof(header)); + } + length = scnprintf(text, sizeof(text), + "attempted=%u initialized=%u bytes=%zu version=%#x blocks=%u block_size=%u mask=%#x data_offset=%u consumer=%u producer=%u header14=%u header18=%u header1c=%u connecting=%u scanning=%u\n", + wifi->fwlog_attempted, wifi->fwlog_ready, wifi->fwlog_size, + get_unaligned_le16(header), get_unaligned_le16(header + 2), + get_unaligned_le32(header + 4), get_unaligned_le32(header + 8), + get_unaligned_le32(header + 12), get_unaligned_le32(header + 16), + get_unaligned_le32(header + 0xb0), + get_unaligned_le32(header + 20), get_unaligned_le32(header + 24), + get_unaligned_le32(header + 28), + !!READ_ONCE(wifi->connecting), !!READ_ONCE(wifi->scan_request)); + mutex_unlock(&wifi->fwlog_lock); + return simple_read_from_buffer(output, count, position, text, length); +} + +static const struct file_operations fwlog_status_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = fwlog_status_read, + .llseek = default_llseek, +}; + +/* The exact firmware's 0x144cfc compares header +0xb0 (producer) against + * +0x10 (host consumer). InitDramLoggerMgr registers a GDMA read of +0x10. + * Only acknowledge records after a private snapshot has been verified. + * Leave three entries reserved for the firmware's three DMA descriptors + * until completion/record sequencing is independently decoded. + */ +static ssize_t fwlog_ack_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + u8 *header; + u32 requested, consumer, producer, pending; + u16 blocks; + int ret; + + ret = kstrtou32_from_user(input, count, 0, &requested); + if (ret) + return ret; + mutex_lock(&wifi->fwlog_lock); + if (!wifi->fwlog_ready || readl(wifi->a.bar + 8) != 2) { + ret = -ENETDOWN; + goto out; + } + header = wifi->fwlog_buffer; + dma_rmb(); + blocks = get_unaligned_le16(header + 2); + consumer = get_unaligned_le32(header + 0x10); + producer = get_unaligned_le32(header + 0xb0); + pending = producer - consumer; + if (get_unaligned_le16(header) != 0x201 || blocks < 8 || + pending > blocks || pending < 3 || requested - consumer > pending - 3) { + ret = -EINVAL; + goto out; + } + WRITE_ONCE(*(__le32 *)(header + 0x10), cpu_to_le32(requested)); + dma_wmb(); + ret = 0; +out: + mutex_unlock(&wifi->fwlog_lock); + return ret ?: count; +} + +static const struct file_operations fwlog_ack_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = fwlog_ack_write, +}; + +/* Root-only binary diagnostics. Save directly to a protected artifact, never + * the console or a text log. The device may update blocks during a read. + */ +static ssize_t fwlog_buffer_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + ssize_t ret; + + mutex_lock(&wifi->fwlog_lock); + if (!wifi->fwlog_ready) { + ret = -ENODATA; + } else { + dma_rmb(); + ret = simple_read_from_buffer(output, count, position, + wifi->fwlog_buffer, wifi->fwlog_size); + } + mutex_unlock(&wifi->fwlog_lock); + return ret; +} + +static const struct file_operations fwlog_buffer_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = fwlog_buffer_read, + .llseek = default_llseek, +}; + +/* The hardware data-ring callback requires the ten-byte WLAN clientdata + * extension after the 44-byte ACIPC header. Without it UMAC asserts at + * 0x11760e. Keep these empty-ring probes explicit until packet DMA is proven. + * The reserved regions below do not overlap control rings or scan buffers. + */ +static ssize_t data_probe_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + u8 message[0x34 + 10] = { 0 }; + unsigned int id, vector, offset, size, bit, doorbell, irq; + bool transfer = false; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &id); + if (ret) + return ret; + if (id == 76) { + vector = 192; + offset = 0x200000; + size = 2048; + bit = 0; + doorbell = 18; + irq = 3; + } else if (id == 14) { + vector = 206; + offset = 0x220000; + size = 256; + bit = 1; + doorbell = 11; + irq = 8; + } else if (id == 15) { + vector = 207; + offset = 0x240000; + size = 128; + bit = 2; + doorbell = 11; + irq = 8; + } else if (id == 24 || id == 25 || id == 12) { + transfer = true; + vector = id; + offset = id == 12 ? 0x520000 : id == 24 ? 0x500000 : 0x540000; + size = 128; + bit = id == 12 ? 4 : id == 24 ? 3 : 5; + doorbell = id == 12 ? 11 : 18; + irq = id == 12 ? 8 : 3; + } else { + return -EINVAL; + } + mutex_lock(&wifi->lock); + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + goto out; + } + if (wifi->connecting || wifi->scan_request || + (wifi->data_probe_done & BIT(bit))) { + ret = -EBUSY; + goto out; + } + wifi->data_probe_done |= BIT(bit); + memset(wifi->a.window + offset, 0, size * 32); + index_write(wifi, transfer ? 1 : 0, vector, 0); + index_write(wifi, transfer ? 3 : 2, vector, 0); + message[0] = transfer ? 1 : 2; + message[2] = transfer && id == 12 ? 0 : 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le16(vector, message + 6); + put_unaligned_le64(wifi->a.window_dma + offset, message + 8); + if (transfer) { + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(size, message + 0x18); + put_unaligned_le16(id == 12 ? 12 : 76, message + 0x1a); + put_unaligned_le16(doorbell, message + 0x1c); + put_unaligned_le16(id == 12 ? 0x220 : 0, message + 0x1e); + put_unaligned_le32(16, message + 0x20); + put_unaligned_le16(irq, message + 0x24); + } else { + put_unaligned_le16(size, message + 0x10); + put_unaligned_le16(id == 15 ? 12 : 0xffff, message + 0x12); + put_unaligned_le16(doorbell, message + 0x14); + put_unaligned_le32(1, message + 0x18); + put_unaligned_le16(irq, message + 0x1c); + } + /* AirshipDataTxTransferRing::initWithOptions maps the host MCBC + * parameters to version 0, hardware interface 0, TID 6, AC 3, and + * six broadcast destination bytes. The default probe is unchanged. + */ + if (id == 25 && data_tx_broadcast) { + message[0x34 + 2] = 6; + message[0x34 + 3] = 3; + eth_broadcast_addr(message + 0x34 + 4); + } + ret = alpha_message(wifi, message, (transfer ? 0x34 : 0x2c) + 10); + msleep(1000); + dev_info(&wifi->control->pdev->dev, + "data %s probe id=%u index=%u extension=10 rc=%d stage=%u head=%u tail=%u\n", + transfer ? "TR" : "CR", id, vector, ret, readl(wifi->a.bar + 8), + index_read(wifi, transfer ? 3 : 0, vector), + index_read(wifi, transfer ? 1 : 2, vector)); + if (!ret && readl(wifi->a.bar + 8) != 2) + ret = -EIO; + if (!ret && transfer && id == 12) { + unsigned int slot; + + /* Give the receive engine bounded, individually identifiable buffers. + * No recycling or network delivery until the completion ABI is proven. + */ + for (slot = 0; slot < size - 1; slot++) { + u8 *descriptor = wifi->a.window + offset + slot * 16; + u32 buffer = DATA_RX_BUFFER + slot * 4096; + + memset(wifi->a.window + buffer, 0, 4096); + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + } + dma_wmb(); + index_write(wifi, 3, vector, size - 1); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + msleep(100); + dev_info(&wifi->control->pdev->dev, + "data RX buffers posted=%u stage=%u tr_tail=%u cr_head=%u\n", + size - 1, readl(wifi->a.bar + 8), index_read(wifi, 1, 12), + index_read(wifi, 0, 207)); + } +out: + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static int transmit_eapol_start(struct centauri_wifi *wifi) +{ + u16 head = index_read(wifi, 3, 24); + u8 *descriptor = wifi->a.window + 0x500000 + head * 32; + u8 *packet = wifi->a.window + 0xf00000 + head * 64; + u64 metadata = 0; + unsigned int attempt; + + if (readl(wifi->a.bar + 8) != 2 || !(wifi->data_probe_done & BIT(3))) + return -ENETDOWN; + /* Eight independent buffers bound the probe without ring reuse. */ + if (head >= 8 || index_read(wifi, 1, 24) != head) + return -EBUSY; + if (eapol_unicast && (!wifi->associated || !is_valid_ether_addr(wifi->bssid))) + return -ENOTCONN; + /* A standards-defined EAPOL-Start, with no credential/key material. + * Descriptor flags 3 mean a DMA buffer plus the 16-byte WLAN footer. + * Interface zero and default TX policy have a zero footer. + */ + memset(packet, 0, 64); + if (eapol_unicast) + memcpy(packet, wifi->bssid, ETH_ALEN); + else + memcpy(packet, "\x01\x80\xc2\x00\x00\x03", ETH_ALEN); + memcpy(packet + ETH_ALEN, wifi->netdev->dev_addr, ETH_ALEN); + put_unaligned_be16(ETH_P_PAE, packet + 12); + packet[14] = 2; + packet[15] = 1; + memset(descriptor, 0, 32); + put_unaligned_le32(3 | (18 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + 0xf00000 + head * 64, descriptor + 4); + put_unaligned_le16(head + 1, descriptor + 12); + if (eapol_control_profile) + metadata |= BIT_ULL(60) | BIT_ULL(48); + if (eapol_relative_lifetime) { + /* AirshipDataPacket::setRelativeExpirationTime uses mode 6 in + * bits 29:27 and microseconds in the second metadata word. + */ + metadata |= 6ULL << 27; + put_unaligned_le64(2000000, descriptor + 24); + } + put_unaligned_le64(metadata, descriptor + 16); + dma_wmb(); + index_write(wifi, 3, 24, head + 1); + dma_wmb(); + writel(1, wifi->a.bar + 0x1030); + for (attempt = 0; attempt < 20; attempt++) { + if (index_read(wifi, 1, 24) == head + 1 || readl(wifi->a.bar + 8) != 2) + break; + usleep_range(5000, 6000); + } + dev_info(&wifi->control->pdev->dev, + "EAPOL-Start DMA probe: slot=%u associated=%d unicast=%d relative_expiry=%d stage=%u tr_tail=%u cr_head=%u\n", + head, wifi->associated, eapol_unicast, eapol_relative_lifetime, readl(wifi->a.bar + 8), + index_read(wifi, 1, 24), index_read(wifi, 0, 192)); + return readl(wifi->a.bar + 8) == 2 ? 0 : -EIO; +} + +static ssize_t eapol_start_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + bool enabled; + int ret; + + ret = kstrtobool_from_user(input, count, &enabled); + if (ret || !enabled) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + if (wifi->connecting || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + ret = transmit_eapol_start(wifi); +out: + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations eapol_start_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = eapol_start_write, +}; + +static const struct file_operations data_probe_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = data_probe_write, +}; + +static ssize_t data_status_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + char status[1536]; + unsigned int i, length; + + mutex_lock(&wifi->lock); + dma_rmb(); + length = scnprintf(status, sizeof(status), + "stage=%u tx_cr=%u tx_tr=%u/%u rx_cr=%u rx_tr=%u/%u received=%u reposted=%u bad=%u samples=%u normal_tx_tr=%u/%u submitted=%u completed=%u failed=%u queued=%u coalesced=%u tx_bad=%u\n", + readl(wifi->a.bar + 8), index_read(wifi, 0, 192), + index_read(wifi, 1, 24), index_read(wifi, 3, 24), + index_read(wifi, 0, 207), index_read(wifi, 1, 12), + index_read(wifi, 3, 12), wifi->data_rx_received, + wifi->data_rx_reposted, wifi->data_rx_bad, + le32_to_cpu(wifi->rx_snapshot.count), + index_read(wifi, 1, 25), index_read(wifi, 3, 25), + wifi->data_tx_submitted, wifi->data_tx_completed, + wifi->data_tx_failed, skb_queue_len(&wifi->data_tx_queue), + wifi->data_tx_coalesced, wifi->data_tx_bad); + for (i = 0; i < 8; i++) { + u8 *entry = wifi->a.window + 0x240000 + i * 32; + + length += scnprintf(status + length, sizeof(status) - length, + "rx_completion[%u]=%08x %08x %08x %08x %08x %08x %08x %08x\n", + i, get_unaligned_le32(entry), get_unaligned_le32(entry + 4), + get_unaligned_le32(entry + 8), get_unaligned_le32(entry + 12), + get_unaligned_le32(entry + 16), get_unaligned_le32(entry + 20), + get_unaligned_le32(entry + 24), get_unaligned_le32(entry + 28)); + } + for (i = 0; i < 8; i++) { + u8 *entry = wifi->a.window + 0x200000 + i * 32; + + length += scnprintf(status + length, sizeof(status) - length, + "tx_completion[%u]=%08x %08x %08x %08x %08x %08x %08x %08x\n", + i, get_unaligned_le32(entry), get_unaligned_le32(entry + 4), + get_unaligned_le32(entry + 8), get_unaligned_le32(entry + 12), + get_unaligned_le32(entry + 16), get_unaligned_le32(entry + 20), + get_unaligned_le32(entry + 24), get_unaligned_le32(entry + 28)); + } + mutex_unlock(&wifi->lock); + return simple_read_from_buffer(output, count, position, status, length); +} + +static const struct file_operations data_status_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = data_status_read, +}; + +/* Contains private network traffic: retrieve only as a protected binary file. + * Samples are copied once into host memory and never change after count 8. + */ +static ssize_t data_rx_snapshot_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + ssize_t ret; + + mutex_lock(&wifi->lock); + ret = simple_read_from_buffer(output, count, position, &wifi->rx_snapshot, + sizeof(wifi->rx_snapshot)); + mutex_unlock(&wifi->lock); + return ret; +} + +static const struct file_operations data_rx_snapshot_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = data_rx_snapshot_read, + .llseek = default_llseek, +}; + +static ssize_t country_status_read(struct file *file, char __user *output, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + char status[96]; + unsigned int length; + int ret; + + mutex_lock(&wifi->lock); + wifi->country_valid = false; + ret = wifi->configured ? command(wifi, 10, NULL, 0) : -ENETDOWN; + length = scnprintf(status, sizeof(status), "rc=%d valid=%d country_bytes=%02x/%02x/%02x\n", + ret, wifi->country_valid, wifi->country[0], wifi->country[1], wifi->country[2]); + mutex_unlock(&wifi->lock); + return simple_read_from_buffer(output, count, position, status, length); +} + +static const struct file_operations country_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = country_status_read, +}; + +static int query_link_status(struct centauri_wifi *wifi) +{ + wifi->link_status_valid = wifi->emlsr_valid = false; + wifi->link_count = wifi->link_status_size = 0; + wifi->link_tlv_count = 0; + return wifi->configured ? command(wifi, 20, NULL, 0) : -ENETDOWN; +} + +static int link_status_show(struct seq_file *seq, void *unused) +{ + struct centauri_wifi *wifi = seq->private; + unsigned int i; + int ret; + + mutex_lock(&wifi->lock); + ret = query_link_status(wifi); + seq_printf(seq, "rc=%d valid=%d size=%u count=%u emlsr=%d/%u\n", ret, + wifi->link_status_valid, wifi->link_status_size, wifi->link_count, + wifi->emlsr_valid, wifi->emlsr_mode); + for (i = 0; i < wifi->link_tlv_count; i++) + seq_printf(seq, "tag=%u length=%u\n", wifi->link_tags[i], wifi->link_lengths[i]); + if (wifi->link_status_valid) + for (i = 0; i < wifi->link_count; i++) + seq_printf(seq, "link=%u prime=%u channel=%u band=%u\n", i, + wifi->link_prime[i], wifi->link_channel[i], wifi->link_band[i]); + mutex_unlock(&wifi->lock); + return 0; +} + +static int link_status_open(struct inode *inode, struct file *file) +{ + return single_open(file, link_status_show, inode->i_private); +} + +static const struct file_operations link_status_ops = { + .owner = THIS_MODULE, + .open = link_status_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int tx_config_show(struct seq_file *seq, void *unused) +{ + struct centauri_wifi *wifi = seq->private; + u8 id; + int ret; + + mutex_lock(&wifi->lock); + seq_printf(seq, "applied=%u\n", wifi->tx_defaults_applied); + for (id = 0; id <= 1; id++) { + wifi->tx_profile_valid = false; + ret = wifi->configured ? command(wifi, 56, &id, sizeof(id)) : -ENETDOWN; + seq_printf(seq, "profile=%u rc=%d status=%u valid=%d", id, ret, + wifi->reply_status, wifi->tx_profile_valid); + if (wifi->tx_profile_valid) + seq_printf(seq, " rate=%#x rts_threshold=%u rts_retries=%u retries=%u type=%u id=%u", + get_unaligned_le32(wifi->tx_profile), + get_unaligned_le16(wifi->tx_profile + 4), wifi->tx_profile[6], + wifi->tx_profile[7], wifi->tx_profile[8], wifi->tx_profile[9]); + seq_putc(seq, '\n'); + } + mutex_unlock(&wifi->lock); + return 0; +} + +static int tx_config_open(struct inode *inode, struct file *file) +{ + return single_open(file, tx_config_show, inode->i_private); +} + +static ssize_t tx_config_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = ((struct seq_file *)file->private_data)->private; + /* Exact configureTxDataCtrlPacketProfile: command 52, TLV 340/10. + * 0xa0154 is the TLV header, not the command ID. + */ + static const u8 control_profile[] = { + 0x54, 1, 10, 0, 0, 8, 0, 0, 0xff, 0xff, 0xff, 0xff, 1, 1, + }; + u8 lifetime[20] = {}; + unsigned int action, i; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &action); + if (ret || (action != 1 && action != 2)) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + if (!wifi->configured || wifi->connecting || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + if (action == 1) { + /* AppleWLAN80211STA::resetTxPacketLifetime selects 2000 ms for + * each AC. The ACI adapter converts to firmware microseconds. + */ + for (i = 0; i < 4; i++) + put_unaligned_le32(2000000, lifetime + 4 + 4 * i); + ret = command(wifi, 43, lifetime, sizeof(lifetime)); + } else { + ret = command(wifi, 52, control_profile, sizeof(control_profile)); + } + if (!ret) + wifi->tx_defaults_applied |= BIT(action - 1); + dev_info(&wifi->control->pdev->dev, "reference TX setup action=%u rc=%d\n", action, ret); +out: + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations tx_config_ops = { + .owner = THIS_MODULE, + .open = tx_config_open, + .read = seq_read, + .write = tx_config_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static ssize_t host_wakeup_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + /* ACIIO80211PowerAdapter::hostWakeup selects state 0 in command 6. + * This firmware's SysCmdSetHostPowerState accepts the fixed 27-byte + * request without TLVs. Keep its existing event filter: the newer host + * adapter's 92-byte event list exceeds this firmware's 86-byte limit. + */ + const u8 payload[3] = {}; + unsigned int action; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &action); + if (ret || action != 1) + return ret ?: -EINVAL; + mutex_lock(&wifi->lock); + if (!wifi->configured || wifi->connecting || wifi->scan_request || wifi->connected) { + ret = -EBUSY; + goto out; + } + ret = command(wifi, 6, payload, sizeof(payload)); + dev_info(&wifi->control->pdev->dev, "host wakeup state=0 rc=%d status=%u stage=%u\n", + ret, wifi->reply_status, readl(wifi->a.bar + 8)); +out: + mutex_unlock(&wifi->lock); + return ret ?: count; +} + +static const struct file_operations host_wakeup_ops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = host_wakeup_write, +}; + +static int query_station_stats(struct centauri_wifi *wifi) +{ + u8 payload[22] = {}; + int ret; + + /* ACIDebugAdapter::queryStaCounterStats: statistic bit 37, all peers. + * Its fixed counter record contains no key or packet payload bytes. + */ + put_unaligned_le64(BIT_ULL(37), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + wifi->station_stats_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + return ret; +} + +static int station_stats_show(struct seq_file *seq, void *unused) +{ + struct centauri_wifi *wifi = seq->private; + unsigned int i; + int ret; + + mutex_lock(&wifi->lock); + ret = query_station_stats(wifi); + seq_printf(seq, "rc=%d valid=%d size=%u tag=%#x length=%u\n", ret, + wifi->station_stats_valid, wifi->stats_reply_size, wifi->stats_tag, wifi->stats_length); + if (!ret && wifi->station_stats_valid) + for (i = 0; i < wifi->stats_length; i += 4) { + u32 value = get_unaligned_le32(wifi->station_stats + i); + + if (value) + seq_printf(seq, "%03x %u\n", i, value); + } + mutex_unlock(&wifi->lock); + return 0; +} + +static int station_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, station_stats_show, inode->i_private); +} + +static const struct file_operations station_stats_ops = { + .owner = THIS_MODULE, + .open = station_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int rx_crypto_stats_show(struct seq_file *seq, void *unused) +{ + /* ACIStatisticsAdapter::printRxChipCounterStats at 0x1809d7b80: + * statistic bit 30, response TLV 0xf921, sixteen LE32 counters. + * Unlike the station record, this fixed structure contains no MACs. + */ + static const char * const names[] = { + "rx_duplicate_errors", "rx_ampdu_duplicate_errors", + "rx_unicast_tkip_replay_errors", "rx_unicast_tkip_decrypt_errors", + "rx_multicast_tkip_replay_errors", "rx_multicast_tkip_decrypt_errors", + "rx_unicast_ccmp_replay_errors", "rx_unicast_ccmp_decrypt_errors", + "rx_multicast_ccmp_replay_errors", "rx_multicast_ccmp_decrypt_errors", + "phy_total_detect_count", "phy_ofdm_detect_count", "phy_11b_detect_count", + "phy_plcp_error_count", "phy_total_receive_count", "phy_plcp_good_count", + }; + struct centauri_wifi *wifi = seq->private; + u8 payload[22] = {}; + unsigned int i; + int ret; + + put_unaligned_le64(BIT_ULL(30), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + mutex_lock(&wifi->lock); + wifi->rx_crypto_stats_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + seq_printf(seq, "rc=%d valid=%d size=%u tag=%#x length=%u\n", ret, + wifi->rx_crypto_stats_valid, wifi->stats_reply_size, wifi->stats_tag, wifi->stats_length); + if (!ret && wifi->rx_crypto_stats_valid) + for (i = 0; i < ARRAY_SIZE(names); i++) + seq_printf(seq, "%s=%u\n", names[i], + get_unaligned_le32(wifi->rx_crypto_stats + i * 4)); + mutex_unlock(&wifi->lock); + return 0; +} + +static int rx_crypto_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, rx_crypto_stats_show, inode->i_private); +} + +static const struct file_operations rx_crypto_stats_ops = { + .owner = THIS_MODULE, + .open = rx_crypto_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int dot11_mib_stats_show(struct seq_file *seq, void *unused) +{ + /* ACIStatisticsAdapter::printDot11MibStats at 0x1809d7fb0: + * statistic bit 25, scope 2/interface 0, TLV 0xf91b, 24 LE64s. + * Keep the host field names: tx_status_success is firmware accounting, + * not independent proof of over-the-air acknowledgement or AP delivery. + */ + static const char * const names[] = { + "rx_msdu_pushed", "rx_msdu_pushed_multi", "rx_mpdu", "rx_mpdu_dup", + "rx_ampdu_dup", "rx_mpdu_drop_error", "rx_msdu_drop_filter", + "rx_holes_0", "rx_holes_1", "rx_holes_2", "rx_holes_3", "rx_holes_4", + "rx_holes_5", "rx_holes_6", "rx_holes_7", "rx_holes_8", + "tx_pulled", "tx_pulled_multi", "tx_pulled_core", "tx_status_success", + "tx_status_abort_request", "tx_status_peer_gone", + "tx_status_retries_exhausted", "tx_status_lifetime_expired", + }; + struct centauri_wifi *wifi = seq->private; + u8 payload[22] = {}; + unsigned int i; + int ret; + + put_unaligned_le64(BIT_ULL(25), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + /* The scope byte follows the eight-byte interface value in TLV 363. */ + payload[21] = 2; + mutex_lock(&wifi->lock); + wifi->dot11_mib_stats_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + seq_printf(seq, "rc=%d valid=%d size=%u tag=%#x length=%u\n", ret, + wifi->dot11_mib_stats_valid, wifi->stats_reply_size, wifi->stats_tag, wifi->stats_length); + if (!ret && wifi->dot11_mib_stats_valid) + for (i = 0; i < ARRAY_SIZE(names); i++) + seq_printf(seq, "%s=%llu\n", names[i], + (unsigned long long)get_unaligned_le64(wifi->dot11_mib_stats + i * 8)); + mutex_unlock(&wifi->lock); + return 0; +} + +static int dot11_mib_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, dot11_mib_stats_show, inode->i_private); +} + +static const struct file_operations dot11_mib_stats_ops = { + .owner = THIS_MODULE, + .open = dot11_mib_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int tx_power_report_show(struct seq_file *seq, void *unused) +{ + /* Exact stock 25G76 WLANACIAdapter::queryLastTxStats, 0x180914d58: + * bit29, scope1/radio bitmap, TLV 0xf920 with 280 bytes. The newer + * macOS27 adapter expects 320 bytes and has a different record stride. + * Stock getLastTxInfo (0x1809150f8) reads the first five 28-byte records. + * These are rate/power observations, not proof of ACK or AP delivery. + */ + struct centauri_wifi *wifi = seq->private; + u8 payload[22] = {}; + unsigned int radio, i; + int ret; + + put_unaligned_le64(BIT_ULL(29), payload); + payload[8] = 1; + put_unaligned_le16(363, payload + 9); + put_unaligned_le16(9, payload + 11); + payload[21] = 1; + mutex_lock(&wifi->lock); + for (radio = 1; radio <= 2; radio++) { + put_unaligned_le64(radio, payload + 13); + wifi->tx_power_report_valid = false; + wifi->stats_reply_size = wifi->stats_tag = wifi->stats_length = 0; + ret = wifi->configured ? command(wifi, 0xb0001, payload, sizeof(payload)) : -ENETDOWN; + if (!ret) { + usleep_range(1000, 2000); + drain(wifi); + } + seq_printf(seq, "radio_bitmap=%u rc=%d valid=%d size=%u tag=%#x length=%u\n", + radio, ret, wifi->tx_power_report_valid, wifi->stats_reply_size, + wifi->stats_tag, wifi->stats_length); + if (ret || !wifi->tx_power_report_valid) + continue; + for (i = 0; i < 5; i++) { + const u8 *record = wifi->tx_power_report + i * 28; + + /* Keep hardware enums and power bytes raw until interpreted + * against this exact firmware; omit unreviewed record fields. + */ + seq_printf(seq, "sample=%u power0_qdb_raw=%u power1_qdb_raw=%u power_qdb_raw=%u " + "hw_preamble=%u hw_rate=%u nss=%u hw_bw=%u hw_gi=%u\n", + i, record[14], record[15], record[16], record[17], + record[20], record[21] + 1, record[24], record[27]); + } + } + mutex_unlock(&wifi->lock); + return 0; +} + +static int tx_power_report_open(struct inode *inode, struct file *file) +{ + return single_open(file, tx_power_report_show, inode->i_private); +} + +static const struct file_operations tx_power_report_ops = { + .owner = THIS_MODULE, + .open = tx_power_report_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +int centauri_wifi_register(struct centauri *c) +{ + struct centauri_wifi *wifi; + unsigned int i; + int ret; + + wifi = kzalloc(sizeof(*wifi), GFP_KERNEL); + if (!wifi) + return -ENOMEM; + c->wifi = wifi; + wifi->control = c; + mutex_init(&wifi->lock); + mutex_init(&wifi->fwlog_lock); + skb_queue_head_init(&wifi->data_tx_queue); + INIT_WORK(&wifi->scan_work, scan_worker); + INIT_WORK(&wifi->connect_work, connect_worker); + INIT_DELAYED_WORK(&wifi->link_work, link_worker); + ret = centauri_alpha_resources(c, &wifi->a); + if (ret) + goto failed; + wifi->wiphy = wiphy_new(&wifi_ops, sizeof(struct centauri_wifi *)); + wifi->netdev = alloc_etherdev(sizeof(struct centauri_wifi *)); + if (!wifi->wiphy || !wifi->netdev) { + ret = -ENOMEM; + goto failed; + } + *(struct centauri_wifi **)wiphy_priv(wifi->wiphy) = wifi; + *(struct centauri_wifi **)netdev_priv(wifi->netdev) = wifi; + set_wiphy_dev(wifi->wiphy, &c->pdev->dev); + wifi->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION); + wifi->wiphy->max_scan_ssids = 1; + wifi->wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM; + wifi->wiphy->reg_notifier = wifi_regulatory; + wifi->wiphy->cipher_suites = cipher_suites; + wifi->wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites); + wiphy_ext_feature_set(wifi->wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); + for (i = 0; i < 13; i++) { + wifi->channels[i].band = NL80211_BAND_2GHZ; + wifi->channels[i].hw_value = i + 1; + wifi->channels[i].center_freq = ieee80211_channel_to_frequency(i + 1, NL80211_BAND_2GHZ); + wifi->channels[i].max_power = 20; + } + for (i = 0; i < ARRAY_SIZE(five_ghz); i++) { + wifi->channels[13 + i].band = NL80211_BAND_5GHZ; + wifi->channels[13 + i].hw_value = five_ghz[i]; + wifi->channels[13 + i].center_freq = ieee80211_channel_to_frequency(five_ghz[i], NL80211_BAND_5GHZ); + wifi->channels[13 + i].max_power = 20; + } + wifi->bands[0].channels = wifi->channels; + wifi->bands[0].n_channels = 13; + wifi->bands[0].bitrates = rates; + wifi->bands[0].n_bitrates = ARRAY_SIZE(rates); + wifi->bands[1].channels = wifi->channels + 13; + wifi->bands[1].n_channels = ARRAY_SIZE(five_ghz); + wifi->bands[1].bitrates = rates + 4; + wifi->bands[1].n_bitrates = ARRAY_SIZE(rates) - 4; + wifi->wiphy->bands[NL80211_BAND_2GHZ] = &wifi->bands[0]; + wifi->wiphy->bands[NL80211_BAND_5GHZ] = &wifi->bands[1]; + ret = wiphy_register(wifi->wiphy); + if (ret) + goto failed; + wifi->registered = true; + wifi->wdev.wiphy = wifi->wiphy; + wifi->wdev.iftype = NL80211_IFTYPE_STATION; + wifi->wdev.netdev = wifi->netdev; + wifi->netdev->ieee80211_ptr = &wifi->wdev; + wifi->netdev->netdev_ops = &net_ops; + SET_NETDEV_DEV(wifi->netdev, &c->pdev->dev); + strscpy(wifi->netdev->name, "wlan%d", IFNAMSIZ); + wifi->netdev->max_mtu = ETH_DATA_LEN; + eth_hw_addr_random(wifi->netdev); + ret = register_netdev(wifi->netdev); + if (ret) + goto failed; + ret = radio_start(wifi); + if (ret) { + dev_err(&c->pdev->dev, "Wi-Fi ring/interface setup failed: %d\n", ret); + goto failed; + } + dev_info(&c->pdev->dev, "cfg80211 interface %s ready for scan tests; packet data unavailable\n", + wifi->netdev->name); + wifi->passphrase_file = debugfs_create_file("passphrase", 0200, c->debug, wifi, &passphrase_ops); + wifi->data_probe_file = debugfs_create_file("data_probe", 0200, c->debug, wifi, &data_probe_ops); + wifi->data_status_file = debugfs_create_file("data_status", 0400, c->debug, wifi, &data_status_ops); + wifi->data_rx_snapshot_file = debugfs_create_file("data_rx_snapshot", 0400, + c->debug, wifi, &data_rx_snapshot_ops); + wifi->station_stats_file = debugfs_create_file("station_stats", 0400, c->debug, wifi, &station_stats_ops); + wifi->rx_crypto_stats_file = debugfs_create_file("rx_crypto_stats", 0400, + c->debug, wifi, &rx_crypto_stats_ops); + wifi->dot11_mib_stats_file = debugfs_create_file("dot11_mib_stats", 0400, + c->debug, wifi, &dot11_mib_stats_ops); + wifi->tx_power_report_file = debugfs_create_file("tx_power_report", 0400, + c->debug, wifi, &tx_power_report_ops); + wifi->eapol_start_file = debugfs_create_file("eapol_start", 0200, c->debug, wifi, &eapol_start_ops); + wifi->country_file = debugfs_create_file("country", 0400, c->debug, wifi, &country_ops); + wifi->link_status_file = debugfs_create_file("link_status", 0400, c->debug, wifi, &link_status_ops); + wifi->tx_config_file = debugfs_create_file("tx_config", 0600, c->debug, wifi, &tx_config_ops); + wifi->host_wakeup_file = debugfs_create_file("host_wakeup", 0200, c->debug, wifi, &host_wakeup_ops); + wifi->fwlog_start_file = debugfs_create_file("fwlog_start", 0200, c->debug, wifi, &fwlog_start_ops); + wifi->fwlog_status_file = debugfs_create_file("fwlog_status", 0400, c->debug, wifi, &fwlog_status_ops); + wifi->fwlog_buffer_file = debugfs_create_file("fwlog_buffer", 0400, c->debug, wifi, &fwlog_buffer_ops); + wifi->fwlog_ack_file = debugfs_create_file("fwlog_ack", 0200, c->debug, wifi, &fwlog_ack_ops); + return 0; +failed: + centauri_wifi_unregister(c); + return ret; +} + +void centauri_wifi_unregister(struct centauri *c) +{ + struct centauri_wifi *wifi = c->wifi; + + if (!wifi) + return; + WRITE_ONCE(wifi->stopping, true); + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + netif_tx_disable(wifi->netdev); + debugfs_remove(wifi->passphrase_file); + debugfs_remove(wifi->data_probe_file); + debugfs_remove(wifi->data_status_file); + debugfs_remove(wifi->data_rx_snapshot_file); + debugfs_remove(wifi->station_stats_file); + debugfs_remove(wifi->rx_crypto_stats_file); + debugfs_remove(wifi->dot11_mib_stats_file); + debugfs_remove(wifi->tx_power_report_file); + debugfs_remove(wifi->eapol_start_file); + debugfs_remove(wifi->country_file); + debugfs_remove(wifi->link_status_file); + debugfs_remove(wifi->tx_config_file); + debugfs_remove(wifi->host_wakeup_file); + debugfs_remove(wifi->fwlog_start_file); + debugfs_remove(wifi->fwlog_status_file); + debugfs_remove(wifi->fwlog_buffer_file); + debugfs_remove(wifi->fwlog_ack_file); + cancel_work_sync(&wifi->scan_work); + cancel_work_sync(&wifi->connect_work); + cancel_delayed_work_sync(&wifi->link_work); + skb_queue_purge(&wifi->data_tx_queue); + if (wifi->scan_request) { + struct cfg80211_scan_info info = { .aborted = true }; + + cfg80211_scan_done(wifi->scan_request, &info); + wifi->scan_request = NULL; + } + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + unregister_netdev(wifi->netdev); + if (wifi->registered) + wiphy_unregister(wifi->wiphy); + if (wifi->wiphy) + wiphy_free(wifi->wiphy); + if (wifi->netdev) + free_netdev(wifi->netdev); + memzero_explicit(wifi, sizeof(*wifi)); + kfree(wifi); + c->wifi = NULL; +} From a03e01fbb531537d7e07fc00689cb694a846d79a Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 10 Sep 2026 01:15:28 -0400 Subject: [PATCH 039/117] Bluetooth: add J813 N1 native transport and encrypted connection bring-up [Thomas: the driver part of c0f6eceacec7 on the J813 line of linux-aurora] Signed-off-by: Thomas Glanzmann --- drivers/bluetooth/Kconfig | 14 + drivers/bluetooth/Makefile | 9 +- drivers/bluetooth/btcentauri.c | 984 ++++++++++++++++++ .../wireless/apple/centauri/centauri_ipc.c | 37 + 4 files changed, 1040 insertions(+), 4 deletions(-) create mode 100644 drivers/bluetooth/btcentauri.c diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index b35637d575a34..832e2d3019ecc 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -506,4 +506,18 @@ config BT_INTEL_PCIE Say Y here to compiler support for Intel Bluetooth PCIe device into the kernel or say M to compile it as module (btintel_pcie) + +config BT_CENTAURI + tristate "Apple N1 Beta Bluetooth transport (experimental)" + depends on PCI && (ARCH_APPLE || COMPILE_TEST) + help + Experimental Bluetooth transport for Apple N1 on J813. + The Centauri Control driver must boot the shared firmware before this + driver binds to Beta. Controller initialization, discovery, BR/EDR ACL, + pairing and encrypted AVDTP discovery have been tested on hardware. + ACL delivery and native startup/connection handling remain opt-in; + audio playback and LE data are unqualified. SCO/ISO is not implemented. + + Say M to build the btcentauri module for bring-up testing. + endmenu diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile index e0f1cea5b6092..74c3834c2c4a3 100644 --- a/drivers/bluetooth/Makefile +++ b/drivers/bluetooth/Makefile @@ -7,8 +7,12 @@ obj-$(CONFIG_BT_HCIVHCI) += hci_vhci.o obj-$(CONFIG_BT_HCIUART) += hci_uart.o obj-$(CONFIG_BT_HCIBCM203X) += bcm203x.o obj-$(CONFIG_BT_HCIBCM4377) += hci_bcm4377.o +obj-$(CONFIG_BT_HCIMT79XX_PCIE) += btmtk_pcie.o obj-$(CONFIG_BT_HCIBPA10X) += bpa10x.o obj-$(CONFIG_BT_HCIBFUSB) += bfusb.o +obj-$(CONFIG_BT_HCIDTL1) += dtl1_cs.o +obj-$(CONFIG_BT_HCIBT3C) += bt3c_cs.o +obj-$(CONFIG_BT_HCIBLUECARD) += bluecard_cs.o obj-$(CONFIG_BT_HCIBTUSB) += btusb.o obj-$(CONFIG_BT_HCIBTSDIO) += btsdio.o @@ -50,7 +54,4 @@ hci_uart-$(CONFIG_BT_HCIUART_AG6XX) += hci_ag6xx.o hci_uart-$(CONFIG_BT_HCIUART_MRVL) += hci_mrvl.o hci_uart-$(CONFIG_BT_HCIUART_AML) += hci_aml.o hci_uart-objs := $(hci_uart-y) - -obj-$(CONFIG_BT_MTK7932_PCIE) += mt7932_bt_pcie.o - -CONTEXT_ANALYSIS := y +obj-$(CONFIG_BT_CENTAURI) += btcentauri.o diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c new file mode 100644 index 0000000000000..d2539539fa162 --- /dev/null +++ b/drivers/bluetooth/btcentauri.c @@ -0,0 +1,984 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Apple N1 Beta command/event transport for J813 bring-up. + * + * The Centauri Control driver must have booted the shared firmware first. + * This driver owns only Beta. It never resets the port or the Wi-Fi function. + * BR/EDR discovery, ACL/SDP, pairing and encrypted AVDTP discovery are qualified. + * ACL remains opt-in; LE data, persistent bonds and audio are unqualified. + * SCO/ISO delivery is not implemented. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CEN_ARENA_SIZE SZ_1M +#define CEN_MTR 0x2000 +#define CEN_MCR 0x2200 +#define CEN_MESSAGES 0x4000 +#define CEN_RX 0x8000 +#define CEN_TX 0xa000 +#define CEN_FOOTER 260 +#define CEN_STRIDE (16 + CEN_FOOTER) +#define CEN_RX_SLOTS 17 +#define CEN_TX_SLOTS 5 +#define CEN_LE_RX 0xc000 +#define CEN_LE_TX 0xe000 +#define CEN_LE_FOOTER 256 +#define CEN_LE_STRIDE (16 + CEN_LE_FOOTER) +#define CEN_LE_SLOTS 17 +#define CEN_MESSAGE_SLOTS 16 +#define CEN_AUX_BASE 0x20000 +#define CEN_CAPTURE_SIZE SZ_1M + +static bool arm_le_data_rings = true; +module_param(arm_le_data_rings, bool, 0444); +MODULE_PARM_DESC(arm_le_data_rings, "Arm native Beta LE rings for radio bring-up"); + +static bool arm_auxiliary_rings; +module_param(arm_auxiliary_rings, bool, 0444); +MODULE_PARM_DESC(arm_auxiliary_rings, "Diagnostic: arm remaining native Beta resources and capture received frames privately"); + +static bool native_startup; +module_param(native_startup, bool, 0444); +MODULE_PARM_DESC(native_startup, "Diagnostic: program a temporary address, InitDone and random seed before the first HCI reset"); + +static bool enable_acl; +module_param(enable_acl, bool, 0444); +MODULE_PARM_DESC(enable_acl, "Experimental ACL data delivery; requires LE and auxiliary rings"); + +static bool native_hrb; +module_param(native_hrb, bool, 0444); +MODULE_PARM_DESC(native_hrb, "Experimental native Proxima Create Connection; requires native startup, ACL and firmware capability bit 9"); + +#define CEN_OP_READ_PROP_FEATURES 0xffa3 +#define CEN_OP_HRB_CREATE_CONN 0xff7b +#define CEN_PROP_HRB_CREATE BIT(9) + +struct cen_aux_spec { + u16 id, slots, footer; + bool receive; +}; + +/* Host native Beta inventory; each resource is opt-in until target-qualified. + * HCI and LE rings remain in their existing, separately qualified arenas. + */ +static const struct cen_aux_spec cen_aux_specs[] = { + { 2, 9, 2592, false }, /* ACL TX */ + { 3, 9, 724, false }, /* SCO TX */ + { 5, 9, 260, false }, /* LEA TX */ + { 6, 5, 1024, false }, /* LLA TX */ + { 8, 9, 2592, true }, /* ACL RX */ + { 9, 9, 724, true }, /* SCO RX */ + { 11, 9, 268, true }, /* LEA RX */ + { 12, 128, 1024, true }, /* Firmware debug RX */ + { 13, 17, 260, true }, /* LMP RX */ + { 14, 9, 1304, false }, /* TSI TX */ + { 15, 9, 1304, true }, /* TSI RX */ +}; + +struct cen_aux_state { + u32 offset, packets; + u16 head, post; +}; + +struct cen_bt { + struct pci_dev *pdev; + struct hci_dev *hdev; + struct dentry *debug; + void __iomem *bar; + u8 *arena; + dma_addr_t dma; + struct mutex lock; + struct delayed_work poll; + struct sk_buff_head txq; + struct sk_buff_head acl_txq, le_txq; + u16 acl_tx_head, le_tx_head; + u16 message_head, message_tag, tx_head, rx_head, rx_post; + u16 le_rx_head, le_rx_post; + u32 le_rx_packets; + struct cen_aux_state aux[ARRAY_SIZE(cen_aux_specs)]; + u8 *capture; + u32 capture_used, capture_records, capture_dropped; + bool ipc_requested, running; + u8 native_startup_steps; + bool hrb_supported, hrb_status_pending; + u32 proprietary_features, hrb_commands, hrb_status_events; + u32 hrb_connection_events; + u8 hrb_last_link_type; +}; + +static u16 cen_tail(struct cen_bt *bt, u16 ring) +{ + return le16_to_cpu(READ_ONCE(*(__le16 *)(bt->arena + 0x7a + ring * 2))); +} + +static void cen_post(struct cen_bt *bt, u16 ring, u16 value) +{ + put_unaligned_le16(value, bt->arena + 0xa8 + ring * 2); + dma_wmb(); + writel(value, bt->bar + 0x1000 + ring * 4); +} + +static int cen_message(struct cen_bt *bt, const void *data, size_t length) +{ + u16 head = bt->message_head, tail, next = (head + 1) % CEN_MESSAGE_SLOTS; + u16 tag = bt->message_tag + 1; + u8 *descriptor = bt->arena + CEN_MTR + head * 16; + u8 *completion = bt->arena + CEN_MCR + head * 32; + int ret; + + if (head >= CEN_MESSAGE_SLOTS || !length || length > 512) + return -EINVAL; + memset(descriptor, 0, 16); + memset(completion, 0, 32); + memcpy(bt->arena + CEN_MESSAGES + head * 512, data, length); + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(bt->dma + CEN_MESSAGES + head * 512, descriptor + 4); + put_unaligned_le16(tag, descriptor + 12); + cen_post(bt, 0, next); + ret = read_poll_timeout(cen_tail, tail, tail == next, + 100, 1500000, false, bt, 0); + if (ret) + return ret; + /* The device publishes the message completion before advancing its head. */ + ret = read_poll_timeout(le16_to_cpu, tail, tail == next, + 100, 1500000, false, + READ_ONCE(*(__le16 *)(bt->arena + 0x78))); + if (ret) + return ret; + dma_rmb(); + if (get_unaligned_le16(completion + 4) != tag || + get_unaligned_le16(completion + 6) != length || + get_unaligned_le32(completion + 8)) + return -EPROTO; + bt->message_head = next; + bt->message_tag = tag; + put_unaligned_le16(bt->message_head, bt->arena + 0xa6); + dma_wmb(); + return 0; +} + +static int cen_open_ring(struct cen_bt *bt, u16 id, u16 slots, u32 offset, u16 footer) +{ + u8 message[0x34] = { 1 }; + u32 words = footer / 4, shift = 0; + + if (!words || footer % 4) + return -EINVAL; + if (words > 255) + shift = min_t(u32, __ffs(words), 4); + if ((words >> shift) > 255) + return -EINVAL; + message[2] = words >> shift; + message[3] = shift << 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le64(bt->dma + offset, message + 8); + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(slots, message + 0x18); + put_unaligned_le16(0xffff, message + 0x1a); + put_unaligned_le16(id, message + 0x1c); + put_unaligned_le16(0x40, message + 0x1e); + return cen_message(bt, message, sizeof(message)); +} + +static void cen_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_RX + slot * CEN_STRIDE; + + memset(descriptor, 0, CEN_STRIDE); + put_unaligned_le32(1 | (CEN_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static void cen_le_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_LE_RX + slot * CEN_LE_STRIDE; + + memset(descriptor, 0, CEN_LE_STRIDE); + put_unaligned_le32(1 | (CEN_LE_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static void cen_capture_frame(struct cen_bt *bt, u16 ring, const u8 *data, u16 length) +{ + u8 *record; + + if (!bt->capture) + return; + if (8U + length > CEN_CAPTURE_SIZE - bt->capture_used) { + bt->capture_dropped++; + return; + } + record = bt->capture + bt->capture_used; + put_unaligned_le16(ring, record); + put_unaligned_le16(length, record + 2); + put_unaligned_le32(bt->capture_records++, record + 4); + memcpy(record + 8, data, length); + bt->capture_used += 8 + length; +} + +static int cen_receive_acl(struct cen_bt *bt, const u8 *data, u32 length) +{ + struct sk_buff *skb; + u32 packet_length, pos; + + if (length < HCI_ACL_HDR_SIZE) + return -EPROTO; + packet_length = HCI_ACL_HDR_SIZE + get_unaligned_le16(data + 2); + if (packet_length > length) + return -EPROTO; + /* Airship RX attributes follow, rather than belong to, the HCI packet. + * Only the native parser's known record sizes are accepted. Their values + * remain opaque and are retained only in the private diagnostic capture. + */ + for (pos = packet_length; pos < length;) { + if (!data[pos]) + break; + if (data[pos] == 1) + pos++; + else if (data[pos] == 2 && length - pos >= 9) + pos += 9; + else + return -EPROTO; + } + skb = bt_skb_alloc(packet_length, GFP_KERNEL); + if (!skb) { + bt->hdev->stat.err_rx++; + return 0; + } + skb_put_data(skb, data, packet_length); + hci_skb_pkt_type(skb) = HCI_ACLDATA_PKT; + bt->hdev->stat.byte_rx += packet_length; + hci_recv_frame(bt->hdev, skb); + return 0; +} + +static void cen_aux_rx_buffer(struct cen_bt *bt, unsigned int index, u16 slot) +{ + const struct cen_aux_spec *spec = &cen_aux_specs[index]; + u8 *descriptor = bt->arena + bt->aux[index].offset + slot * (16 + spec->footer); + + memset(descriptor, 0, 16 + spec->footer); + put_unaligned_le32(1 | (spec->footer << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static int cen_aux_start(struct cen_bt *bt) +{ + u32 offset = CEN_AUX_BASE; + unsigned int i, slot; + int ret; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u32 length = spec->slots * (16U + spec->footer); + + if (offset > CEN_ARENA_SIZE || length > CEN_ARENA_SIZE - offset) + return -ENOSPC; + state->offset = offset; + offset = ALIGN(offset + length, SZ_4K); + ret = cen_open_ring(bt, spec->id, spec->slots, state->offset, spec->footer); + if (ret) { + dev_err(&bt->pdev->dev, "Auxiliary ring %u open failed: %d\n", spec->id, ret); + return ret; + } + if (!spec->receive) + continue; + for (slot = 0; slot < spec->slots - 1; slot++) + cen_aux_rx_buffer(bt, i, slot); + state->post = spec->slots - 1; + cen_post(bt, spec->id, state->post); + } + return 0; +} + +static int cen_aux_drain(struct cen_bt *bt, bool repost) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u16 tail; + int budget = spec->slots; + + if (!spec->receive) + continue; + tail = cen_tail(bt, spec->id); + if (tail >= spec->slots) + return -EPROTO; + while (state->head != tail && budget--) { + u8 *descriptor = bt->arena + state->offset + state->head * (16 + spec->footer); + u32 length; + + dma_rmb(); + length = get_unaligned_le32(descriptor) >> 8; + if (length > spec->footer) + return -EPROTO; + cen_capture_frame(bt, spec->id, descriptor + 16, length); + if (repost && enable_acl && spec->id == 8 && + cen_receive_acl(bt, descriptor + 16, length)) + return -EPROTO; + state->packets++; + state->head = (state->head + 1) % spec->slots; + if (repost) { + cen_aux_rx_buffer(bt, i, state->post); + state->post = (state->post + 1) % spec->slots; + cen_post(bt, spec->id, state->post); + } + } + } + return 0; +} + +static int cen_start(struct cen_bt *bt) +{ + static const u16 indices[] = { 0x78, 0x7a, 0xa6, 0xa8 }; + u8 config[4] = { 9, 0, 0xff, 0xff }; + u32 status; + int i, ret; + + put_unaligned_le16(0x300, bt->arena); + put_unaligned_le16(0x68, bt->arena + 2); + put_unaligned_le64(bt->dma + 0x68, bt->arena + 8); + for (i = 0; i < ARRAY_SIZE(indices); i++) + put_unaligned_le64(bt->dma + indices[i], bt->arena + 0x10 + i * 8); + put_unaligned_le16(1, bt->arena + 0x30); + put_unaligned_le16(22, bt->arena + 0x32); + put_unaligned_le64(bt->dma + CEN_MCR, bt->arena + 0x34); + put_unaligned_le64(bt->dma + CEN_MTR, bt->arena + 0x3c); + put_unaligned_le16(16, bt->arena + 0x44); + put_unaligned_le16(16, bt->arena + 0x46); + /* The 25G76 native context encodes the MCR footer in four-byte words. */ + bt->arena[0x53] = 4; + put_unaligned_le64(bt->dma + 0x10000, bt->arena + 0x58); + pci_set_master(bt->pdev); + writel(lower_32_bits(bt->dma), bt->bar + 0xc); + writel(upper_32_bits(bt->dma), bt->bar + 0x10); + writel(CEN_ARENA_SIZE, bt->bar + 0x14); + bt->ipc_requested = true; + writel(1, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 1, 100, 2000000); + if (ret) + return ret; + dma_wmb(); + writel(lower_32_bits(bt->dma), bt->bar + 4); + writel(upper_32_bits(bt->dma), bt->bar + 8); + writel(2, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 2, 100, 2000000); + if (ret) + return ret; + ret = cen_message(bt, config, sizeof(config)); + if (!ret) + ret = cen_open_ring(bt, 7, CEN_RX_SLOTS, CEN_RX, CEN_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 1, CEN_TX_SLOTS, CEN_TX, CEN_FOOTER); + if (!ret && arm_le_data_rings) + ret = cen_open_ring(bt, 10, CEN_LE_SLOTS, CEN_LE_RX, CEN_LE_FOOTER); + if (!ret && arm_le_data_rings) + ret = cen_open_ring(bt, 4, CEN_LE_SLOTS, CEN_LE_TX, CEN_LE_FOOTER); + if (ret) + return ret; + for (i = 0; i < CEN_RX_SLOTS - 1; i++) + cen_rx_buffer(bt, i); + bt->rx_post = CEN_RX_SLOTS - 1; + cen_post(bt, 7, bt->rx_post); + if (arm_le_data_rings) { + for (i = 0; i < CEN_LE_SLOTS - 1; i++) + cen_le_rx_buffer(bt, i); + bt->le_rx_post = CEN_LE_SLOTS - 1; + cen_post(bt, 10, bt->le_rx_post); + } + if (arm_auxiliary_rings) + return cen_aux_start(bt); + return 0; +} + +static int cen_transmit(struct cen_bt *bt, struct sk_buff_head *queue, + u16 ring, u16 slots, u32 offset, u16 footer, u16 *head) +{ + struct sk_buff *skb; + u16 tail, next; + u8 *descriptor; + u32 wire_length; + + while (!skb_queue_empty(queue)) { + tail = cen_tail(bt, ring); + if (tail >= slots) + return -EPROTO; + next = (*head + 1) % slots; + if (next == tail) + break; + dma_rmb(); + skb = skb_dequeue(queue); + if (!skb) + break; + descriptor = bt->arena + offset + *head * (16 + footer); + memset(descriptor, 0, 16 + footer); + put_unaligned_le16(*head + 1, descriptor + 12); + memcpy(descriptor + 16, skb->data, skb->len); + wire_length = skb->len; + if (ring == 1 && READ_ONCE(bt->hrb_supported) && + get_unaligned_le16(skb->data) == HCI_OP_CREATE_CONN) { + u8 *command = descriptor + 16; + + /* Native Proxima uses the ordinary 13-byte parameter layout + * followed by two zero HRB fields (no forced band). It sends + * tagging=false and role_switch=0. Address byte order matches + * standard HCI. Keep the core's sent skb/opcode unchanged. + */ + put_unaligned_le16(CEN_OP_HRB_CREATE_CONN, command); + command[2] = 15; + command[HCI_COMMAND_HDR_SIZE + 9] = 0; + command[HCI_COMMAND_HDR_SIZE + 12] = 0; + wire_length = HCI_COMMAND_HDR_SIZE + 15; + bt->hrb_status_pending = true; + bt->hrb_commands++; + cen_capture_frame(bt, 1, command, wire_length); + } + put_unaligned_le32(1 | (wire_length << 8), descriptor); + *head = next; + cen_post(bt, ring, next); + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT) + bt->hdev->stat.cmd_tx++; + else + bt->hdev->stat.acl_tx++; + bt->hdev->stat.byte_tx += wire_length; + kfree_skb(skb); + } + return 0; +} + +static void cen_poll(struct work_struct *work) +{ + struct cen_bt *bt = container_of(to_delayed_work(work), struct cen_bt, poll); + struct hci_dev *hdev = bt->hdev; + struct sk_buff *skb; + u16 tail; + u8 *descriptor; + u32 length; + int budget = CEN_RX_SLOTS; + + mutex_lock(&bt->lock); + if (!bt->running) + goto unlock; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 2) { + /* Preserve already completed diagnostics after abort, without posting + * any more buffers or ringing doorbells to the failed firmware. + */ + if (arm_auxiliary_rings && readl(bt->bar) == 2) + cen_aux_drain(bt, false); + goto protocol_error; + } + if (arm_auxiliary_rings && cen_aux_drain(bt, true)) + goto protocol_error; + if (arm_le_data_rings) { + tail = cen_tail(bt, 10); + if (tail >= CEN_LE_SLOTS) + goto protocol_error; + while (bt->le_rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_LE_RX + bt->le_rx_head * CEN_LE_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length > CEN_LE_FOOTER) + goto protocol_error; + bt->le_rx_packets++; + cen_capture_frame(bt, 10, descriptor + 16, length); + if (enable_acl && cen_receive_acl(bt, descriptor + 16, length)) + goto protocol_error; + bt->le_rx_head = (bt->le_rx_head + 1) % CEN_LE_SLOTS; + cen_le_rx_buffer(bt, bt->le_rx_post); + bt->le_rx_post = (bt->le_rx_post + 1) % CEN_LE_SLOTS; + cen_post(bt, 10, bt->le_rx_post); + } + budget = CEN_RX_SLOTS; + } + tail = cen_tail(bt, 7); + if (tail >= CEN_RX_SLOTS) + goto protocol_error; + while (bt->rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_RX + bt->rx_head * CEN_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length < HCI_EVENT_HDR_SIZE || length > CEN_FOOTER || + length != descriptor[17] + HCI_EVENT_HDR_SIZE) + goto protocol_error; + skb = bt_skb_alloc(length, GFP_KERNEL); + if (skb) { + skb_put_data(skb, descriptor + 16, length); + /* The firmware's status belongs to the standard connection + * request tracked by Linux. Capture the native response before + * restoring only its opcode; preserve status and command credit. + */ + if (bt->hrb_status_pending && length == 6 && + skb->data[0] == HCI_EV_CMD_STATUS && + get_unaligned_le16(skb->data + 4) == CEN_OP_HRB_CREATE_CONN) { + cen_capture_frame(bt, 7, skb->data, length); + put_unaligned_le16(HCI_OP_CREATE_CONN, skb->data + 4); + bt->hrb_status_pending = false; + bt->hrb_status_events++; + } + /* Native HCI flow treats 0xf0/0xf1 Connection Complete as + * ACL, with additional proprietary capability flags. Linux + * otherwise cannot find its pending ACL connection by type. + * Preserve the original event privately before normalizing it. + */ + if (READ_ONCE(bt->hrb_supported) && + length == HCI_EVENT_HDR_SIZE + sizeof(struct hci_ev_conn_complete) && + skb->data[0] == HCI_EV_CONN_COMPLETE && + (skb->data[11] & 0xfe) == 0xf0) { + cen_capture_frame(bt, 7, skb->data, length); + bt->hrb_last_link_type = skb->data[11]; + skb->data[11] = ACL_LINK; + bt->hrb_connection_events++; + } + hci_skb_pkt_type(skb) = HCI_EVENT_PKT; + hdev->stat.byte_rx += length; + hci_recv_frame(hdev, skb); + } else { + hdev->stat.err_rx++; + } + bt->rx_head = (bt->rx_head + 1) % CEN_RX_SLOTS; + cen_rx_buffer(bt, bt->rx_post); + bt->rx_post = (bt->rx_post + 1) % CEN_RX_SLOTS; + cen_post(bt, 7, bt->rx_post); + } + if (cen_transmit(bt, &bt->txq, 1, CEN_TX_SLOTS, CEN_TX, + CEN_FOOTER, &bt->tx_head)) + goto protocol_error; + /* cen_aux_specs[0] is the native BR/EDR ACL TX resource (ring 2). */ + if (enable_acl && + (cen_transmit(bt, &bt->acl_txq, 2, cen_aux_specs[0].slots, + bt->aux[0].offset, cen_aux_specs[0].footer, + &bt->acl_tx_head) || + cen_transmit(bt, &bt->le_txq, 4, CEN_LE_SLOTS, CEN_LE_TX, + CEN_LE_FOOTER, &bt->le_tx_head))) + goto protocol_error; + schedule_delayed_work(&bt->poll, msecs_to_jiffies(1)); + goto unlock; +protocol_error: + bt->running = false; + hdev->stat.err_rx++; + dev_err(&bt->pdev->dev, "Beta transport stopped (exec=%u ipc=%u); polling disabled\n", + readl(bt->bar + 0x18), readl(bt->bar)); +unlock: + mutex_unlock(&bt->lock); +} + +static int cen_hci_open(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = true; + schedule_delayed_work(&bt->poll, 0); + mutex_unlock(&bt->lock); + return 0; +} + +static int cen_hci_close(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = false; + bt->hrb_status_pending = false; + mutex_unlock(&bt->lock); + cancel_delayed_work_sync(&bt->poll); + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_vendor_command(struct hci_dev *hdev, u16 opcode, + const void *data, u8 length) +{ + struct sk_buff *skb; + int ret; + + skb = __hci_cmd_sync(hdev, opcode, length, data, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len < 1 ? -EPROTO : -bt_to_errno(skb->data[0]); + kfree_skb(skb); + return ret; +} + +static int cen_hci_post_init(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff *skb; + u32 features; + int ret; + + if (!native_hrb) + return 0; + WRITE_ONCE(bt->hrb_supported, false); + skb = __hci_cmd_sync(hdev, CEN_OP_READ_PROP_FEATURES, 0, NULL, + HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len == 5 ? -bt_to_errno(skb->data[0]) : -EPROTO; + features = skb->len == 5 ? get_unaligned_le32(skb->data + 1) : 0; + kfree_skb(skb); + WRITE_ONCE(bt->proprietary_features, features); + if (ret) + return ret; + if (!(features & CEN_PROP_HRB_CREATE)) + return -EOPNOTSUPP; + WRITE_ONCE(bt->hrb_supported, true); + bt_dev_info(hdev, "Native Proxima Create Connection capability confirmed"); + return 0; +} + +static int cen_hci_setup(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + u8 address[6], init_done[16], seed[24]; + int ret; + + if (!native_startup) + return 0; + + /* Native PROXIMA initialization starts with these three commands, then + * HCI Reset. The core calls setup before its ordinary reset/init sequence. + * Address provisioning is still diagnostic: use a temporary locally + * administered address, in native WriteBdAddr display/network byte order. + */ + ret = get_random_bytes_wait(address, sizeof(address)); + if (ret) + return ret; + address[0] = (address[0] & 0xfc) | 0x02; + get_random_bytes(seed, sizeof(seed)); + memset(init_done, 0xff, sizeof(init_done)); + ret = cen_vendor_command(hdev, 0xfc06, address, sizeof(address)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 1); + ret = cen_vendor_command(hdev, 0xff0b, init_done, sizeof(init_done)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 2); + ret = cen_vendor_command(hdev, 0xfe44, seed, sizeof(seed)); + if (!ret) + WRITE_ONCE(bt->native_startup_steps, 3); +out: + memzero_explicit(seed, sizeof(seed)); + if (ret) + bt_dev_err(hdev, "Native startup stopped after %u commands: %d", + bt->native_startup_steps, ret); + else + bt_dev_info(hdev, "Native startup accepted; temporary diagnostic address in use"); + return ret; +} + +static int cen_hci_flush(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_hci_send(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff_head *queue = &bt->txq; + struct hci_conn *conn; + u16 handle; + u8 link_type; + int ret = 0; + + if (hci_skb_pkt_type(skb) == HCI_ACLDATA_PKT && enable_acl) { + if (skb->len < HCI_ACL_HDR_SIZE || + skb->len != HCI_ACL_HDR_SIZE + get_unaligned_le16(skb->data + 2)) { + ret = -EMSGSIZE; + goto out; + } + handle = get_unaligned_le16(skb->data); + rcu_read_lock(); + conn = hci_conn_hash_lookup_handle(hdev, hci_handle(handle)); + link_type = conn ? conn->type : 0xff; + rcu_read_unlock(); + if (link_type == LE_LINK && (handle & 0x3000) != 0x3000) { + queue = &bt->le_txq; + if (skb->len > CEN_LE_FOOTER) + ret = -EMSGSIZE; + } else if (link_type == ACL_LINK) { + queue = &bt->acl_txq; + if (skb->len > cen_aux_specs[0].footer) + ret = -EMSGSIZE; + } else { + ret = -EOPNOTSUPP; + } + if (ret) + goto out; + } else if (hci_skb_pkt_type(skb) != HCI_COMMAND_PKT) { + ret = -EOPNOTSUPP; + goto out; + } else if (skb->len < HCI_COMMAND_HDR_SIZE || skb->len > CEN_FOOTER || + skb->len != HCI_COMMAND_HDR_SIZE + skb->data[2]) { + ret = -EMSGSIZE; + goto out; + } + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT && native_hrb) { + u16 opcode = get_unaligned_le16(skb->data); + + /* Do not mix unmanaged native commands with translated requests. */ + if (opcode == CEN_OP_HRB_CREATE_CONN) { + ret = -EOPNOTSUPP; + goto out; + } + if (opcode == HCI_OP_CREATE_CONN && + skb->len != HCI_COMMAND_HDR_SIZE + sizeof(struct hci_cp_create_conn)) { + ret = -EMSGSIZE; + goto out; + } + } + mutex_lock(&bt->lock); + if (!bt->running) + ret = -ENETDOWN; + else if (skb_queue_len(queue) >= 64) + ret = -ENOBUFS; + else { + skb_queue_tail(queue, skb); + mod_delayed_work(system_wq, &bt->poll, 0); + } + mutex_unlock(&bt->lock); + if (!ret) + return 0; +out: + /* hci_send_frame() frees rejected packets. Ownership transfers to this + * driver only when the skb is queued and send returns success. + */ + return ret; +} + +static void cen_stop(struct cen_bt *bt) +{ + pci_clear_master(bt->pdev); + kvfree(bt->capture); + bt->capture = NULL; + /* IPC control 0 caused a firmware abort in the initial hardware probe. + * Until coordinated firmware teardown is understood, retain any published + * arena after detaching. A fresh firmware boot is required before rebinding. + */ + if (bt->ipc_requested || !pci_wait_for_pending_transaction(bt->pdev)) + dev_warn(&bt->pdev->dev, "Beta detach requires fresh firmware boot; retaining DMA arena\n"); + else + dma_free_coherent(&bt->pdev->dev, CEN_ARENA_SIZE, bt->arena, bt->dma); +} + +static int cen_status_show(struct seq_file *s, void *unused) +{ + struct cen_bt *bt = s->private; + unsigned int i; + + mutex_lock(&bt->lock); + seq_printf(s, "exec=%u ipc=%u running=%u tx_head=%u tx_consumed=%u rx_head=%u rx_consumed=%u queued=%u\n", + readl(bt->bar + 0x18), readl(bt->bar), bt->running, + bt->tx_head, cen_tail(bt, 1), bt->rx_head, cen_tail(bt, 7), + skb_queue_len(&bt->txq)); + seq_printf(s, "commands=%u events=%u tx_bytes=%u rx_bytes=%u tx_errors=%u rx_errors=%u\n", + bt->hdev->stat.cmd_tx, bt->hdev->stat.evt_rx, + bt->hdev->stat.byte_tx, bt->hdev->stat.byte_rx, + bt->hdev->stat.err_tx, bt->hdev->stat.err_rx); + seq_printf(s, "le_rings=%u le_rx_packets=%u\n", arm_le_data_rings, bt->le_rx_packets); + seq_printf(s, "native_startup=%u native_startup_steps=%u\n", + native_startup, READ_ONCE(bt->native_startup_steps)); + seq_printf(s, "native_hrb=%u hrb_supported=%u proprietary_features=0x%08x hrb_commands=%u hrb_status_events=%u hrb_pending=%u\n", + native_hrb, READ_ONCE(bt->hrb_supported), + READ_ONCE(bt->proprietary_features), bt->hrb_commands, + bt->hrb_status_events, bt->hrb_status_pending); + seq_printf(s, "hrb_connection_events=%u hrb_last_link_type=0x%02x\n", + bt->hrb_connection_events, bt->hrb_last_link_type); + seq_printf(s, "acl_enabled=%u acl_tx=%u acl_rx=%u acl_queued=%u le_queued=%u\n", + enable_acl, bt->hdev->stat.acl_tx, bt->hdev->stat.acl_rx, + skb_queue_len(&bt->acl_txq), skb_queue_len(&bt->le_txq)); + seq_printf(s, "auxiliary_rings=%u messages=%u message_head=%u capture_bytes=%u capture_records=%u capture_dropped=%u\n", + arm_auxiliary_rings, bt->message_tag, bt->message_head, + bt->capture_used, bt->capture_records, bt->capture_dropped); + if (arm_auxiliary_rings) + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) + if (cen_aux_specs[i].receive) + seq_printf(s, "aux_rx ring=%u packets=%u consumed=%u device_tail=%u\n", + cen_aux_specs[i].id, bt->aux[i].packets, + bt->aux[i].head, cen_tail(bt, cen_aux_specs[i].id)); + mutex_unlock(&bt->lock); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(cen_status); + +static ssize_t cen_capture_read(struct file *file, char __user *buffer, + size_t count, loff_t *pos) +{ + struct cen_bt *bt = file->private_data; + ssize_t ret; + + mutex_lock(&bt->lock); + ret = simple_read_from_buffer(buffer, count, pos, bt->capture, bt->capture_used); + mutex_unlock(&bt->lock); + return ret; +} + +static const struct file_operations cen_capture_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = cen_capture_read, + .llseek = default_llseek, +}; + +static ssize_t cen_capture_reset(struct file *file, const char __user *buffer, + size_t count, loff_t *pos) +{ + struct cen_bt *bt = file->private_data; + unsigned int value; + int ret = kstrtouint_from_user(buffer, count, 0, &value); + + if (ret || value != 1) + return ret ?: -EINVAL; + mutex_lock(&bt->lock); + bt->capture_used = 0; + bt->capture_records = 0; + bt->capture_dropped = 0; + mutex_unlock(&bt->lock); + return count; +} + +static const struct file_operations cen_capture_reset_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = cen_capture_reset, +}; + +static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct cen_bt *bt; + struct hci_dev *hdev; + int ret; + + if (!of_machine_is_compatible("apple,j813") || PCI_FUNC(pdev->devfn) != 2) + return -ENODEV; + bt = devm_kzalloc(&pdev->dev, sizeof(*bt), GFP_KERNEL); + if (!bt) + return -ENOMEM; + bt->pdev = pdev; + mutex_init(&bt->lock); + skb_queue_head_init(&bt->txq); + skb_queue_head_init(&bt->acl_txq); + skb_queue_head_init(&bt->le_txq); + if (enable_acl && (!arm_le_data_rings || !arm_auxiliary_rings)) + return -EINVAL; + if (native_hrb && (!native_startup || !enable_acl)) + return -EINVAL; + INIT_DELAYED_WORK(&bt->poll, cen_poll); + ret = pcim_enable_device(pdev); + if (ret) + return ret; + if (pci_resource_len(pdev, 0) != SZ_64K) + return -ENODEV; + ret = pcim_iomap_regions(pdev, BIT(0), "btcentauri"); + if (ret) + return ret; + bt->bar = pcim_iomap_table(pdev)[0]; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 0) + return dev_err_probe(&pdev->dev, -EBUSY, + "Beta requires running firmware and unowned IPC (exec=%u ipc=%u)\n", + readl(bt->bar + 0x18), readl(bt->bar)); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; + bt->arena = dma_alloc_coherent(&pdev->dev, CEN_ARENA_SIZE, &bt->dma, GFP_KERNEL); + if (!bt->arena) + return -ENOMEM; + memset(bt->arena, 0, CEN_ARENA_SIZE); + if (arm_auxiliary_rings) { + bt->capture = kvzalloc(CEN_CAPTURE_SIZE, GFP_KERNEL); + if (!bt->capture) { + ret = -ENOMEM; + goto stop; + } + } + ret = cen_start(bt); + if (ret) { + dev_err(&pdev->dev, "Beta IPC initialization failed: %d\n", ret); + goto stop; + } + hdev = hci_alloc_dev(); + if (!hdev) { + ret = -ENOMEM; + goto stop; + } + bt->hdev = hdev; + hdev->bus = HCI_PCI; + hdev->open = cen_hci_open; + hdev->close = cen_hci_close; + hdev->flush = cen_hci_flush; + hdev->send = cen_hci_send; + hdev->setup = cen_hci_setup; + hdev->post_init = cen_hci_post_init; + SET_HCIDEV_DEV(hdev, &pdev->dev); + hci_set_drvdata(hdev, bt); + pci_set_drvdata(pdev, bt); + ret = hci_register_dev(hdev); + if (ret) { + hci_free_dev(hdev); + goto stop; + } + bt->debug = debugfs_create_dir("btcentauri", NULL); + debugfs_create_file("status", 0400, bt->debug, bt, &cen_status_fops); + if (bt->capture) { + debugfs_create_file("auxiliary_capture", 0400, bt->debug, bt, &cen_capture_fops); + debugfs_create_file("capture_reset", 0200, bt->debug, bt, &cen_capture_reset_fops); + } + dev_info(&pdev->dev, "Beta HCI transport registered; ACL enabled=%u, SCO/ISO unsupported\n", + enable_acl); + return 0; +stop: + cen_stop(bt); + return ret; +} + +static void cen_remove(struct pci_dev *pdev) +{ + struct cen_bt *bt = pci_get_drvdata(pdev); + + debugfs_remove_recursive(bt->debug); + hci_unregister_dev(bt->hdev); + cen_hci_close(bt->hdev); + hci_free_dev(bt->hdev); + cen_stop(bt); +} + +static const struct pci_device_id cen_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903) }, + { } +}; +MODULE_DEVICE_TABLE(pci, cen_ids); + +static struct pci_driver cen_driver = { + .name = "btcentauri", + .id_table = cen_ids, + .probe = cen_probe, + .remove = cen_remove, +}; +module_pci_driver(cen_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 Beta HCI command/event transport for J813 bring-up"); diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c index 6eec46d831888..0db35959d114d 100644 --- a/drivers/net/wireless/apple/centauri/centauri_ipc.c +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -3,10 +3,12 @@ * Wire layout source: AppleCentauriCommon/scripts/alpha_wifi_scan.py v259. * Polling and bounded bootstrap rings are deliberate bring-up restrictions. */ +#include #include #include #include #include +#include #include #include #include @@ -79,6 +81,40 @@ static int ftab_entry(const void *image, size_t size, const char tag[4], return -ENOENT; } +/* The native secondary FTAB reserves a bounded "cras" region. Capture it + * after a firmware abort; these private diagnostics may contain device data. + */ +static ssize_t firmware_crash_read(struct file *file, char __user *buffer, + size_t count, loff_t *pos) +{ + struct centauri *c = file->private_data; + struct cen_buffer *secondary; + u32 offset, length; + ssize_t ret; + + mutex_lock(&c->boot_lock); + if (!c->ipc || !c->ipc->secondary) { + ret = -ENODEV; + goto unlock; + } + secondary = c->ipc->secondary; + ret = ftab_entry(secondary->cpu, secondary->size, "cras", &offset, &length, false); + if (ret) + goto unlock; + dma_rmb(); + ret = simple_read_from_buffer(buffer, count, pos, secondary->cpu + offset, length); +unlock: + mutex_unlock(&c->boot_lock); + return ret; +} + +static const struct file_operations firmware_crash_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = firmware_crash_read, + .llseek = default_llseek, +}; + static int secondary_init(struct centauri *c) { u32 offset, length, count, i, required; @@ -112,6 +148,7 @@ static int secondary_init(struct centauri *c) pci_write_config_dword(c->pdev, 0xf8c, upper_32_bits(c->ipc->secondary->dma)); pci_write_config_dword(c->pdev, 0xf90, required); dev_info(&c->pdev->dev, "secondary memory: seed=%u required=%u\n", length, required); + debugfs_create_file("firmware_crash", 0400, c->debug, c, &firmware_crash_fops); return 0; } From 83b16cf519eff7d3e2f34faa77541892984ecca4 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:14:39 -0500 Subject: [PATCH 040/117] centauri: add the J873 to the per-machine board table The J813 constants of the port cycle and the ROM boot word were hard-coded. Move them into one table with a row per machine that has an N1 and add the J873 (the M6 Mac mini): firmware file name, the word written to BAR+0x807c, the root port device ID, and the port, PHY and PERST# addresses with the one differing config +0x130 value, all taken from the machine's ADT. Measured on an M6 Mac mini with the rest of this series: the ROM stage, the port cycle and the Control firmware reach their running stage. Needs the T8152 SoC and PCIe nodes (base layer). [Thomas: the driver part of 70d215465d13 on the J813 line of linux-aurora] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Thomas Glanzmann --- drivers/bluetooth/btcentauri.c | 5 +-- .../net/wireless/apple/centauri/centauri.h | 36 ++++++++++++++++++- .../wireless/apple/centauri/centauri_pci.c | 17 ++++++--- .../wireless/apple/centauri/centauri_port.c | 28 ++++++++------- 4 files changed, 66 insertions(+), 20 deletions(-) diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c index d2539539fa162..2a250f2f0bd7c 100644 --- a/drivers/bluetooth/btcentauri.c +++ b/drivers/bluetooth/btcentauri.c @@ -874,7 +874,8 @@ static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) struct hci_dev *hdev; int ret; - if (!of_machine_is_compatible("apple,j813") || PCI_FUNC(pdev->devfn) != 2) + if (!(of_machine_is_compatible("apple,j813") || of_machine_is_compatible("apple,j873")) || + PCI_FUNC(pdev->devfn) != 2) return -ENODEV; bt = devm_kzalloc(&pdev->dev, sizeof(*bt), GFP_KERNEL); if (!bt) @@ -981,4 +982,4 @@ static struct pci_driver cen_driver = { }; module_pci_driver(cen_driver); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Apple N1 Beta HCI command/event transport for J813 bring-up"); +MODULE_DESCRIPTION("Apple N1 Beta HCI command/event transport for J813 and J873 bring-up"); diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h index f7fdf4105a3b3..6a67ec1db50fd 100644 --- a/drivers/net/wireless/apple/centauri/centauri.h +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -3,6 +3,7 @@ #define AURORA_CENTAURI_H #include #include +#include #include #include @@ -36,7 +37,40 @@ struct centauri { struct centauri_wifi *wifi; }; -int centauri_j813_port_cycle(struct pci_dev *pdev); +/* One row per machine with an N1: the FTAB file name, the word written to BAR+0x807c, + * (board-id << 16) | chip-id from the machine's ADT /chosen, and the physical addresses and + * the one differing register value of the port cycle (centauri_port.c). + * J813: board 0x24, chip 0x8142 (measured); port, PHY lane and PERST pad from the live ADT. + * J873 (Mac mini M6): board 0x24, chip 0x8152 from its ADT, the composition taken from the J813 + * word; port 0 config 0x388028000, lane 0x38f010000 and PERST# gpio0 pin 3 (0x31a000000 + 4 * 3) + * from its ADT; config +0x130 is 0x208 as in the m1n1 T8152 bring-up that links the port. + */ +struct centauri_board { + const char *compatible; + const char *firmware; + u32 board_word; + u64 port, phy, pad; + u32 tune_130; + u16 bridge_id; /* root port device ID, 0 = any Apple root port */ +}; + +static inline const struct centauri_board *centauri_board(void) +{ + static const struct centauri_board boards[] = { + { "apple,j813", "apple/centauri/j813-ftab.bin", 0x00248142, + 0x488028000ULL, 0x48f010000ULL, 0x39a000300ULL, 0x03020000, 0x100c }, + { "apple,j873", "apple/centauri/j873-ftab.bin", 0x00248152, + 0x388028000ULL, 0x38f010000ULL, 0x31a00000cULL, 0x208, 0 }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(boards); i++) + if (of_machine_is_compatible(boards[i].compatible)) + return &boards[i]; + return NULL; +} + +int centauri_port_cycle(struct pci_dev *pdev); int centauri_start_ipc(struct centauri *c); void centauri_free_ipc(struct centauri *c); int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *resources); diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c index f3f7c6413f0a3..d2b783a0b335b 100644 --- a/drivers/net/wireless/apple/centauri/centauri_pci.c +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -20,6 +20,10 @@ static struct dentry *centauri_debug; +static unsigned int board_word; +module_param(board_word, uint, 0644); +MODULE_PARM_DESC(board_word, "Diagnostic: word written to BAR0+0x807c before the ROM boot, 0 = per-machine table value"); + static int centauri_status_show(struct seq_file *s, void *unused) { struct centauri *c = s->private; @@ -64,19 +68,19 @@ static void centauri_free_boot(struct centauri *c) static int centauri_rom_boot(struct centauri *c) { const struct firmware *fw; + const struct centauri_board *board = centauri_board(); struct device *dev = &c->pdev->dev; u32 stage; int ret; - if ((c->pdev->device != 0x1900 && c->pdev->device != 0x1901) || - !of_machine_is_compatible("apple,j813")) + if ((c->pdev->device != 0x1900 && c->pdev->device != 0x1901) || !board) return -ENODEV; if (!c->link_ready || c->boot_attempted) return -EBUSY; stage = readl(c->bar + 0x8000); if (stage != 0 && stage != 4 && stage != 1) return -EBUSY; - ret = request_firmware(&fw, "apple/centauri/j813-ftab.bin", dev); + ret = request_firmware(&fw, board->firmware, dev); if (ret) return ret; if (fw->size < 0x30 || fw->size > SZ_64M || @@ -116,7 +120,9 @@ static int centauri_rom_boot(struct centauri *c) pci_write_config_word(c->pdev, c->msi_cap + PCI_MSI_FLAGS, (c->msi_original & ~PCI_MSI_FLAGS_QSIZE) | PCI_MSI_FLAGS_ENABLE); pci_set_master(c->pdev); - writel(0x00248142, c->bar + 0x807c); /* J813 board 0x24, T8142 chip. */ + dev_info(dev, "ROM boot: %s, %zu bytes, board word %#x\n", board->firmware, fw->size, + board_word ?: board->board_word); + writel(board_word ?: board->board_word, c->bar + 0x807c); writel(0, c->bar + 0x803c); writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); @@ -168,7 +174,7 @@ static ssize_t centauri_boot_write(struct file *file, const char __user *buffer, ret = -EBUSY; } else { c->link_ready = false; - ret = centauri_j813_port_cycle(c->pdev); + ret = centauri_port_cycle(c->pdev); if (!ret) { c->link_ready = true; dev_info(&c->pdev->dev, "post-reset execution stage=%#x\n", @@ -276,3 +282,4 @@ module_exit(centauri_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Apple N1 Centauri native PCI and ACIPC bring-up backend"); MODULE_FIRMWARE("apple/centauri/j813-ftab.bin"); +MODULE_FIRMWARE("apple/centauri/j873-ftab.bin"); diff --git a/drivers/net/wireless/apple/centauri/centauri_port.c b/drivers/net/wireless/apple/centauri/centauri_port.c index eea0a1ba11e0e..5b2b1f0bbb977 100644 --- a/drivers/net/wireless/apple/centauri/centauri_port.c +++ b/drivers/net/wireless/apple/centauri/centauri_port.c @@ -1,9 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only -/* J813 preboot -> boot port cycle, transcribed from the measured proxy +/* Preboot -> boot port cycle, transcribed from the J813 measured proxy * alpha_wifi_scan.py pcie_cycle/apple_teardown/apple_bringup sequence. - * These apertures were verified from the live ADT before this Linux boot. - * This temporary board helper belongs in the host-controller driver once - * T8142 platform/clock/GPIO bindings are available. + * Apertures and the PERST pad come from the per-machine table in centauri.h; those were + * verified from the live ADT. The J873 row reuses the J813 sequence with T8152 addresses and + * config +0x130 = 0x208: the reset table and the lane window bits equal the T8152 bring-up that + * links port 0 in m1n1, but the cycle itself (link down, retrain with firmware running) has not + * been run on a J873. This temporary board helper belongs in the host-controller driver. */ #include #include @@ -24,7 +26,7 @@ static bool wait_bits(void __iomem *address, u32 mask, u32 wanted, u32 us) return !readl_poll_timeout(address, value, (value & mask) == wanted, 100, us); } -int centauri_j813_port_cycle(struct pci_dev *pdev) +int centauri_port_cycle(struct pci_dev *pdev) { static const u32 reset[][2] = { {0x88, 0x110}, {0x100, ~0U}, {0x148, ~0U}, {0x210, ~0U}, @@ -32,10 +34,11 @@ int centauri_j813_port_cycle(struct pci_dev *pdev) {0x16c, 0}, {0x13c, 0}, {0x800, 0x00100100}, {0x808, 0x001001ff}, {0x82c, 0x00010000}, }; - static const u32 tune[][2] = { - {0x130, 0x03020000}, {0x140, 0x10}, {0x144, 0x00253770}, + u32 tune[][2] = { + {0x130, 0}, {0x140, 0x10}, {0x144, 0x00253770}, {0x21c, 0}, {0x834, 0}, {0x83c, 0}, }; + const struct centauri_board *board = centauri_board(); struct pci_dev *bridge = pdev->bus->self; void __iomem *port, *phy, *pad; u32 value, gpio, identity; @@ -44,12 +47,13 @@ int centauri_j813_port_cycle(struct pci_dev *pdev) int ret = -ETIMEDOUT, pm; bool ack, l2; - if (!of_machine_is_compatible("apple,j813") || !bridge || - bridge->vendor != PCI_VENDOR_ID_APPLE || bridge->device != 0x100c) + if (!board || !board->port || !bridge || bridge->vendor != PCI_VENDOR_ID_APPLE || + (board->bridge_id && bridge->device != board->bridge_id)) return -ENODEV; - port = ioremap(0x488028000ULL, 0x4000); - phy = ioremap(0x48f010000ULL, 0x1000); - pad = ioremap(0x39a000300ULL, sizeof(u32)); + tune[0][1] = board->tune_130; + port = ioremap(board->port, 0x4000); + phy = ioremap(board->phy, 0x1000); + pad = ioremap(board->pad, sizeof(u32)); if (!port || !phy || !pad) { ret = -ENOMEM; goto out; From df33b2efa9e6d10c222ef49b11f0e6ce5e0ecdfb Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:14:39 -0500 Subject: [PATCH 041/117] centauri: write the RID2SID and MSIMAP tables back after the port cycle The J813 port cycle zeroes 16 words at port+0x3000 and 256 words at port+0x3800. On the J873 those are the RID2SID and MSIMAP tables that pcie-apple programmed. The cycle then drops the stream entry of function 0 and its DMA arrives as SID 0 (DART "BPF reject"). A board with keep_port_tables set gets both tables written back after the cycle. Measured on an M6 Mac mini: with this change the Control firmware reaches its OS stage after the cycle. [Thomas: the driver part of fcbcd1529d03 on the J813 line of linux-aurora] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Thomas Glanzmann --- .../net/wireless/apple/centauri/centauri.h | 9 +++++++-- .../wireless/apple/centauri/centauri_port.c | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h index 6a67ec1db50fd..9348c00984ccc 100644 --- a/drivers/net/wireless/apple/centauri/centauri.h +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -44,6 +44,10 @@ struct centauri { * J873 (Mac mini M6): board 0x24, chip 0x8152 from its ADT, the composition taken from the J813 * word; port 0 config 0x388028000, lane 0x38f010000 and PERST# gpio0 pin 3 (0x31a000000 + 4 * 3) * from its ADT; config +0x130 is 0x208 as in the m1n1 T8152 bring-up that links the port. + * keep_port_tables: the J813 sequence zeroes the 16 words at port+0x3000 and the 256 words at + * port+0x3800. On the J873 those are the RID2SID and MSIMAP tables that pcie-apple programmed + * (PORT_T602X_RID2SID, PORT_T602X_MSIMAP); the cycle then drops function 0's stream entry and + * its DMA arrives as SID 0 (DART "BPF reject"). Such a board gets both tables written back. */ struct centauri_board { const char *compatible; @@ -52,15 +56,16 @@ struct centauri_board { u64 port, phy, pad; u32 tune_130; u16 bridge_id; /* root port device ID, 0 = any Apple root port */ + bool keep_port_tables; }; static inline const struct centauri_board *centauri_board(void) { static const struct centauri_board boards[] = { { "apple,j813", "apple/centauri/j813-ftab.bin", 0x00248142, - 0x488028000ULL, 0x48f010000ULL, 0x39a000300ULL, 0x03020000, 0x100c }, + 0x488028000ULL, 0x48f010000ULL, 0x39a000300ULL, 0x03020000, 0x100c, false }, { "apple,j873", "apple/centauri/j873-ftab.bin", 0x00248152, - 0x388028000ULL, 0x38f010000ULL, 0x31a00000cULL, 0x208, 0 }, + 0x388028000ULL, 0x38f010000ULL, 0x31a00000cULL, 0x208, 0, true }, }; int i; diff --git a/drivers/net/wireless/apple/centauri/centauri_port.c b/drivers/net/wireless/apple/centauri/centauri_port.c index 5b2b1f0bbb977..62a9456e74ea9 100644 --- a/drivers/net/wireless/apple/centauri/centauri_port.c +++ b/drivers/net/wireless/apple/centauri/centauri_port.c @@ -46,6 +46,8 @@ int centauri_port_cycle(struct pci_dev *pdev) unsigned int lanes, i; int ret = -ETIMEDOUT, pm; bool ack, l2; + u32 rid2sid[16], msimap[256], valid = 0; + bool saved = false; if (!board || !board->port || !bridge || bridge->vendor != PCI_VENDOR_ID_APPLE || (board->bridge_id && bridge->device != board->bridge_id)) @@ -69,6 +71,13 @@ int centauri_port_cycle(struct pci_dev *pdev) ret = pci_save_state(pdev); if (ret) goto out; + if (board->keep_port_tables) { + for (i = 0; i < ARRAY_SIZE(rid2sid); i++) + rid2sid[i] = readl(port + 0x3000 + 4 * i); + for (i = 0; i < ARRAY_SIZE(msimap); i++) + msimap[i] = readl(port + 0x3800 + 4 * i); + saved = true; + } pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &link); lanes = (link & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT; lanes = lanes >= 1 && lanes <= 16 ? BIT(lanes) - 1 : 1; @@ -186,6 +195,16 @@ int centauri_port_cycle(struct pci_dev *pdev) timed_out: ret = -ETIMEDOUT; out: + if (saved) { + for (i = 0; i < ARRAY_SIZE(rid2sid); i++) { + writel(rid2sid[i], port + 0x3000 + 4 * i); + valid += !!(rid2sid[i] & BIT(31)); + } + for (i = 0; i < ARRAY_SIZE(msimap); i++) + writel(msimap[i], port + 0x3800 + 4 * i); + dev_info(&pdev->dev, "restored %u RID2SID entries and the MSI map (ret %d)\n", + valid, ret); + } if (pad) iounmap(pad); if (phy) From 9b29bfbc545a6d0d7034bb86782e586c40a5cb20 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:14:39 -0500 Subject: [PATCH 042/117] centauri: map IPC buffers through the struct device of their function Each function of the N1 has its own IOMMU stream on the J873 (RID2SID entries 1, 2 and 3 and separate DART domains). A buffer that the firmware of a function reads or writes must be mapped through that function's own struct device. Function 0 is Control, function 1 Alpha (Wi-Fi) and function 2 Beta (Bluetooth). Measured on an M6 Mac mini: Control and Alpha complete their IPC handshakes with this change. [Thomas: the driver part of 12a5fd0d8fc0 on the J813 line of linux-aurora] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Thomas Glanzmann --- .../wireless/apple/centauri/centauri_ipc.c | 58 +++++++++++++------ 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c index 0db35959d114d..5f88f9a4390c9 100644 --- a/drivers/net/wireless/apple/centauri/centauri_ipc.c +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -16,6 +16,7 @@ struct cen_buffer { struct list_head list; + struct device *dev; void *cpu; dma_addr_t dma; size_t size; @@ -34,22 +35,35 @@ struct centauri_ipc { struct cen_context control, alpha; struct cen_buffer *secondary, *window, *cchi_rx, *cchi_tx, *debug_rx; struct cen_buffer *alpha_log; + struct cen_buffer *msi[3]; u16 cchi_head, cchi_tail; }; -static struct cen_buffer *buffer_alloc(struct centauri *c, size_t size) +/* Each function has its own IOMMU stream on the J873 (RID2SID entries 1, 2, 3 and separate DART + * domains), so a buffer a function's firmware reads or writes has to be mapped through that + * function's own struct device. Function 0 is the Control firmware, 1 Alpha (Wi-Fi), 2 Beta. + */ +static struct cen_buffer *buffer_alloc(struct centauri *c, unsigned int function, size_t size) { struct cen_buffer *buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + struct pci_dev *owner = function ? c->ipc->function[function] : c->pdev; if (!buffer) return NULL; + if (!owner) { + kfree(buffer); + return NULL; + } + buffer->dev = &owner->dev; buffer->size = PAGE_ALIGN(size); - buffer->cpu = dma_alloc_coherent(&c->pdev->dev, buffer->size, &buffer->dma, GFP_KERNEL); + buffer->cpu = dma_alloc_coherent(buffer->dev, buffer->size, &buffer->dma, GFP_KERNEL); if (!buffer->cpu) { kfree(buffer); return NULL; } memset(buffer->cpu, 0, buffer->size); + dev_dbg(&c->pdev->dev, "dma buffer fn%u %#llx..%#llx (%zu bytes)\n", function, + (u64)buffer->dma, (u64)buffer->dma + buffer->size, buffer->size); list_add_tail(&buffer->list, &c->ipc->buffers); return buffer; } @@ -139,7 +153,7 @@ static int secondary_init(struct centauri *c) return -EINVAL; required = max_t(u32, required, end); } - c->ipc->secondary = buffer_alloc(c, required); + c->ipc->secondary = buffer_alloc(c, 0, required); if (!c->ipc->secondary) return -ENOMEM; memcpy(c->ipc->secondary->cpu, seed, length); @@ -152,15 +166,15 @@ static int secondary_init(struct centauri *c) return 0; } -static int context_init(struct centauri *c, struct cen_context *ctx, +static int context_init(struct centauri *c, unsigned int function, struct cen_context *ctx, u16 tr, u16 cr, dma_addr_t scratch, size_t scratch_size) { u8 *header; dma_addr_t dma; int i; - ctx->buffer = buffer_alloc(c, SZ_32K); - ctx->messages = buffer_alloc(c, SZ_16K); + ctx->buffer = buffer_alloc(c, function, SZ_32K); + ctx->messages = buffer_alloc(c, function, SZ_16K); if (!ctx->buffer || !ctx->messages) return -ENOMEM; ctx->tr_count = tr; @@ -256,9 +270,9 @@ static int cchi_init(struct centauri *c) u16 tail; int i, ret; - ipc->cchi_rx = buffer_alloc(c, SZ_32K); - ipc->cchi_tx = buffer_alloc(c, SZ_32K); - ipc->debug_rx = buffer_alloc(c, SZ_16K); + ipc->cchi_rx = buffer_alloc(c, 0, SZ_32K); + ipc->cchi_tx = buffer_alloc(c, 0, SZ_32K); + ipc->debug_rx = buffer_alloc(c, 0, SZ_16K); if (!ipc->cchi_rx || !ipc->cchi_tx || !ipc->debug_rx) return -ENOMEM; ret = message(&ipc->control, c->bar + 0x9000, config, sizeof(config)); @@ -311,7 +325,7 @@ static int memswap_handoff(struct centauri *c, unsigned int function, const char ret = ftab_entry(secondary->cpu, secondary->size, tag, &offset, &size, false); if (ret) return ret; - working = buffer_alloc(c, size); + working = buffer_alloc(c, function, size); if (!working) return -ENOMEM; dma_rmb(); @@ -329,6 +343,7 @@ static int enumerate_functions(struct centauri *c) struct centauri_ipc *ipc = c->ipc; u16 control; int i, cap; + dma_addr_t msi; pci_lock_rescan_remove(); /* The ROM personality was single-function; boot exposes Alpha and Beta. */ @@ -352,10 +367,16 @@ static int enumerate_functions(struct centauri *c) pci_read_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, &control); if (!(control & PCI_MSI_FLAGS_64BIT)) return -EOPNOTSUPP; + if (i) { + ipc->msi[i] = buffer_alloc(c, i, SZ_16K); + if (!ipc->msi[i]) + return -ENOMEM; + } + msi = i ? ipc->msi[i]->dma : c->msi_dma; pci_write_config_dword(ipc->function[i], cap + PCI_MSI_ADDRESS_LO, - lower_32_bits(c->msi_dma + 0x400 * i)); + lower_32_bits(msi)); pci_write_config_dword(ipc->function[i], cap + PCI_MSI_ADDRESS_HI, - upper_32_bits(c->msi_dma + 0x400 * i)); + upper_32_bits(msi)); pci_write_config_word(ipc->function[i], cap + PCI_MSI_DATA_64, 0xa5a0 + 0x100 * i); pci_write_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, (control & ~PCI_MSI_FLAGS_QSIZE) | (5 << 4) | PCI_MSI_FLAGS_ENABLE); @@ -383,7 +404,7 @@ int centauri_start_ipc(struct centauri *c) ret = secondary_init(c); if (ret) goto failed; - ret = context_init(c, &ipc->control, 13, 1, 0, 0); + ret = context_init(c, 0, &ipc->control, 13, 1, 0, 0); if (ret) goto failed; writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); @@ -417,13 +438,14 @@ int centauri_start_ipc(struct centauri *c) ret = ret ?: -EIO; goto failed; } - ipc->window = buffer_alloc(c, SZ_16M); + ipc->window = buffer_alloc(c, 1, SZ_16M); if (!ipc->window) { ret = -ENOMEM; goto failed; } - ret = context_init(c, &ipc->alpha, 192, 234, - ipc->window->dma + CEN_ALPHA_SCRATCH_OFFSET, CEN_ALPHA_SCRATCH_SIZE); + ret = context_init(c, 1, &ipc->alpha, 192, 234, + ipc->window->dma + CEN_ALPHA_SCRATCH_OFFSET, + CEN_ALPHA_SCRATCH_SIZE); if (ret) goto failed; writel(lower_32_bits(ipc->window->dma), ipc->bar[1] + 0x18); @@ -463,7 +485,7 @@ void centauri_free_ipc(struct centauri *c) pci_clear_master(ipc->function[i]); } list_for_each_entry_safe(buffer, next, &ipc->buffers, list) { - dma_free_coherent(&c->pdev->dev, buffer->size, buffer->cpu, buffer->dma); + dma_free_coherent(buffer->dev, buffer->size, buffer->cpu, buffer->dma); list_del(&buffer->list); kfree(buffer); } @@ -505,7 +527,7 @@ int centauri_alpha_log_buffer(struct centauri *c, void **cpu, dma_addr_t *dma, s if (!ipc || !ipc->window) return -ENODEV; if (!ipc->alpha_log) - ipc->alpha_log = buffer_alloc(c, SZ_4M); + ipc->alpha_log = buffer_alloc(c, 1, SZ_4M); if (!ipc->alpha_log) return -ENOMEM; *cpu = ipc->alpha_log->cpu; From 9b4c0c4a4d05f54ee5c34ae22c7d3584db1ab9f2 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:14:39 -0500 Subject: [PATCH 043/117] centauri: leave the Bluetooth function to btcentauri Function 0x1903 is the Beta (Bluetooth) function and btcentauri claims it. apple_centauri needs only the functions 0x1900, 0x1901 and 0x1902, so stop it from claiming 0x1903. Measured on an M6 Mac mini: btcentauri binds 0x1903 and registers its HCI transport. [Thomas: the driver part of 0f318aebe42f on the J813 line of linux-aurora] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Thomas Glanzmann --- drivers/net/wireless/apple/centauri/centauri_pci.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c index d2b783a0b335b..23fff2160a533 100644 --- a/drivers/net/wireless/apple/centauri/centauri_pci.c +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -245,11 +245,14 @@ static void centauri_remove(struct pci_dev *pdev) centauri_free_boot(c); } +/* 0x1903 is the Beta (Bluetooth) function: btcentauri claims it. This driver needs functions + * 0 and 1 (1 is enabled here, its BAR is used by the Wi-Fi code) and reaches function 2 only + * through config space and its struct device for DMA. + */ static const struct pci_device_id centauri_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1900), .driver_data = 0x8000 }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1901), .driver_data = 0x8000 }, { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1902), .driver_data = 0x0008 }, - { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903), .driver_data = 0x0018 }, { } }; MODULE_DEVICE_TABLE(pci, centauri_ids); From 103e321802cb215a2ea8552b8ce1c6bb2f43dd08 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:14:39 -0500 Subject: [PATCH 044/117] btcentauri: report N1 legacy scan responses with event type 0x1a The N1 reports legacy scan responses in LE Extended Advertising Reports whose event type has the low byte 0x18 (legacy PDU and scan response, neither connectable nor scannable) and other bits in the high byte (seen: 0x2518, 0x2618, 0x2718; accepted legacy types arrive as 0x2613 and 0x2610). The core accepts only 0x1a and 0x1b for a legacy scan response and drops the others as "Unknown advertising packet type". Report the low byte 0x18 as 0x1a and keep the high byte. Measured on a Mac mini M6 (J873), 15 s LE discovery with btmgmt (LE on, power on, find), native startup and the LE and auxiliary rings armed: - without the change: 72 to 74 reports, 71 distinct addresses, 12 to 13 "Unknown advertising packet type: 0x18" kernel lines per run; - with the change: 89 reports, 83 distinct addresses, 0 such lines; btmon shows the scan responses as event types 0x251a, 0x261a and 0x271a. A first version of this patch compared the whole 16-bit type with 0x0018 and never matched; the high byte has to be ignored. Not measured: device names or scan response data content, connection, pairing, audio. [Thomas: the driver part of d3d8dab9dbf5 on the J813 line of linux-aurora] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Thomas Glanzmann --- drivers/bluetooth/btcentauri.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c index 2a250f2f0bd7c..0bd8fedf3c14e 100644 --- a/drivers/bluetooth/btcentauri.c +++ b/drivers/bluetooth/btcentauri.c @@ -463,6 +463,32 @@ static int cen_transmit(struct cen_bt *bt, struct sk_buff_head *queue, return 0; } +/* The N1 reports legacy scan responses in LE Extended Advertising Reports with an event type whose low + * byte is 0x18 (legacy PDU and scan response, neither connectable nor scannable) and other bits in the + * high byte (seen: 0x2518, 0x2618, 0x2718). Linux accepts only the Core values 0x1a and 0x1b for a + * legacy scan response and drops anything else as "Unknown advertising packet type", so report the + * low byte 0x18 as 0x1a and keep the high byte. + */ +static void cen_normalize_ext_adv(struct sk_buff *skb) +{ + unsigned int offset = HCI_EVENT_HDR_SIZE + 2, reports; + + if (skb->len < offset + 1 || skb->data[0] != HCI_EV_LE_META || + skb->data[2] != HCI_EV_LE_EXT_ADV_REPORT) + return; + reports = skb->data[3]; + while (reports--) { + struct hci_ev_le_ext_adv_info *info; + + if (offset + sizeof(*info) > skb->len) + return; + info = (struct hci_ev_le_ext_adv_info *)(skb->data + offset); + if ((le16_to_cpu(info->type) & 0x00ff) == 0x0018) + info->type = cpu_to_le16((le16_to_cpu(info->type) & 0xff00) | 0x001a); + offset += sizeof(*info) + info->length; + } +} + static void cen_poll(struct work_struct *work) { struct cen_bt *bt = container_of(to_delayed_work(work), struct cen_bt, poll); @@ -546,6 +572,7 @@ static void cen_poll(struct work_struct *work) skb->data[11] = ACL_LINK; bt->hrb_connection_events++; } + cen_normalize_ext_adv(skb); hci_skb_pkt_type(skb) = HCI_EVENT_PKT; hdev->stat.byte_rx += length; hci_recv_frame(hdev, skb); From 7e9f42cb2f481e9fc8cdb650ee2696daa8e52da6 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:14:39 -0500 Subject: [PATCH 045/117] centauri: use the macOS 27 scan TLVs and link TLV 386 on the J873 The macOS 27.0 WLAN adapter differs from the 26.6.1 one the J813 recipe came from. startInterface carries the three link addresses in TLV 386 (was 385). The scan request (command 0x1000f) carries a channel-list TLV 0x104 and an OCE TLV 0x165 after the fixed 14 bytes. Both are selected per board. Measured on an M6 Mac mini: start and scan commands are accepted. Whether the TLVs are required is untested. [Thomas: the driver part of 4061bf02639e on the J813 line of linux-aurora] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Thomas Glanzmann --- .../net/wireless/apple/centauri/centauri.h | 12 ++++- .../wireless/apple/centauri/centauri_wifi.c | 49 ++++++++++++++++--- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h index 9348c00984ccc..a6aae0c4c10c1 100644 --- a/drivers/net/wireless/apple/centauri/centauri.h +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -48,6 +48,10 @@ struct centauri { * port+0x3800. On the J873 those are the RID2SID and MSIMAP tables that pcie-apple programmed * (PORT_T602X_RID2SID, PORT_T602X_MSIMAP); the cycle then drops function 0's stream entry and * its DMA arrives as SID 0 (DART "BPF reject"). Such a board gets both tables written back. + * link_tlv, scan_tlvs: the macOS 27.0 WLAN adapter differs from the 26.6.1 one the J813 recipe + * came from. startInterface carries the three link addresses in TLV 386 (was 385), and the scan + * request (command 0x1000f) carries a channel-list TLV 0x104 and an OCE TLV 0x165 after the fixed + * 14 bytes. A J873 scan without them crashed the UMAC in SCHED_Req_Add. */ struct centauri_board { const char *compatible; @@ -57,15 +61,19 @@ struct centauri_board { u32 tune_130; u16 bridge_id; /* root port device ID, 0 = any Apple root port */ bool keep_port_tables; + u16 link_tlv; + bool scan_tlvs; }; static inline const struct centauri_board *centauri_board(void) { static const struct centauri_board boards[] = { { "apple,j813", "apple/centauri/j813-ftab.bin", 0x00248142, - 0x488028000ULL, 0x48f010000ULL, 0x39a000300ULL, 0x03020000, 0x100c, false }, + 0x488028000ULL, 0x48f010000ULL, 0x39a000300ULL, 0x03020000, 0x100c, false, + 385, false }, { "apple,j873", "apple/centauri/j873-ftab.bin", 0x00248152, - 0x388028000ULL, 0x38f010000ULL, 0x31a00000cULL, 0x208, 0, true }, + 0x388028000ULL, 0x38f010000ULL, 0x31a00000cULL, 0x208, 0, true, + 386, true }, }; int i; diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index 21201241f49ec..86a119dea0ea9 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -759,13 +759,15 @@ static int radio_start(struct centauri_wifi *wifi) unsigned int ring, slot; int ret; - /* Stock 25G76 startInterface (0x1808e26ec) supplies TLV 385, - * length 18. Firmware 0x5736c0 otherwise generates three unrelated + /* The stock 26.x startInterface supplies TLV 385, length 18; the macOS 27.0 + * adapter uses 386 (board table). Firmware 0x5736c0 otherwise generates three unrelated * random addresses. This diagnostic keeps the sole active link equal * to the netdev identity; it does not enable multi-link association. */ if (station_link_addresses) { - put_unaligned_le16(385, start + 5); + const struct centauri_board *board = centauri_board(); + + put_unaligned_le16(board ? board->link_tlv : 385, start + 5); put_unaligned_le16(3 * ETH_ALEN, start + 7); for (slot = 0; slot < 3; slot++) ether_addr_copy(start + 9 + slot * ETH_ALEN, wifi->netdev->dev_addr); @@ -799,12 +801,47 @@ static int radio_start(struct centauri_wifi *wifi) return 0; } +static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, u16 length); + +/* Fixed scan request (command 0x1000f), 14 bytes, chunk number first. */ +static size_t build_scan(struct centauri_wifi *wifi, unsigned int chunk, u8 *buf) +{ + static const u8 fixed[14] = { 1, 0, 0, 0x80, 0, 2, 0, 0x10, 110, 0, 110, 0, 0, 0 }; + const struct centauri_board *board = centauri_board(); + u8 list[1 + 4 * 13], oce[5] = { 0, 0, 1, 0, 0 }; + unsigned int first = chunk * 13, n, i; + size_t size = sizeof(fixed); + + memcpy(buf, fixed, sizeof(fixed)); + buf[0] = chunk + 1; + if (!board || !board->scan_tlvs) + return size; + /* The macOS 27.0 scan request appends TLV 0x104, a count and then + * {channel, band (0 = 2.4 GHz, 1 = 5 GHz), 0, 0} per channel, and TLV 0x165, an OCE block + * whose flags byte always has bit 0 set. One chunk covers 13 entries of the wiphy table. + */ + n = min_t(unsigned int, 13, 13 + ARRAY_SIZE(five_ghz) - first); + list[0] = n; + for (i = 0; i < n; i++) { + const struct ieee80211_channel *channel = &wifi->channels[first + i]; + + list[1 + 4 * i] = channel->hw_value; + list[2 + 4 * i] = channel->band == NL80211_BAND_5GHZ; + list[3 + 4 * i] = 0; + list[4 + 4 * i] = 0; + } + append_tlv(buf, &size, 0x104, list, 1 + 4 * n); + append_tlv(buf, &size, 0x165, oce, sizeof(oce)); + return size; +} + static void scan_worker(struct work_struct *work) { struct centauri_wifi *wifi = container_of(work, struct centauri_wifi, scan_work); struct cfg80211_scan_info info = {}; struct cfg80211_scan_request *request; - u8 scan[14] = { 1, 0, 0, 0x80, 0, 2, 0, 0x10, 110, 0, 110, 0, 0, 0 }; + u8 scan[14 + 4 + 1 + 4 * 13 + 4 + 5]; + size_t scan_size; unsigned int chunk; int ret = 0; @@ -814,8 +851,8 @@ static void scan_worker(struct work_struct *work) bool stopped = false; unsigned long deadline; - scan[0] = chunk + 1; - ret = command(wifi, 65551, scan, sizeof(scan)); + scan_size = build_scan(wifi, chunk, scan); + ret = command(wifi, 65551, scan, scan_size); if (ret) break; deadline = jiffies + msecs_to_jiffies(8000); From 4bb4ef09581effef2b0545b8eea8a9be0a52bac5 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:14:39 -0500 Subject: [PATCH 046/117] centauri: send a non-zero passive dwell in the scan request The Alpha firmware of the J873 asserts in SCHED_Req_Add when the scan state machine selects a dwell of zero. The scan request carries the active minimum and maximum dwell at frame +0x20 and +0x22 and the passive dwell at +0x24 (in ms, as macOS fills them from the scan request). The driver sent 0 as passive dwell. The whole firmware then trapped at the first scan with PHY, UMAC crash handler and SCHED_Req_Add assertions. Send 110 ms active and 110 ms passive. Also correct the comment in centauri.h that blamed the missing TLVs. Measured on an M6 Mac mini with the original firmware table and no calibration data: the scan returns rc=0 with 24 channels and 51 BSS (31 to 51 BSS in three runs). [Thomas: the driver part of 6d4959cc6edb on the J813 line of linux-aurora] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Thomas Glanzmann --- drivers/net/wireless/apple/centauri/centauri.h | 3 ++- drivers/net/wireless/apple/centauri/centauri_wifi.c | 13 ++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h index a6aae0c4c10c1..43256ac8ec40e 100644 --- a/drivers/net/wireless/apple/centauri/centauri.h +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -51,7 +51,8 @@ struct centauri { * link_tlv, scan_tlvs: the macOS 27.0 WLAN adapter differs from the 26.6.1 one the J813 recipe * came from. startInterface carries the three link addresses in TLV 386 (was 385), and the scan * request (command 0x1000f) carries a channel-list TLV 0x104 and an OCE TLV 0x165 after the fixed - * 14 bytes. A J873 scan without them crashed the UMAC in SCHED_Req_Add. + * 14 bytes. The J873 scan crash in SCHED_Req_Add came from a zero passive dwell in those 14 bytes + * (see build_scan), not from the missing TLVs; whether the TLVs are required is untested. */ struct centauri_board { const char *compatible; diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index 86a119dea0ea9..978f314973317 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -803,15 +803,26 @@ static int radio_start(struct centauri_wifi *wifi) static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, u16 length); +/* Dwell times in ms for the scan request. The macOS 27 scan adapter copies + * the first request dwell field into frame +0x20 and +0x22 and the second into +0x24 and logs + * "dwelltime active[%d, %d] passive %d". A zero passive dwell makes the J873 scan state machine + * assert in SCHED_Req_Add (the dwell it selects must not be zero) and the whole firmware traps. + */ +#define SCAN_ACTIVE_DWELL_MS 110 +#define SCAN_PASSIVE_DWELL_MS 110 + /* Fixed scan request (command 0x1000f), 14 bytes, chunk number first. */ static size_t build_scan(struct centauri_wifi *wifi, unsigned int chunk, u8 *buf) { - static const u8 fixed[14] = { 1, 0, 0, 0x80, 0, 2, 0, 0x10, 110, 0, 110, 0, 0, 0 }; + u8 fixed[14] = { 1, 0, 0, 0x80, 0, 2, 0, 0x10 }; const struct centauri_board *board = centauri_board(); u8 list[1 + 4 * 13], oce[5] = { 0, 0, 1, 0, 0 }; unsigned int first = chunk * 13, n, i; size_t size = sizeof(fixed); + put_unaligned_le16(SCAN_ACTIVE_DWELL_MS, fixed + 8); + put_unaligned_le16(SCAN_ACTIVE_DWELL_MS, fixed + 10); + put_unaligned_le16(SCAN_PASSIVE_DWELL_MS, fixed + 12); memcpy(buf, fixed, sizeof(fixed)); buf[0] = chunk + 1; if (!board || !board->scan_tlvs) From 5f10cfcb3cd8d0ec4d670b062739c10f0e24c895 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:14:39 -0500 Subject: [PATCH 047/117] centauri: send TLV 376 of the join command with 14 bytes The macOS 27.0.1 adapter sends the MLD configuration TLV 0x178 of the join command (0x20000) with 14 bytes: the 12 default bytes and two zero bytes. With 12 bytes the Alpha firmware of the J873 rejects the join with firmware_status 347 and the 4-way handshake never starts. Measured on an M6 Mac mini: a WPA2-PSK join (CCMP, 5 GHz) completes the 4-way handshake. About 15 s after the join the Alpha firmware still traps (PHY5G irq_isr.c:105, PHY2G power_manager.c:394); that is not fixed by this change. [Thomas: the driver part of 57482a6f8eda on the J813 line of linux-aurora] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Thomas Glanzmann --- drivers/net/wireless/apple/centauri/centauri_wifi.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index 978f314973317..95f9ad2d0b208 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -942,13 +942,15 @@ static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, u16 static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, struct cfg80211_connect_params *params) { - /* Stock 25G76 ACIConnectionAdapter::connectAp (0x1808e94cc) emits a - * 12-byte TLV 376. The newer SDK 27 adapter uses 14 bytes. Preserve the - * unspecified-link masks; all-zero is not the non-MLO default. + /* The stock 26.x join emits a 12-byte TLV 376. The macOS 27.0.1 + * adapter writes tag 0x178 with length 14: the MLD configuration bytes + * that its default initialisation fills with the defaults below, bytes + * 12 and 13 being 0 and the flags byte (bits from the connection parameters, 0 by default). + * Preserve the unspecified-link masks; all-zero is not the non-MLO default. */ - static const u8 mld_defaults[12] = { + static const u8 mld_defaults[14] = { 1, 0, 1, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 3, + 0xff, 0xff, 0xff, 3, 0, 0, }; /* Firmware connection-context defaults (aciw 0x511ee8). TLV 0xf913 * replaces all five values. Sending zeros arms Supp_AddTimer with a From f33161c10707577ba06bef882320228f3af97841 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:27:41 -0500 Subject: [PATCH 048/117] centauri: tell the firmware the host is awake before a join Without the host power state the Alpha firmware treats the host as asleep, powers the PHYs down and traps about 15 s after a join (PHY5G irq_isr.c:105, PHY2G power_manager.c:394, and in a log with a post-join frame also LMAC2G "PMGR - Assert while waiting"). Before the join, send the three commands that the host_wakeup and tx_config debugfs files already send: command 6 with a zero body (host power state awake), command 43 (packet lifetime) and command 52 (control profile). The debugfs files stay; the module skips what they applied. The new host_setup parameter (default on) turns this off. Measured on a Mac mini M6 (J873), WPA2-PSK, 5 GHz, with the data rings set up through the data_probe files as before: - the firmware stayed alive (stage 2) for the whole test, about 17 minutes connected in the run with the three commands sent through debugfs, and for the whole test in the run with this change; before, it trapped within 15 s in 14 of 14 variants; - DHCP lease over Wi-Fi; ping through wlan0; iperf3 to a host on the same network bound to the wlan0 address: 44.7 / 58.9 Mbit/s up / down with the debugfs commands, 43.7 / 40.2 Mbit/s with this change; - a 600 s iperf3 at 20 Mbit/s moved 1.40 GBytes without a stall. Not measured: more than 20 minutes, suspend, roaming, 2.4 GHz, WPA3. Some ping windows lost packets (0 to 50 percent in 10 to 20 packet samples). [Thomas: the driver part of 1242f12e876b on the J813 line of linux-aurora] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Thomas Glanzmann --- .../wireless/apple/centauri/centauri_wifi.c | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index 95f9ad2d0b208..59f947bc7c34c 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -29,6 +29,10 @@ static_assert(DATA_TX_BUFFER >= CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= DATA_RX_BUFFER); +static bool host_setup = true; +module_param(host_setup, bool, 0644); +MODULE_PARM_DESC(host_setup, "Before a join, tell the firmware the host is awake (command 6) and apply the reference TX defaults (commands 43 and 52) when the tx_config files have not; 1 = on (default)"); + static bool verify_pmk; module_param(verify_pmk, bool, 0400); MODULE_PARM_DESC(verify_pmk, "Compare firmware PMK with the requested key without exposing either"); @@ -131,6 +135,7 @@ struct centauri_wifi { size_t fwlog_size; bool fwlog_attempted, fwlog_ready; u8 tx_profile[10], tx_defaults_applied; + bool host_awake_sent; bool tx_profile_valid; u8 link_count, link_prime[3], link_channel[3], link_band[3], emlsr_mode; bool link_status_valid, emlsr_valid; @@ -1057,6 +1062,39 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, return ret; } +/* Without the host power state the firmware treats the host as asleep, powers the PHYs down and traps about 15 s after the + * join. Same three commands as the host_wakeup and tx_config debugfs files; called with wifi->lock held. + */ +static void host_setup_commands(struct centauri_wifi *wifi) +{ + static const u8 awake[3] = {}; + static const u8 control_profile[] = { 0x54, 1, 10, 0, 0, 8, 0, 0, 0xff, 0xff, 0xff, 0xff, 1, 1 }; + u8 lifetime[20] = {}; + int ret, i; + + if (!host_setup) + return; + if (!wifi->host_awake_sent) { + ret = command(wifi, 6, awake, sizeof(awake)); + dev_info(&wifi->control->pdev->dev, "host power state awake: rc=%d status=%u\n", ret, wifi->reply_status); + wifi->host_awake_sent = !ret; + } + if (!(wifi->tx_defaults_applied & BIT(0))) { + for (i = 0; i < 4; i++) + put_unaligned_le32(2000000, lifetime + 4 + 4 * i); + ret = command(wifi, 43, lifetime, sizeof(lifetime)); + if (!ret) + wifi->tx_defaults_applied |= BIT(0); + dev_info(&wifi->control->pdev->dev, "TX packet lifetime defaults: rc=%d\n", ret); + } + if (!(wifi->tx_defaults_applied & BIT(1))) { + ret = command(wifi, 52, control_profile, sizeof(control_profile)); + if (!ret) + wifi->tx_defaults_applied |= BIT(1); + dev_info(&wifi->control->pdev->dev, "TX control profile defaults: rc=%d\n", ret); + } +} + static void connect_worker(struct work_struct *work) { struct centauri_wifi *wifi = container_of(work, struct centauri_wifi, connect_work); @@ -1066,6 +1104,7 @@ static void connect_worker(struct work_struct *work) int ret; mutex_lock(&wifi->lock); + host_setup_commands(wifi); slot = wifi->command_head; ret = command(wifi, 131072, wifi->connect_payload, wifi->connect_size); command_status = wifi->reply_status; From f614abe5a67fbde70e1fdbc5acac77ad0f9454a2 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:20:29 -0500 Subject: [PATCH 049/117] centauri: switch the firmware roam scan off after a join After the join the firmware runs its own roam and background scans. No host roam profile exists, and each scan takes the radio off channel for seconds. Send ACI command 0x20016 (no body) once the join succeeded, unless the new roam_scan parameter is set (default off, so the scan is off). Measured on a Mac mini M6 (J873), WPA2-PSK, 5 GHz, two boots with the host-awake commands from the previous patch, a ping every 0.5 s for 120 s to the Wi-Fi gateway, bound to wlan0: - without the command: 240 sent, 68 received (71.7 percent loss), received per 10 s: 13 0 13 0 10 3 3 10 0 13 0 3 0 of 20; - with the command in a separate boot: 240 of 240 received, rtt 9 to 73 ms, every 10 s bucket 20 of 20; - with this patch (default settings, third boot): 240 of 240 received, rtt 9.5 to 96 ms, 20 of 20 in every bucket. This is a choice for a stationary machine: with the scan off the firmware does not look for a better access point. A host roam profile (command 0x20014) would be the way to bring roaming back; it was accepted by the firmware but not tested as a replacement. [Thomas: the driver part of 00c55963147d on the J813 line of linux-aurora] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Thomas Glanzmann --- drivers/net/wireless/apple/centauri/centauri_wifi.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index 59f947bc7c34c..842c2117953a7 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -29,6 +29,10 @@ static_assert(DATA_TX_BUFFER >= CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= DATA_RX_BUFFER); +static bool roam_scan; +module_param(roam_scan, bool, 0644); +MODULE_PARM_DESC(roam_scan, "Let the firmware run its own roam and background scans after a join; 0 = off (default). The scans take the radio off channel for seconds (72 percent ping loss measured) until the host programs a roam profile"); + static bool host_setup = true; module_param(host_setup, bool, 0644); MODULE_PARM_DESC(host_setup, "Before a join, tell the firmware the host is awake (command 6) and apply the reference TX defaults (commands 43 and 52) when the tx_config files have not; 1 = on (default)"); @@ -1166,6 +1170,11 @@ static void connect_worker(struct work_struct *work) status = !ret && wifi->connect_status == 0 ? WLAN_STATUS_SUCCESS : WLAN_STATUS_UNSPECIFIED_FAILURE; wifi->connecting = false; wifi->connected = status == WLAN_STATUS_SUCCESS; + if (wifi->connected && !roam_scan) { + int roam_ret = command(wifi, 0x20016, NULL, 0); + + dev_info(&wifi->control->pdev->dev, "roam scan off: rc=%d status=%u\n", roam_ret, wifi->reply_status); + } memzero_explicit(wifi->expected_pmk, sizeof(wifi->expected_pmk)); /* The command and connection completion have consumed the PMK buffer. */ if (slot < ring_size[1]) From cd97f4c14e8145954b9296968169260bcffa6f2f Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:48:37 -0500 Subject: [PATCH 050/117] centauri: open the data rings and join with the PMK alone A connection needed three manual steps through debugfs: the passphrase file, the data_probe ids 76, 24, 15 and 12 before the join, and id 25 after it. Do the ring setup from the connect path (parameter auto_rings, default on): the four rings under the connect lock before the join, the normal TX ring once the connection result is reported, so the join stays inside the supplicant's timeout. The data_probe files keep working and are skipped for rings that are already open. The data_probe writer is split into a helper that the connect path calls with the lock held. The firmware accepts a join with the PMK from the supplicant alone, so the passphrase debugfs file and TLV 313 are gone. Measured on a Mac mini M6 (J873), WPA2-PSK, 5 GHz, wpa_supplicant started with a PSK configuration and nothing else (no passphrase file, no data_probe, no host_wakeup or tx_config write): the join completed, DHCP gave a lease, the log shows the four rings at join time, the host-awake and TX default commands, the roam scan switch-off and ring 25 after the result; 20 of 20 pings through wlan0 came back and iperf3 gave 56.3 Mbit/s up and 38.4 down over 10 s each. Measured on the same machine with the module installed and the standard services enabled (wpa_supplicant-nl80211@wlan0 and systemd-networkd, nothing started by hand): after one reboot wlan0 had its address 18 s after boot; the log shows the rings, the host-awake and TX default commands, the roam scan switch-off and ring 25 between 12.6 s and 17.0 s; 20 of 20 pings through wlan0 came back and iperf3 gave 54.8 Mbit/s up and 40.2 down over 10 s each. NetworkManager could not be used on that image: its iwd backend does not start on the test kernel, which lacks CONFIG_CRYPTO_USER_API_HASH, CONFIG_CRYPTO_USER_API_SKCIPHER and CONFIG_KEY_DH_OPERATIONS among others. [Thomas: the driver part of 8b8ec1a2ee3e on the J813 line of linux-aurora] Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Signed-off-by: Thomas Glanzmann --- .../wireless/apple/centauri/centauri_wifi.c | 121 ++++++++---------- 1 file changed, 54 insertions(+), 67 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index 842c2117953a7..fd863f1fe506b 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -29,6 +29,12 @@ static_assert(DATA_TX_BUFFER >= CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= DATA_RX_BUFFER); +static bool auto_rings = true; +module_param(auto_rings, bool, 0644); +MODULE_PARM_DESC(auto_rings, "Open the data rings (data_probe ids 76, 24, 15, 12 before the join, 25 after it) from the connect path; 1 = on (default). The data_probe debugfs files still work and are skipped for rings already open"); + +static int data_probe_locked(struct centauri_wifi *wifi, unsigned int id, bool internal); + static bool roam_scan; module_param(roam_scan, bool, 0644); MODULE_PARM_DESC(roam_scan, "Let the firmware run its own roam and background scans after a join; 0 = off (default). The scans take the radio off channel for seconds (72 percent ping loss measured) until the host programs a roam profile"); @@ -114,9 +120,6 @@ struct centauri_wifi { u16 connect_status; u8 bssid[ETH_ALEN], connect_payload[1024], request_ie[256]; size_t connect_size, request_ie_size; - u8 passphrase[64]; - size_t passphrase_size; - struct dentry *passphrase_file; struct dentry *data_probe_file; struct dentry *data_status_file; struct dentry *data_rx_snapshot_file; @@ -973,7 +976,7 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, struct cfg80211_bss *bss; const struct cfg80211_bss_ies *ies; const u8 *rsn, *ap_rsn; - u8 rsn_body[255], ssid[35], key[66], zero[20] = {}, pmf; + u8 rsn_body[255], ssid[35], zero[20] = {}, pmf; u8 *payload = wifi->connect_payload; size_t size = 130, rsn_size; u16 caps = 0; @@ -992,10 +995,6 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, ret = -EBUSY; goto out; } - if (!wifi->passphrase_size) { - ret = -ENOKEY; - goto out; - } bss = cfg80211_get_bss(wiphy, params->channel, params->bssid, params->ssid, params->ssid_len, IEEE80211_BSS_TYPE_ESS, IEEE80211_PRIVACY_ON); if (!bss) { @@ -1045,11 +1044,9 @@ static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, memcpy(ssid + 3, params->ssid, params->ssid_len); append_tlv(payload, &size, 342, ssid, 3 + params->ssid_len); append_tlv(payload, &size, WLAN_EID_RSN, rsn_body, rsn_size); - /* The fixed field carries the PMK; TLV 313 carries the passphrase. */ - put_unaligned_le16(wifi->passphrase_size, key); - memcpy(key + 2, wifi->passphrase, wifi->passphrase_size); - append_tlv(payload, &size, 313, key, 2 + wifi->passphrase_size); - memzero_explicit(key, sizeof(key)); + /* The fixed field carries the PMK. The firmware accepts a join with the PMK alone (measured); the macOS TLV 313 carries the + * passphrase and is not sent. + */ append_tlv(payload, &size, 316, &pmf, 1); append_tlv(payload, &size, 376, mld_defaults, sizeof(mld_defaults)); wifi->connect_size = size; @@ -1108,6 +1105,17 @@ static void connect_worker(struct work_struct *work) int ret; mutex_lock(&wifi->lock); + if (auto_rings) { + static const unsigned int ring_ids[] = { 76, 24, 15, 12 }; + unsigned int n; + + for (n = 0; n < ARRAY_SIZE(ring_ids); n++) { + int ring_ret = data_probe_locked(wifi, ring_ids[n], true); + + if (ring_ret && ring_ret != -EBUSY) + dev_warn(&wifi->control->pdev->dev, "data ring %u setup failed: %d\n", ring_ids[n], ring_ret); + } + } host_setup_commands(wifi); slot = wifi->command_head; ret = command(wifi, 131072, wifi->connect_payload, wifi->connect_size); @@ -1185,6 +1193,18 @@ static void connect_worker(struct work_struct *work) ret, command_status, wifi->connect_status, wifi->connect_event); cfg80211_connect_result(wifi->netdev, wifi->bssid, wifi->request_ie, wifi->request_ie_size, NULL, 0, status, GFP_KERNEL); + if (status == WLAN_STATUS_SUCCESS && auto_rings && !READ_ONCE(wifi->stopping)) { + /* The normal TX ring is opened after the connection result, as in the manual recipe (data_probe 25 after wpa completed); + * doing it after cfg80211_connect_result keeps the join inside the supplicant's timeout. + */ + int ring_ret; + + mutex_lock(&wifi->lock); + ring_ret = data_probe_locked(wifi, 25, true); + mutex_unlock(&wifi->lock); + if (ring_ret && ring_ret != -EBUSY) + dev_warn(&wifi->control->pdev->dev, "data ring 25 setup failed: %d\n", ring_ret); + } if (status == WLAN_STATUS_SUCCESS && !READ_ONCE(wifi->stopping)) { netif_carrier_on(wifi->netdev); if (wifi->data_probe_done & BIT(5)) @@ -1328,46 +1348,6 @@ static const struct net_device_ops net_ops = { .ndo_validate_addr = eth_validate_addr, }; -static ssize_t passphrase_write(struct file *file, const char __user *input, - size_t count, loff_t *position) -{ - struct centauri_wifi *wifi = file->private_data; - u8 value[64]; - size_t length = count; - int ret = 0; - - if (count < 8 || count > sizeof(value)) - return -EINVAL; - if (copy_from_user(value, input, count)) { - ret = -EFAULT; - goto out; - } - if (value[length - 1] == '\n') - length--; - if (length < 8 || length > 63) { - ret = -EINVAL; - goto out; - } - mutex_lock(&wifi->lock); - if (wifi->connecting) { - ret = -EBUSY; - } else { - memzero_explicit(wifi->passphrase, sizeof(wifi->passphrase)); - memcpy(wifi->passphrase, value, length); - wifi->passphrase_size = length; - } - mutex_unlock(&wifi->lock); -out: - memzero_explicit(value, sizeof(value)); - return ret ?: count; -} - -static const struct file_operations passphrase_ops = { - .owner = THIS_MODULE, - .open = simple_open, - .write = passphrase_write, -}; - /* Airship direct_memregion::map (0x1802864e0) sends opcode 8, a region ID, * byte size and a 64-bit DVA in a 24-byte header. Alpha's region-debug is ID 1. * Firmware 0x175340 passes those fields to DramPhyService_handle_buffer; @@ -1552,18 +1532,14 @@ static const struct file_operations fwlog_buffer_ops = { * 0x11760e. Keep these empty-ring probes explicit until packet DMA is proven. * The reserved regions below do not overlap control rings or scan buffers. */ -static ssize_t data_probe_write(struct file *file, const char __user *input, - size_t count, loff_t *position) +/* Called with wifi->lock held. From the connect path (internal) the "connecting" flag is expected to be set. */ +static int data_probe_locked(struct centauri_wifi *wifi, unsigned int id, bool internal) { - struct centauri_wifi *wifi = file->private_data; u8 message[0x34 + 10] = { 0 }; - unsigned int id, vector, offset, size, bit, doorbell, irq; + unsigned int vector, offset, size, bit, doorbell, irq; bool transfer = false; int ret; - ret = kstrtouint_from_user(input, count, 0, &id); - if (ret) - return ret; if (id == 76) { vector = 192; offset = 0x200000; @@ -1596,15 +1572,14 @@ static ssize_t data_probe_write(struct file *file, const char __user *input, } else { return -EINVAL; } - mutex_lock(&wifi->lock); if (readl(wifi->a.bar + 8) != 2) { ret = -EIO; - goto out; + return ret; } - if (wifi->connecting || wifi->scan_request || + if ((!internal && wifi->connecting) || wifi->scan_request || (wifi->data_probe_done & BIT(bit))) { ret = -EBUSY; - goto out; + return ret; } wifi->data_probe_done |= BIT(bit); memset(wifi->a.window + offset, 0, size * 32); @@ -1673,7 +1648,21 @@ static ssize_t data_probe_write(struct file *file, const char __user *input, size - 1, readl(wifi->a.bar + 8), index_read(wifi, 1, 12), index_read(wifi, 0, 207)); } -out: + return ret; +} + +static ssize_t data_probe_write(struct file *file, const char __user *input, + size_t count, loff_t *position) +{ + struct centauri_wifi *wifi = file->private_data; + unsigned int id; + int ret; + + ret = kstrtouint_from_user(input, count, 0, &id); + if (ret) + return ret; + mutex_lock(&wifi->lock); + ret = data_probe_locked(wifi, id, false); mutex_unlock(&wifi->lock); return ret ?: count; } @@ -2333,7 +2322,6 @@ int centauri_wifi_register(struct centauri *c) } dev_info(&c->pdev->dev, "cfg80211 interface %s ready for scan tests; packet data unavailable\n", wifi->netdev->name); - wifi->passphrase_file = debugfs_create_file("passphrase", 0200, c->debug, wifi, &passphrase_ops); wifi->data_probe_file = debugfs_create_file("data_probe", 0200, c->debug, wifi, &data_probe_ops); wifi->data_status_file = debugfs_create_file("data_status", 0400, c->debug, wifi, &data_status_ops); wifi->data_rx_snapshot_file = debugfs_create_file("data_rx_snapshot", 0400, @@ -2369,7 +2357,6 @@ void centauri_wifi_unregister(struct centauri *c) WRITE_ONCE(wifi->stopping, true); if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) netif_tx_disable(wifi->netdev); - debugfs_remove(wifi->passphrase_file); debugfs_remove(wifi->data_probe_file); debugfs_remove(wifi->data_status_file); debugfs_remove(wifi->data_rx_snapshot_file); From ad55a0b8127d9c13333d2111376ee8e17b2b57d1 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 25 Oct 2026 23:02:30 +0100 Subject: [PATCH 051/117] centauri: report station counters, signal and channel wavemon and iw had nothing to show for the N1: the driver implemented no get_station or get_channel. Report the packet, byte, failure and drop counters the data path already keeps, the signal of the joined BSS as cfg80211 has it from the last scan (no firmware query for live link statistics is known yet), and the joined channel at the 20 MHz the driver associates with. Bitrates and transmit power stay unreported. [Thomas: the driver part of 767a663a63de on the J813 line of linux-aurora] Signed-off-by: Thomas Glanzmann --- .../wireless/apple/centauri/centauri_wifi.c | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index fd863f1fe506b..df694b9c13a00 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -1297,11 +1297,81 @@ static void wifi_regulatory(struct wiphy *wiphy, struct regulatory_request *requ mutex_unlock(&wifi->lock); } +/* + * The joined BSS as cfg80211 knows it from the last scan. No firmware query + * for live link statistics is known yet, so its signal is the scan's. + */ +static struct cfg80211_bss *wifi_joined_bss(struct centauri_wifi *wifi) +{ + if (!wifi->connected) + return NULL; + return cfg80211_get_bss(wifi->wiphy, NULL, wifi->bssid, NULL, 0, + IEEE80211_BSS_TYPE_ESS, IEEE80211_PRIVACY_ANY); +} + +static int wifi_get_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_info *sinfo) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + struct net_device_stats *stats = &wifi->netdev->stats; + struct cfg80211_bss *bss; + int ret = 0; + + mutex_lock(&wifi->lock); + if (!wifi->connected || !ether_addr_equal(mac, wifi->bssid)) { + ret = -ENOENT; + goto out; + } + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS) | + BIT_ULL(NL80211_STA_INFO_TX_PACKETS) | + BIT_ULL(NL80211_STA_INFO_RX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_TX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_TX_FAILED) | + BIT_ULL(NL80211_STA_INFO_RX_DROP_MISC); + sinfo->rx_packets = stats->rx_packets; + sinfo->tx_packets = stats->tx_packets; + sinfo->rx_bytes = stats->rx_bytes; + sinfo->tx_bytes = stats->tx_bytes; + sinfo->tx_failed = stats->tx_errors; + sinfo->rx_dropped_misc = stats->rx_dropped; + + bss = wifi_joined_bss(wifi); + if (bss) { + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); + sinfo->signal = bss->signal / 100; /* mBm */ + cfg80211_put_bss(wiphy, bss); + } +out: + mutex_unlock(&wifi->lock); + return ret; +} + +static int wifi_get_channel(struct wiphy *wiphy, struct wireless_dev *wdev, + unsigned int link_id, struct cfg80211_chan_def *chandef) +{ + struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); + struct cfg80211_bss *bss; + int ret = -ENODATA; + + mutex_lock(&wifi->lock); + bss = wifi_joined_bss(wifi); + if (bss) { + /* The join asks the firmware for a 20 MHz channel. */ + cfg80211_chandef_create(chandef, bss->channel, NL80211_CHAN_NO_HT); + cfg80211_put_bss(wiphy, bss); + ret = 0; + } + mutex_unlock(&wifi->lock); + return ret; +} + static const struct cfg80211_ops wifi_ops = { .scan = wifi_scan, .connect = wifi_connect, .disconnect = wifi_disconnect, .set_power_mgmt = wifi_power_mgmt, + .get_station = wifi_get_station, + .get_channel = wifi_get_channel, }; static int net_open(struct net_device *netdev) From 019c0ab369b31347b83748e01adaf11c101e0360 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 26 Oct 2026 00:43:13 +0100 Subject: [PATCH 052/117] centauri, btcentauri: match the J873 as apple,j873g The J873 device tree now uses the machine compatible from the binding, "apple,j873g" (after the ADT target-type J873g). Match that in the centauri board table and in btcentauri. [Thomas: the driver part of 9e911df6e4c0 on the J813 line of linux-aurora] Signed-off-by: Thomas Glanzmann --- drivers/bluetooth/btcentauri.c | 2 +- drivers/net/wireless/apple/centauri/centauri.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c index 0bd8fedf3c14e..82f623b6de581 100644 --- a/drivers/bluetooth/btcentauri.c +++ b/drivers/bluetooth/btcentauri.c @@ -901,7 +901,7 @@ static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) struct hci_dev *hdev; int ret; - if (!(of_machine_is_compatible("apple,j813") || of_machine_is_compatible("apple,j873")) || + if (!(of_machine_is_compatible("apple,j813") || of_machine_is_compatible("apple,j873g")) || PCI_FUNC(pdev->devfn) != 2) return -ENODEV; bt = devm_kzalloc(&pdev->dev, sizeof(*bt), GFP_KERNEL); diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h index 43256ac8ec40e..c8111fe433cc5 100644 --- a/drivers/net/wireless/apple/centauri/centauri.h +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -72,7 +72,7 @@ static inline const struct centauri_board *centauri_board(void) { "apple,j813", "apple/centauri/j813-ftab.bin", 0x00248142, 0x488028000ULL, 0x48f010000ULL, 0x39a000300ULL, 0x03020000, 0x100c, false, 385, false }, - { "apple,j873", "apple/centauri/j873-ftab.bin", 0x00248152, + { "apple,j873g", "apple/centauri/j873-ftab.bin", 0x00248152, 0x388028000ULL, 0x38f010000ULL, 0x31a00000cULL, 0x208, 0, true, 386, true }, }; From b8b74070b334d82f5fc655d04198de59c1c8f2bb Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 23:20:50 +0100 Subject: [PATCH 053/117] wifi: apple: centauri: Keep the N1 functions running across s2idle Without PM callbacks the PCI core's default suspend disables the function, which turns bus mastering off under the running N1 firmware. The firmware cannot be restarted without the bootloader's link training, and after resume every ring was dead (each scan failed with -EIO). Save the config space ourselves, so the core leaves the function enabled and in D0, and keep the PCIe host's power domains up through the noirq phase (device_set_awake_path()). Tested on J813: Wi-Fi associated and passing traffic after s2idle. [Thomas: the driver part of fd2382058a0a on the J813 line of linux-aurora] Signed-off-by: Thomas Glanzmann --- .../net/wireless/apple/centauri/centauri_pci.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c index 23fff2160a533..86ad99c48db2e 100644 --- a/drivers/net/wireless/apple/centauri/centauri_pci.c +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -245,6 +245,22 @@ static void centauri_remove(struct pci_dev *pdev) centauri_free_boot(c); } +/* + * The N1 firmware keeps running across s2idle: it cannot be restarted without + * the bootloader's link training. Without PM callbacks the PCI core disables a + * function at suspend, which turns bus mastering off under the running + * firmware and leaves every ring dead (each scan then fails with -EIO). Save + * the state ourselves so the core keeps the function enabled and in D0, and + * keep the PCIe host's power domains up through the noirq phase. + */ +static int centauri_suspend(struct device *dev) +{ + device_set_awake_path(dev); + return pci_save_state(to_pci_dev(dev)); +} + +static DEFINE_SIMPLE_DEV_PM_OPS(centauri_pm_ops, centauri_suspend, NULL); + /* 0x1903 is the Beta (Bluetooth) function: btcentauri claims it. This driver needs functions * 0 and 1 (1 is enabled here, its BAR is used by the Wi-Fi code) and reaches function 2 only * through config space and its struct device for DMA. @@ -262,6 +278,7 @@ static struct pci_driver centauri_driver = { .id_table = centauri_ids, .probe = centauri_probe, .remove = centauri_remove, + .driver.pm = pm_sleep_ptr(¢auri_pm_ops), }; static int __init centauri_init(void) From 8c7511bef728b45cf2491e069efd47f96e3de6bc Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 23:20:50 +0100 Subject: [PATCH 054/117] Bluetooth: btcentauri: Keep the transport running across s2idle As for the Wi-Fi functions: without PM callbacks the PCI core turned bus mastering off at suspend, and the next poll stopped the transport for good. Save the config space so the function stays enabled and in D0, keep the PCIe host powered, and pause the poll work across the sleep. Tested on J813: hci0 UP RUNNING after s2idle. [Thomas: the driver part of c1f436307d70 on the J813 line of linux-aurora] Signed-off-by: Thomas Glanzmann --- drivers/bluetooth/btcentauri.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c index 82f623b6de581..5830c8b2285e4 100644 --- a/drivers/bluetooth/btcentauri.c +++ b/drivers/bluetooth/btcentauri.c @@ -995,6 +995,34 @@ static void cen_remove(struct pci_dev *pdev) cen_stop(bt); } +/* + * As for the Wi-Fi functions (centauri_pci.c): the firmware keeps running, so + * keep the function enabled and in D0 (the PCI core's default suspend turns + * bus mastering off, and the next poll stopped the transport for good) and + * the PCIe host powered. Polling pauses across the sleep. + */ +static int cen_suspend(struct device *dev) +{ + struct cen_bt *bt = pci_get_drvdata(to_pci_dev(dev)); + + cancel_delayed_work_sync(&bt->poll); + device_set_awake_path(dev); + return pci_save_state(to_pci_dev(dev)); +} + +static int cen_resume(struct device *dev) +{ + struct cen_bt *bt = pci_get_drvdata(to_pci_dev(dev)); + + mutex_lock(&bt->lock); + if (bt->running) + schedule_delayed_work(&bt->poll, 0); + mutex_unlock(&bt->lock); + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(cen_pm_ops, cen_suspend, cen_resume); + static const struct pci_device_id cen_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903) }, { } @@ -1006,6 +1034,7 @@ static struct pci_driver cen_driver = { .id_table = cen_ids, .probe = cen_probe, .remove = cen_remove, + .driver.pm = pm_sleep_ptr(&cen_pm_ops), }; module_pci_driver(cen_driver); MODULE_LICENSE("GPL"); From 1d0c6ba354e3e8d81e63dc20d035f4d0822d6f56 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Fri, 30 Oct 2026 05:35:29 +0100 Subject: [PATCH 055/117] PCI: apple: MSI controller for a bootloader-trained port On T8142 (J813) the bootloader trains apcie port 0 and Linux drives the root port as a generic ECAM host, which has no MSI support: the N1's functions could only be polled. The port still turns writes to the MSI doorbell into AIC interrupts through its MSI map, as on T8140/T602X. Add pcie-apple's MSI parent domain as a standalone driver ("apple,t8142-pcie-msi"): it programs the port's MSI address, MSI map and MSI enable and parses msi-ranges like pcie-apple. It depends on PCIE_APPLE=y for the doorbell address and the DART's reserved doorbell IOVA. Signed-off-by: Thomas Glanzmann --- drivers/pci/controller/Kconfig | 8 + drivers/pci/controller/Makefile | 1 + drivers/pci/controller/pcie-apple-msi.c | 185 ++++++++++++++++++++++++ 3 files changed, 194 insertions(+) create mode 100644 drivers/pci/controller/pcie-apple-msi.c diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 7a72e76b08262..a58abe6b1921d 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -83,6 +83,14 @@ config PCIE_APPLE If unsure, say Y if you have an Apple Silicon system. +config PCIE_APPLE_MSI + bool "Apple PCIe MSI controller for a bootloader-trained port" + depends on PCIE_APPLE=y + select IRQ_MSI_LIB + help + MSIs for an Apple PCIe port that Linux drives as a generic ECAM + host because the bootloader trained the link (T8142 Wi-Fi). + config PCI_VERSATILE bool "ARM Versatile PB PCI controller" depends on ARCH_VERSATILE || COMPILE_TEST diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 978c38a61f44c..d09974900927c 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -40,6 +40,7 @@ obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o obj-$(CONFIG_PCIE_HISI_ERR) += pcie-hisi-error.o obj-$(CONFIG_PCIE_APPLE) += pcie-apple.o +obj-$(CONFIG_PCIE_APPLE_MSI) += pcie-apple-msi.o obj-$(CONFIG_PCIE_MT7621) += pcie-mt7621.o obj-$(CONFIG_PCIE_ASPEED) += pcie-aspeed.o diff --git a/drivers/pci/controller/pcie-apple-msi.c b/drivers/pci/controller/pcie-apple-msi.c new file mode 100644 index 0000000000000..452e124c43b9b --- /dev/null +++ b/drivers/pci/controller/pcie-apple-msi.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MSI controller for one Apple PCIe port whose link the bootloader trained. + * + * pcie-apple owns the port when Linux brings the link up itself. On T8142 + * (J813) the bootloader trains apcie port 0 and Linux drives the root port as + * a generic ECAM host, which has no MSI support. The port still turns writes + * to the MSI doorbell into AIC interrupts through its MSI map; this driver + * programs that map and provides the parent domain, as pcie-apple does. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PORT_MSICFG 0x0124 +#define PORT_MSICFG_EN BIT(0) +#define PORT_MSIADDR 0x016c +#define PORT_MSIMAP 0x3800 +#define PORT_MSIMAP_ENABLE BIT(31) + +#define DOORBELL_ADDR CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR + +struct apple_msi { + struct mutex lock; + struct irq_fwspec fwspec; + unsigned long *bitmap; + u32 nvecs; +}; + +static void apple_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) +{ + msg->address_hi = upper_32_bits(DOORBELL_ADDR); + msg->address_lo = lower_32_bits(DOORBELL_ADDR); + msg->data = data->hwirq; +} + +static struct irq_chip apple_msi_bottom_chip = { + .name = "MSI", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_set_type = irq_chip_set_type_parent, + .irq_compose_msi_msg = apple_msi_compose_msg, +}; + +static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct apple_msi *msi = domain->host_data; + struct irq_fwspec fwspec = msi->fwspec; + unsigned int i; + int ret, hwirq; + + mutex_lock(&msi->lock); + hwirq = bitmap_find_free_region(msi->bitmap, msi->nvecs, + order_base_2(nr_irqs)); + mutex_unlock(&msi->lock); + if (hwirq < 0) + return -ENOSPC; + + fwspec.param[fwspec.param_count - 2] += hwirq; + + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &fwspec); + if (ret) { + mutex_lock(&msi->lock); + bitmap_release_region(msi->bitmap, hwirq, order_base_2(nr_irqs)); + mutex_unlock(&msi->lock); + return ret; + } + + for (i = 0; i < nr_irqs; i++) + irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, + &apple_msi_bottom_chip, msi); + return 0; +} + +static void apple_msi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct apple_msi *msi = domain->host_data; + + mutex_lock(&msi->lock); + bitmap_release_region(msi->bitmap, d->hwirq, order_base_2(nr_irqs)); + mutex_unlock(&msi->lock); +} + +static const struct irq_domain_ops apple_msi_domain_ops = { + .alloc = apple_msi_domain_alloc, + .free = apple_msi_domain_free, +}; + +static const struct msi_parent_ops apple_msi_parent_ops = { + .supported_flags = (MSI_GENERIC_FLAGS_MASK | + MSI_FLAG_PCI_MSIX | + MSI_FLAG_MULTI_PCI_MSI), + .required_flags = (MSI_FLAG_USE_DEF_DOM_OPS | + MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSI_MASK_PARENT), + .chip_flags = MSI_CHIP_FLAG_SET_EOI, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int apple_msi_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct fwnode_handle *fwnode = dev_fwnode(dev); + struct of_phandle_args args = {}; + struct irq_domain_info info = { + .fwnode = fwnode, + .ops = &apple_msi_domain_ops, + }; + struct apple_msi *msi; + void __iomem *port; + int ret; + u32 i; + + msi = devm_kzalloc(dev, sizeof(*msi), GFP_KERNEL); + if (!msi) + return -ENOMEM; + mutex_init(&msi->lock); + + port = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(port)) + return PTR_ERR(port); + + ret = of_parse_phandle_with_args(dev->of_node, "msi-ranges", + "#interrupt-cells", 0, &args); + if (ret) + return ret; + ret = of_property_read_u32_index(dev->of_node, "msi-ranges", + args.args_count + 1, &msi->nvecs); + if (ret) { + of_node_put(args.np); + return ret; + } + of_phandle_args_to_fwspec(args.np, args.args, args.args_count, + &msi->fwspec); + of_node_put(args.np); + + msi->bitmap = devm_bitmap_zalloc(dev, msi->nvecs, GFP_KERNEL); + if (!msi->bitmap) + return -ENOMEM; + + info.parent = irq_find_matching_fwspec(&msi->fwspec, DOMAIN_BUS_WIRED); + if (!info.parent) + return dev_err_probe(dev, -EPROBE_DEFER, "no parent domain\n"); + info.size = msi->nvecs; + info.host_data = msi; + + /* MSI data N goes to the Nth interrupt of msi-ranges. */ + writel(lower_32_bits(DOORBELL_ADDR), port + PORT_MSIADDR); + for (i = 0; i < msi->nvecs; i++) + writel(PORT_MSIMAP_ENABLE | i, port + PORT_MSIMAP + 4 * i); + writel(readl(port + PORT_MSICFG) | PORT_MSICFG_EN, port + PORT_MSICFG); + + if (!msi_create_parent_irq_domain(&info, &apple_msi_parent_ops)) + return -ENOMEM; + dev_info(dev, "%u MSIs at %#x\n", msi->nvecs, DOORBELL_ADDR); + return 0; +} + +static const struct of_device_id apple_msi_of_match[] = { + { .compatible = "apple,t8142-pcie-msi" }, + { } +}; + +static struct platform_driver apple_msi_driver = { + .probe = apple_msi_probe, + .driver = { + .name = "pcie-apple-msi", + .of_match_table = apple_msi_of_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(apple_msi_driver); From 68d0acaba211f6f106a4b866373f3d7972f0cbbf Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Fri, 30 Oct 2026 05:35:29 +0100 Subject: [PATCH 056/117] wifi: centauri: interrupt-driven data path with real MSIs The data rings were serviced by a 20 ms delayed work (24 ms at HZ=250) that drains at most 127 RX completions and reclaims the 128 TX slots per pass, which capped the J813 at ~65 Mbit/s with 0-24 ms of added latency. When the port has an MSI controller, give Control and Alpha up to 16 MSIs each (the port has 32) and kick the data worker from every MSI; Beta and ports without a controller keep the DMA mailbox. The poll stays as a fallback. The port cycle's reset clears MSICFG, MSIADDR and the MSI map, so save and restore them on every board (the RID2SID table still only with keep_port_tables). J813: iperf3 65 -> 579 Mbit/s to the host, 60 -> 451-505 Mbit/s from it, ping 15.7 -> 2.9 ms average. Signed-off-by: Thomas Glanzmann --- .../net/wireless/apple/centauri/centauri.h | 4 +- .../wireless/apple/centauri/centauri_ipc.c | 50 +++++++++++++++++++ .../wireless/apple/centauri/centauri_port.c | 29 ++++++----- .../wireless/apple/centauri/centauri_wifi.c | 9 ++++ 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h index c8111fe433cc5..330b9e1a207d9 100644 --- a/drivers/net/wireless/apple/centauri/centauri.h +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -47,7 +47,8 @@ struct centauri { * keep_port_tables: the J813 sequence zeroes the 16 words at port+0x3000 and the 256 words at * port+0x3800. On the J873 those are the RID2SID and MSIMAP tables that pcie-apple programmed * (PORT_T602X_RID2SID, PORT_T602X_MSIMAP); the cycle then drops function 0's stream entry and - * its DMA arrives as SID 0 (DART "BPF reject"). Such a board gets both tables written back. + * its DMA arrives as SID 0 (DART "BPF reject"). Such a board gets its RID2SID table written + * back; the MSI map and the MSI address/config are restored on every board. * link_tlv, scan_tlvs: the macOS 27.0 WLAN adapter differs from the 26.6.1 one the J813 recipe * came from. startInterface carries the three link addresses in TLV 386 (was 385), and the scan * request (command 0x1000f) carries a channel-list TLV 0x104 and an OCE TLV 0x165 after the fixed @@ -91,4 +92,5 @@ int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *res int centauri_alpha_log_buffer(struct centauri *c, void **cpu, dma_addr_t *dma, size_t *size); int centauri_wifi_register(struct centauri *c); void centauri_wifi_unregister(struct centauri *c); +void centauri_wifi_kick(struct centauri *c); #endif diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c index 5f88f9a4390c9..37bbeff58a288 100644 --- a/drivers/net/wireless/apple/centauri/centauri_ipc.c +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -5,6 +5,7 @@ */ #include #include +#include #include #include #include @@ -36,6 +37,7 @@ struct centauri_ipc { struct cen_buffer *secondary, *window, *cchi_rx, *cchi_tx, *debug_rx; struct cen_buffer *alpha_log; struct cen_buffer *msi[3]; + unsigned int vectors[3]; u16 cchi_head, cchi_tail; }; @@ -338,6 +340,46 @@ static int memswap_handoff(struct centauri *c, unsigned int function, const char return 0; } +static irqreturn_t function_irq(int irq, void *data) +{ + centauri_wifi_kick(data); + return IRQ_HANDLED; +} + +/* Real MSIs need an MSI controller on the port (pcie-apple, pcie-apple-msi). The port has 32 + * vectors; Control and Alpha get up to 16 each. Without a controller the function keeps the + * DMA mailbox below and the Wi-Fi data path is polled. + */ +static int request_vectors(struct centauri *c, unsigned int function) +{ + struct pci_dev *pdev = c->ipc->function[function]; + int count, i, ret; + + count = pci_alloc_irq_vectors(pdev, 1, 16, PCI_IRQ_MSI); + if (count < 0) + return count; + for (i = 0; i < count; i++) { + ret = request_irq(pci_irq_vector(pdev, i), function_irq, 0, "centauri", c); + if (ret) { + while (i--) + free_irq(pci_irq_vector(pdev, i), c); + pci_free_irq_vectors(pdev); + return ret; + } + } + c->ipc->vectors[function] = count; + return 0; +} + +static void free_vectors(struct centauri *c, unsigned int function) +{ + struct pci_dev *pdev = c->ipc->function[function]; + + while (c->ipc->vectors[function]) + free_irq(pci_irq_vector(pdev, --c->ipc->vectors[function]), c); + pci_free_irq_vectors(pdev); +} + static int enumerate_functions(struct centauri *c) { struct centauri_ipc *ipc = c->ipc; @@ -367,6 +409,11 @@ static int enumerate_functions(struct centauri *c) pci_read_config_word(ipc->function[i], cap + PCI_MSI_FLAGS, &control); if (!(control & PCI_MSI_FLAGS_64BIT)) return -EOPNOTSUPP; + if (i < 2 && !request_vectors(c, i)) { + dev_info(&c->pdev->dev, "function %d: %u MSIs\n", i, ipc->vectors[i]); + pci_set_master(ipc->function[i]); + continue; + } if (i) { ipc->msi[i] = buffer_alloc(c, i, SZ_16K); if (!ipc->msi[i]) @@ -472,6 +519,9 @@ void centauri_free_ipc(struct centauri *c) if (!ipc) return; + for (i = 0; i < 3; i++) + if (ipc->vectors[i]) + free_vectors(c, i); centauri_wifi_unregister(c); for (i = 0; i < 3; i++) { if (!ipc->function[i]) diff --git a/drivers/net/wireless/apple/centauri/centauri_port.c b/drivers/net/wireless/apple/centauri/centauri_port.c index 62a9456e74ea9..59eadc3f8521c 100644 --- a/drivers/net/wireless/apple/centauri/centauri_port.c +++ b/drivers/net/wireless/apple/centauri/centauri_port.c @@ -46,7 +46,7 @@ int centauri_port_cycle(struct pci_dev *pdev) unsigned int lanes, i; int ret = -ETIMEDOUT, pm; bool ack, l2; - u32 rid2sid[16], msimap[256], valid = 0; + u32 rid2sid[16], msimap[256], msicfg = 0, msiaddr = 0, valid = 0; bool saved = false; if (!board || !board->port || !bridge || bridge->vendor != PCI_VENDOR_ID_APPLE || @@ -71,13 +71,15 @@ int centauri_port_cycle(struct pci_dev *pdev) ret = pci_save_state(pdev); if (ret) goto out; - if (board->keep_port_tables) { + if (board->keep_port_tables) for (i = 0; i < ARRAY_SIZE(rid2sid); i++) rid2sid[i] = readl(port + 0x3000 + 4 * i); - for (i = 0; i < ARRAY_SIZE(msimap); i++) - msimap[i] = readl(port + 0x3800 + 4 * i); - saved = true; - } + /* The reset below also clears an MSI controller's setup (pcie-apple, pcie-apple-msi). */ + for (i = 0; i < ARRAY_SIZE(msimap); i++) + msimap[i] = readl(port + 0x3800 + 4 * i); + msicfg = readl(port + 0x124); + msiaddr = readl(port + 0x16c); + saved = true; pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &link); lanes = (link & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT; lanes = lanes >= 1 && lanes <= 16 ? BIT(lanes) - 1 : 1; @@ -196,14 +198,17 @@ int centauri_port_cycle(struct pci_dev *pdev) ret = -ETIMEDOUT; out: if (saved) { - for (i = 0; i < ARRAY_SIZE(rid2sid); i++) { - writel(rid2sid[i], port + 0x3000 + 4 * i); - valid += !!(rid2sid[i] & BIT(31)); - } + if (board->keep_port_tables) + for (i = 0; i < ARRAY_SIZE(rid2sid); i++) { + writel(rid2sid[i], port + 0x3000 + 4 * i); + valid += !!(rid2sid[i] & BIT(31)); + } for (i = 0; i < ARRAY_SIZE(msimap); i++) writel(msimap[i], port + 0x3800 + 4 * i); - dev_info(&pdev->dev, "restored %u RID2SID entries and the MSI map (ret %d)\n", - valid, ret); + writel(msiaddr, port + 0x16c); + writel(msicfg, port + 0x124); + dev_info(&pdev->dev, "restored %u RID2SID entries and the MSI setup %#x/%#x (ret %d)\n", + valid, msicfg, msiaddr, ret); } if (pad) iounmap(pad); diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index df694b9c13a00..f0da841721374 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -1235,6 +1235,15 @@ static void link_worker(struct work_struct *work) } } +/* MSI from Control or Alpha: service the data rings now instead of at the next poll. */ +void centauri_wifi_kick(struct centauri *c) +{ + struct centauri_wifi *wifi = READ_ONCE(c->wifi); + + if (wifi && READ_ONCE(wifi->connected) && !READ_ONCE(wifi->stopping)) + mod_delayed_work(system_wq, &wifi->link_work, 0); +} + static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason) { struct centauri_wifi *wifi = *(struct centauri_wifi **)wiphy_priv(wiphy); From eb01c70f93803d20f47737c58c227d3d5b17832c Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Fri, 30 Oct 2026 06:00:33 +0100 Subject: [PATCH 057/117] wifi: centauri: give Alpha a full block of 16 MSIs, ask for it first The data rings tell the firmware to signal Alpha vectors 3 and 8. On the J873 the 10GbE NIC shares pcie-apple's 32 vectors with the N1; Control asked first and took the only free block of 16, Alpha got one vector, and the Alpha firmware faulted (stage 3) as soon as the first data ring was opened, so every later command failed with -EIO. Allocate Alpha before Control and only as a whole block of 16; a function that cannot get one keeps the DMA mailbox. On the J873 Control now stays on the mailbox and Alpha gets its 16 vectors. J873 (Mac mini M6, 5 GHz ch 44, -34 dBm): iperf3 60 -> 511 Mbit/s to the mini, 60 -> 530 Mbit/s from it, ping 17 -> 2.4 ms average. Signed-off-by: Thomas Glanzmann --- drivers/net/wireless/apple/centauri/centauri_ipc.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c index 37bbeff58a288..4c26c70853815 100644 --- a/drivers/net/wireless/apple/centauri/centauri_ipc.c +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -346,16 +346,17 @@ static irqreturn_t function_irq(int irq, void *data) return IRQ_HANDLED; } -/* Real MSIs need an MSI controller on the port (pcie-apple, pcie-apple-msi). The port has 32 - * vectors; Control and Alpha get up to 16 each. Without a controller the function keeps the - * DMA mailbox below and the Wi-Fi data path is polled. +/* Real MSIs need an MSI controller on the port (pcie-apple, pcie-apple-msi). The data rings + * name Alpha vectors 3 and 8, so a function gets all 16 vectors or none; one that gets none + * keeps the DMA mailbox below (polled). Alpha asks first: on the J873 the 10GbE NIC shares + * the controller's 32 vectors and only one block of 16 is left. */ static int request_vectors(struct centauri *c, unsigned int function) { struct pci_dev *pdev = c->ipc->function[function]; int count, i, ret; - count = pci_alloc_irq_vectors(pdev, 1, 16, PCI_IRQ_MSI); + count = pci_alloc_irq_vectors(pdev, 16, 16, PCI_IRQ_MSI); if (count < 0) return count; for (i = 0; i < count; i++) { @@ -392,7 +393,7 @@ static int enumerate_functions(struct centauri *c) c->pdev->multifunction = 1; pci_rescan_bus(c->pdev->bus); pci_unlock_rescan_remove(); - for (i = 0; i < 3; i++) { + for (i = 2; i >= 0; i--) { ipc->function[i] = pci_get_domain_bus_and_slot(pci_domain_nr(c->pdev->bus), c->pdev->bus->number, PCI_DEVFN(0, i)); if (!ipc->function[i] || ipc->function[i]->vendor != PCI_VENDOR_ID_APPLE || From 2d95032110fdefb8f75538539e459eb499f2ff36 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:33:34 +0100 Subject: [PATCH 058/117] dt-bindings: interrupt-controller: Add the Apple PCIe port MSI map The MSI map of one Apple PCIe root port that the bootloader trained and Linux drives as a generic ECAM host (the J813 N1 on T8142 apcie port 0). Signed-off-by: Thomas Glanzmann --- .../interrupt-controller/apple,pcie-msi.yaml | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 Documentation/devicetree/bindings/interrupt-controller/apple,pcie-msi.yaml diff --git a/Documentation/devicetree/bindings/interrupt-controller/apple,pcie-msi.yaml b/Documentation/devicetree/bindings/interrupt-controller/apple,pcie-msi.yaml new file mode 100644 index 0000000000000..e08ca3bf7108a --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/apple,pcie-msi.yaml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/interrupt-controller/apple,pcie-msi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple PCIe port MSI controller + +maintainers: + - Thomas Glanzmann + +description: | + The MSI map of one Apple PCIe root port, for a port that the bootloader + trained and that Linux drives as a generic ECAM host. Endpoint MSI writes + to the doorbell are routed by this map to a contiguous range of AIC + interrupts. + +allOf: + - $ref: /schemas/interrupt-controller/msi-controller.yaml# + +properties: + compatible: + const: apple,t8142-pcie-msi + + reg: + maxItems: 1 + description: The port's MSI map registers. + + msi-controller: true + + msi-ranges: + maxItems: 1 + +required: + - compatible + - reg + - msi-controller + - msi-ranges + +unevaluatedProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + msi-controller@488028000 { + compatible = "apple,t8142-pcie-msi"; + reg = <0x4 0x88028000 0x0 0x8000>; + msi-controller; + msi-ranges = <&aic AIC_IRQ 1741 IRQ_TYPE_EDGE_RISING 32>; + }; + }; From ecbb337176305dcb8aa8eb843e5e7aa10748c501 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:39:33 +0100 Subject: [PATCH 059/117] soc: apple: rtkit: Take a crash buffer the firmware maps itself The T8142 AOP keeps its crash buffer in its own slot of its DART stream and asks for it on endpoint 1 with that IOVA. A client without shmem_setup refused the request, so the firmware never got its crash buffer acknowledged. APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG takes such a buffer as the firmware's, without a CPU mapping, and acknowledges the request unchanged; a later crash is reported without a log. RtKit::new_with_firmware_crashlog() selects it at runtime, so a driver can use it for one SoC only. Signed-off-by: Thomas Glanzmann --- drivers/soc/apple/rtkit.c | 15 +++++++++++++++ include/linux/soc/apple/rtkit.h | 7 +++++++ rust/bindings/bindings_helper.h | 1 + rust/kernel/soc/apple/rtkit.rs | 16 ++++++++++++++++ 4 files changed, 39 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index d352083aaa26c..9eaa28d16c46d 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -410,6 +410,13 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n", request.size, &request.iova); + if ((rtk->ops->flags & APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG) && + ep == APPLE_RTKIT_EP_CRASHLOG && request.iova) { + *buffer = request; + buffer->is_mapped = true; + return apple_rtkit_send_message(rtk, ep, msg, NULL, false); + } + if (request.iova && !rtk->ops->shmem_setup && !rtk->ops->shmem_setup_ep) { err = -EINVAL; goto error; @@ -734,6 +741,14 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) dev_err(rtk->dev, "RTKit: co-processor has crashed\n"); + /* A firmware-owned crash buffer has no CPU mapping to read the log from. */ + if (!rtk->crashlog_buffer.buffer && !rtk->crashlog_buffer.iomem) { + rtk->crashed = true; + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); + return; + } + /* * create a shadow copy here to make sure the co-processor isn't able * to change the log while we're dumping it. this also ensures diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 65bd28a6f0240..4c27e012c2c75 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -57,6 +57,13 @@ struct apple_rtkit_inherited_buffer { /* This client requires endpoint 7 setup before IOP power can be acknowledged. */ #define APPLE_RTKIT_NEEDS_COREANALYTICS BIT(1) +/* + * The co-processor keeps its crash buffer in memory it maps itself (the T8142 + * AOP) and asks for it with that IOVA: take the buffer as the firmware's, + * without a CPU mapping, and acknowledge the request unchanged. + */ +#define APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG BIT(2) + /* * Struct to represent implementation-specific RTKit operations. * diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 56f9f84689804..2731a39a0255c 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -158,6 +158,7 @@ const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; const u32 RUST_CONST_HELPER_APPLE_RTKIT_NEEDS_COREANALYTICS = APPLE_RTKIT_NEEDS_COREANALYTICS; +const u32 RUST_CONST_HELPER_APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG = APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG; const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 73896fea53e00..ef9c9e7f27b0f 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -245,6 +245,11 @@ impl RtKit { ..Self::VTABLE }; + const FIRMWARE_CRASHLOG_VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + flags: Self::VTABLE.flags | bindings::APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG, + ..Self::VTABLE + }; + /// Creates a new RTKit client for a given device and optional mailbox name or index. pub fn new( dev: &device::Device, @@ -268,6 +273,17 @@ impl RtKit { Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::COREANALYTICS_VTABLE) } + /// Creates a client whose co-processor keeps its crash buffer in memory it + /// maps itself and asks for it with that IOVA (the T8142 AOP). + pub fn new_with_firmware_crashlog( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ) -> Result { + Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::FIRMWARE_CRASHLOG_VTABLE) + } + fn new_with_ops( dev: &device::Device, mbox_name: Option<&'static CStr>, From 5f9fcba6eb36f0c74c69763e35885775bad4b5fe Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 07:26:19 +0100 Subject: [PATCH 060/117] drm/asahi: mmu: Map at a fixed address with a guard of any size Vm::map_at() reserves at most one guard page after a mapping. The M5 backend maps the driver's pools at the same address in each client VM with two guard pages after each, as it does when it allocates them. Add Vm::map_at_with_guard_size() and express map_at() through it. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/mmu.rs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index f146b2762d812..4960285975d0b 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2538,9 +2538,24 @@ impl Vm { prot: Prot, guard: bool, ) -> Result { - let reserved_size = size - .checked_add(if guard { UAT_PGSZ } else { 0 }) - .ok_or(EOVERFLOW)?; + self.map_at_with_guard_size(addr, size, gem, prot, if guard { UAT_PGSZ } else { 0 }) + } + + /// Map a GEM object into this Vm at a specific address, followed by an unmapped guard of + /// `guard_size` bytes (a multiple of the page size). + #[inline(never)] + pub(crate) fn map_at_with_guard_size( + &self, + addr: u64, + size: usize, + gem: &gem::Object, + prot: Prot, + guard_size: usize, + ) -> Result { + if guard_size & UAT_PGMSK != 0 { + return Err(EINVAL); + } + let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; // End the bound-device lease before entering firmware/GPUVM waits. let (_bind_lease, vm_bo) = { From d3d5efcb391fea58ab6eb7134aca51f447b5b8da Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 07:26:20 +0100 Subject: [PATCH 061/117] drm/asahi: mmu: Keep the kernel range of a VM The kernel range of a VM, reserved at VM_CREATE for kernel-managed objects, is only handed to GPUVM. The M5 backend places its per-client objects in that range and checks that userspace commands stay out of it. Keep the range in the Vm and add an accessor. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/mmu.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 4960285975d0b..b36ceaa1dcc8c 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1033,6 +1033,9 @@ pub(crate) struct Vm { device_binding: Arc, dev: ARef, id: u64, + /// The range reserved for kernel-managed objects at creation. + #[cfg_attr(not(CONFIG_DRM_ASAHI_M5), allow(dead_code))] + kernel_range: Range, mapping_epoch: Option>, bind_sequence: Arc, lifetime: Option>, @@ -2159,7 +2162,7 @@ impl Vm { &dev, &*dummy_obj.gem, gpuvm_range, - kernel_range, + kernel_range.clone(), VmInner { dev: dev.clone(), is_kernel, @@ -2181,6 +2184,7 @@ impl Vm { device_binding, dev: dev.as_ref().as_ref().into(), id, + kernel_range, gpuvm, mapping_epoch, bind_sequence, @@ -2194,6 +2198,12 @@ impl Vm { }) } + /// The range reserved for kernel-managed objects when this VM was created. + #[cfg_attr(not(CONFIG_DRM_ASAHI_M5), allow(dead_code))] + pub(crate) fn kernel_range(&self) -> Range { + self.kernel_range.clone() + } + /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { self.status = Some(Arc::pin_init( From 5d499059f29e59482ed8cb2ae1812e7eced4010f Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 07:26:20 +0100 Subject: [PATCH 062/117] drm/asahi: mmu: Add M5 application contexts The M5 (T8142) 25G76 firmware runs application work in a TTBAT context whose lower half is the client VM's root and whose upper half is an empty root of its own; the firmware's upper mappings are not shared with applications. The context is claimed when the work is set up and published explicitly once it is prepared, and a later client takes it over without giving up the context number. Add M5ExecutionContext for that, built only with the M5 backend. It claims numbers from the execution context mask, publishes and retracts under the handoff lock, and a mapping change reaches it through the existing invalidation of every context that names a VM's root. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/mmu.rs | 4 + drivers/gpu/drm/asahi/mmu/m5_context.rs | 173 ++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 drivers/gpu/drm/asahi/mmu/m5_context.rs diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index b36ceaa1dcc8c..fbf035af02916 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -113,11 +113,15 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; mod lifetime; +#[cfg(CONFIG_DRM_ASAHI_M5)] +mod m5_context; mod ppl; mod residency; mod shared; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; +#[cfg(CONFIG_DRM_ASAHI_M5)] +pub(crate) use m5_context::M5ExecutionContext; pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region diff --git a/drivers/gpu/drm/asahi/mmu/m5_context.rs b/drivers/gpu/drm/asahi/mmu/m5_context.rs new file mode 100644 index 0000000000000..5548d5d0dab6c --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/m5_context.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 (T8142) application contexts. +//! +//! The 25G76 firmware runs application work in a context whose lower half is the client VM's +//! root and whose upper half is a private, empty root: the firmware's own upper mappings are +//! not shared into application contexts. The context is claimed when a queue is set up and its +//! roots are published explicitly, once the work that names it has been prepared. Mapping +//! changes reach a published context through [`VmInner::tlbi_contexts_naming_root`]. + +use super::*; + +/// A claimed M5 application context and the VMs it names. +pub(crate) struct M5ExecutionContext { + inner: Arc, + id: u8, + low: Vm, + high: Vm, + published: bool, +} + +impl Uat { + /// Claims an application context for `vm`, a client VM of this UAT. Nothing is published. + pub(crate) fn new_m5_execution_context(&self, vm: &Vm) -> Result { + self.check_client_vm(vm)?; + let id = { + let mut inner = self.inner.lock(); + let free = !inner.execution_contexts & EXECUTION_CONTEXT_MASK; + if free == 0 { + return Err(ENOSPC); + } + let id = free.trailing_zeros() as usize; + inner.execution_contexts |= 1u64 << id; + id + }; + let high = match self.new_vm(0x8142_1000 + id as u64, self.geometry.user_range()) { + Ok(high) => high, + Err(error) => { + self.inner.lock().execution_contexts &= !(1u64 << id); + return Err(error); + } + }; + Ok(M5ExecutionContext { + inner: self.inner.clone(), + id: id as u8, + low: vm.clone(), + high, + published: false, + }) + } + + fn check_client_vm(&self, vm: &Vm) -> Result { + let owner = vm.gpuvm.lock_inner(); + if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) + { + return Err(EINVAL); + } + Ok(()) + } +} + +impl M5ExecutionContext { + /// The client VM named as the lower half. + pub(crate) fn vm(&self) -> &Vm { + &self.low + } + + /// The hardware context number, as the firmware records name it. + pub(crate) fn context_id(&self) -> u8 { + self.id + } + + /// The TTBAT words of this context. + pub(crate) fn roots(&self) -> [u64; 2] { + let id = usize::from(self.id); + [tagged_root(self.low.ttb(), id), tagged_root(self.high.ttb(), id)] + } + + /// Publishes the roots. A slot that is not empty is never replaced. + pub(crate) fn publish(&mut self) -> Result { + if self.published { + return Err(EBUSY); + } + let [low, high] = self.roots(); + let id = usize::from(self.id); + { + let inner = self.inner.lock(); + inner.handoff_lock(); + let slot = &inner.ttbs()[id]; + let empty = + slot.ttb0.load(Ordering::Acquire) == 0 && slot.ttb1.load(Ordering::Acquire) == 0; + if empty { + slot.ttb0.store(low, Ordering::Release); + slot.ttb1.store(high, Ordering::Release); + inner.publish_slot(id); + } + inner.handoff_unlock(); + if !empty { + return Err(EBUSY); + } + } + self.published = true; + mem::sync(); + mem::tlbi_asid(self.id); + mem::sync(); + Ok(()) + } + + /// Whether this context's roots are published and still in place. + pub(crate) fn is_current(&self) -> bool { + if !self.published { + return false; + } + let [low, high] = self.roots(); + let inner = self.inner.lock(); + let slot = &inner.ttbs()[usize::from(self.id)]; + slot.ttb0.load(Ordering::Acquire) == low && slot.ttb1.load(Ordering::Acquire) == high + } + + /// Names another client VM of the same UAT, keeping the context number. Published roots + /// are retracted first; the caller has made sure no work of the previous VM is outstanding. + pub(crate) fn rebind(&mut self, vm: &Vm) -> Result { + { + let owner = vm.gpuvm.lock_inner(); + if owner.is_kernel + || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) + { + return Err(EINVAL); + } + } + self.retract(); + self.low = vm.clone(); + Ok(()) + } + + fn retract(&mut self) { + if !self.published { + return; + } + let [low, high] = self.roots(); + let id = usize::from(self.id); + { + let inner = self.inner.lock(); + inner.handoff_lock(); + let slot = &inner.ttbs()[id]; + // Only our own pair is retracted. + if slot.ttb0.load(Ordering::Acquire) == low && slot.ttb1.load(Ordering::Acquire) == high + { + slot.ttb0.store(0, Ordering::Release); + slot.ttb1.store(0, Ordering::Release); + inner.publish_slot(id); + } + inner.handoff_unlock(); + } + mem::sync(); + mem::tlbi_asid(self.id); + mem::sync(); + self.published = false; + } +} + +impl Drop for M5ExecutionContext { + fn drop(&mut self) { + self.retract(); + let id = usize::from(self.id); + let mut inner = self.inner.lock(); + let slot = &inner.ttbs()[id]; + // A slot someone else filled is not ours to hand out again: keep it claimed. + if slot.ttb0.load(Ordering::Acquire) == 0 && slot.ttb1.load(Ordering::Acquire) == 0 { + inner.execution_contexts &= !(1u64 << id); + } + } +} From 5455cbb3f0fc3c94651bfe2f0c29c2caa027e299 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 07:26:25 +0100 Subject: [PATCH 063/117] drm/asahi: regs: Add the M5 PMGR boot control The native 25G76 driver sets the low bit of the GPU's PMGR boot control word (SGX + 0xe90000) before either M5 firmware instance starts; the word reads 0x30000 on J813. Add the write, for the M5 backend only, refusing any other starting value. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/regs.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 2b04726e43878..7ed0d552a4879 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -770,6 +770,26 @@ impl G17Resources { self.sgx_read32::() } + /// M5: set the low bit of the PMGR boot control word at SGX+0xe90000, as the native + /// 25G76 driver does before either firmware role starts. The word reads 0x30000 (or + /// 0x30001 when already set) on J813; anything else is refused. + #[cfg(CONFIG_DRM_ASAHI_M5)] + pub(crate) fn m5_match_pmgr_boot_control(&self) -> Result { + const OFFSET: usize = 0xe90000; + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let before = sgx.try_read32(OFFSET)?; + if before != 0x30000 && before != 0x30001 { + dev_err!(self.dev.as_ref(), "M5: unexpected PMGR boot control {:#x}\n", before); + return Err(EIO); + } + sgx.try_write32(before | 1, OFFSET)?; + if sgx.try_read32(OFFSET)? != 0x30001 { + return Err(EIO); + } + Ok(()) + } + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. pub(crate) fn enable_g17_gating(&self) -> Result { From 9ddfb406d442068d11232db1bbaf0152f1ddfdd1 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 07:27:08 +0100 Subject: [PATCH 064/117] drm/asahi: Add the M5 (T8142) backend Add a backend for the M5 GPU (J813, G17G) running the 25G76 firmware (RTKit-3255.160.4), as ProbeConfig::M5 for apple,agx-t8142 behind CONFIG_DRM_ASAHI_M5. It changes nothing for the other SoCs. The 25G76 firmware has two coprocessors, GFX and GFX1, as the A18 Pro has, but a HAL300 interface that differs from the 26A428 and 25G83 builds the shared G17 code runs. The backend therefore builds its own initialization graph and talks to the firmware over its own queues, while using the shared parts of the driver: - the UAT is the shared one with published roots, adopted and mirrored as on the A18 Pro, plus the M5 application contexts; - user VMs carry a status and a job lifetime, and the backend uses the tracked submission model of file.rs; - the UAPI is the shared one. The backend reports VM_STATUS and SOFT_FAULTS and one hardware command per submission. The boot loader (aura) loads the firmware, checks both TEXT images against the 25G76 build and says so in /chosen/aurora,m5-gpu-firmware-profile; the backend refuses to start without it. One render pair and one compute queue serve all clients and are handed from client VM to client VM; a submission is polled to retirement before the ioctl returns. Renders and compute are admitted only within the state the firmware programs are known to handle, and refused with ENOTSUPP otherwise. There is no recovery yet: a job that fails stops both coprocessors and retains every firmware-visible object until reboot. The driver's pools are mapped at their own address in each client VM the job is handed to, except in the VM that owns them, and the compute storage is reserved together with the parameter buffer, so that its address is free in every such VM. Tested on J813 (M5, 8 GPU cores) on the aura one-shot trial boot, with Mesa j813, against the same Mesa on the J813 kernel this code comes from: GLES2 16016 pass, 930 fail (identical per case); GLES3 40832 pass, 3547 fail, 2 timeouts (identical but for one known flake); GLES31 32813 pass, 4517 fail (t70: 32804/4520; the 32 differing cases are 9 fail->pass and flakes); the sway desktop renders on the GPU from boot, also during the CTS runs. This consolidates the M5 work of Acelogic and the G17P render, UAPI and initdata encoders of DjDeveloperr , with contributions by Ryan Murray, Eryk Wieliczko and Chris Kearney, from the J813 kernel tree, ported onto integration's driver. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/Kconfig | 8 + drivers/gpu/drm/asahi/asahi_core.rs | 6 + drivers/gpu/drm/asahi/driver.rs | 41 + drivers/gpu/drm/asahi/m5/channel.rs | 183 ++ drivers/gpu/drm/asahi/m5/command.rs | 1233 +++++++++++++ drivers/gpu/drm/asahi/m5/completion.rs | 61 + drivers/gpu/drm/asahi/m5/compute.rs | 246 +++ drivers/gpu/drm/asahi/m5/drm.rs | 386 ++++ drivers/gpu/drm/asahi/m5/fw_msg.rs | 57 + drivers/gpu/drm/asahi/m5/initdata.rs | 1630 +++++++++++++++++ drivers/gpu/drm/asahi/m5/live_boot.rs | 527 ++++++ drivers/gpu/drm/asahi/m5/mapped.rs | 145 ++ drivers/gpu/drm/asahi/m5/mod.rs | 91 + drivers/gpu/drm/asahi/m5/provider.rs | 199 ++ drivers/gpu/drm/asahi/m5/render.rs | 508 +++++ drivers/gpu/drm/asahi/m5/render_desc.rs | 1138 ++++++++++++ drivers/gpu/drm/asahi/m5/render_target.rs | 72 + drivers/gpu/drm/asahi/m5/resources.rs | 631 +++++++ .../asahi/m5/resources/application_compute.rs | 305 +++ .../drm/asahi/m5/resources/firmware_view.rs | 75 + .../drm/asahi/m5/resources/graphics_job.rs | 512 ++++++ .../drm/asahi/m5/resources/graphics_memory.rs | 198 ++ .../drm/asahi/m5/resources/graphics_queues.rs | 261 +++ .../drm/asahi/m5/resources/graphics_usc.rs | 203 ++ .../asahi/m5/resources/parameter_buffer.rs | 139 ++ .../drm/asahi/m5/resources/target_memory.rs | 75 + drivers/gpu/drm/asahi/m5/sksm.rs | 271 +++ drivers/gpu/drm/asahi/m5/submission.rs | 51 + drivers/gpu/drm/asahi/m5/vm_layout.rs | 38 + include/uapi/drm/asahi_drm.h | 5 +- 30 files changed, 9293 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/asahi/m5/channel.rs create mode 100644 drivers/gpu/drm/asahi/m5/command.rs create mode 100644 drivers/gpu/drm/asahi/m5/completion.rs create mode 100644 drivers/gpu/drm/asahi/m5/compute.rs create mode 100644 drivers/gpu/drm/asahi/m5/drm.rs create mode 100644 drivers/gpu/drm/asahi/m5/fw_msg.rs create mode 100644 drivers/gpu/drm/asahi/m5/initdata.rs create mode 100644 drivers/gpu/drm/asahi/m5/live_boot.rs create mode 100644 drivers/gpu/drm/asahi/m5/mapped.rs create mode 100644 drivers/gpu/drm/asahi/m5/mod.rs create mode 100644 drivers/gpu/drm/asahi/m5/provider.rs create mode 100644 drivers/gpu/drm/asahi/m5/render.rs create mode 100644 drivers/gpu/drm/asahi/m5/render_desc.rs create mode 100644 drivers/gpu/drm/asahi/m5/render_target.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/application_compute.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/firmware_view.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/graphics_job.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/graphics_memory.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/graphics_queues.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/graphics_usc.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/parameter_buffer.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/target_memory.rs create mode 100644 drivers/gpu/drm/asahi/m5/sksm.rs create mode 100644 drivers/gpu/drm/asahi/m5/submission.rs create mode 100644 drivers/gpu/drm/asahi/m5/vm_layout.rs diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 6b28266b6b4d1..af5e7e9e8cb87 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -43,6 +43,14 @@ config DRM_ASAHI_G16 An aborted receipt disables that command class until reboot. Suspend, hibernation and kexec are blocked while native graphics owns DMA. +config DRM_ASAHI_M5 + bool "Apple M5 (G17G, T8142) support (experimental)" + depends on DRM_ASAHI && ARM64 + help + Backend for the M5 GPU (J813 MacBook Air) running the 25G76 + firmware (RTKit-3255.160.4), as loaded and published by the aura + boot loader (/chosen/aurora,m5-gpu-firmware-profile). + config DRM_ASAHI_DEBUG_ALLOCATOR bool "Use debug allocator" depends on DRM_ASAHI diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index decf16ea09647..17b64ec578d5f 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -22,6 +22,8 @@ mod gem; #[cfg(CONFIG_DRM_ASAHI_G16)] mod g16; +#[cfg(CONFIG_DRM_ASAHI_M5)] +mod m5; mod gpu; mod hw; mod initdata; @@ -198,6 +200,10 @@ module! { default: 1, description: "Count held GPU memory in its creator's RSS for the OOM killer (1 = on)", }, + m5_fw_trace: u32 { + default: 0, + description: "M5: firmware trace classes (power bit 0, submission bit 1, recovery classifier bit 9)", + }, g16_deadline_ms: u32 { default: 5000, description: "G16: administrative completion-stamp deadline, ms (0 disables; long running work can exceed it)", diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index ff82d8c118e85..554a82285c865 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -142,6 +142,8 @@ pub(crate) enum ProbeConfig { Legacy(&'static hw::HwConfig), G17(&'static g17::Config), M3(crate::m3::Chip), + /// The M5 (T8142) backend, refused unless it is built. + M5, } // OF Device ID table. Linux 7.3's of_device_table!() takes no attributes on @@ -165,6 +167,10 @@ macro_rules! asahi_of_device_table { of::DeviceId::new(c"apple,agx-t6041"), ProbeConfig::G17(&hw::t6041::CONFIG) ), + ( + of::DeviceId::new(c"apple,agx-t8142"), + ProbeConfig::M5 + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) @@ -253,6 +259,10 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::Legacy(cfg) => *cfg, ProbeConfig::G17(cfg) => return AsahiPlatformDriver::probe_g17(pdev, cfg), ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), + #[cfg(CONFIG_DRM_ASAHI_M5)] + ProbeConfig::M5 => return AsahiPlatformDriver::probe_m5(pdev), + #[cfg(not(CONFIG_DRM_ASAHI_M5))] + ProbeConfig::M5 => return Err(ENODEV), }; // SAFETY: The selected SoC's UAT output width bounds all device DMA. @@ -444,6 +454,37 @@ impl AsahiPlatformDriver { } } +#[cfg(CONFIG_DRM_ASAHI_M5)] +impl AsahiPlatformDriver { + fn probe_m5<'bound>( + pdev: &'bound platform::Device>, + ) -> Result> { + // SAFETY: The M5 UAT has 42 output bits. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; + let resources = regs::Resources::new(pdev)?; + let res = regs::G17Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let gpu = crate::m5::probe(pdev, &drm, res)?; + let data = try_pin_init!(AsahiDrmRegistrationData { + gpu: gpu as Arc, + resources, + }); + // SAFETY: The registration is stored and unregistered by platform data destruction. + // The M5 firmware session retains its firmware-visible state until reboot. + let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + Ok(AsahiPlatformDriverData { + _device: reg.device().into(), + _reg: reg, + g17: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, + raw_dev: core::ptr::null_mut(), + raw_stats: core::ptr::null_mut(), + stats: None, + }) + } +} + #[cfg(CONFIG_DRM_ASAHI_G16)] impl AsahiPlatformDriver { pub(crate) fn probe_g16<'bound>( diff --git a/drivers/gpu/drm/asahi/m5/channel.rs b/drivers/gpu/drm/asahi/m5/channel.rs new file mode 100644 index 0000000000000..259d79209a0f7 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/channel.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8142/25G76 channel configuration, separate from Neo's HAL200 path. +//! +//! Native AGXCLChannelSKSM::submitBuffer inlines this packed 0x86-byte +//! record at 0xfffffe000898488c..0x8984bd8. The controlled native sequence +//! is a selector-3 register program, this tag-15 record, then a tag-14 kick. +//! These builders do not map memory, install a queue or publish a producer. +//! Every pointer and event identifier must belong to the retained session. + +pub(crate) const M5_CHANNEL_CONFIG_BYTES: usize = 0x86; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5ChannelError { + QueueId, + Priority, + Address, + Context, + DataMaster, +} + +/// Optional TA parameter-buffer event, separate from the first event tuple. +/// The native TA writer at 25G76 0x8a249ac..0x8a249e4 obtains these values +/// from its retained PM manager and command descriptor. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ParameterBufferEvent { + pub(crate) address: u64, + pub(crate) timestamp: u64, + /// TA descriptor +0x1218, independently selected by the owned PB table. + pub(crate) hardware_buffer_id: u32, + /// Native PM manager +0x79c. This is not the SKSM queue ID. + pub(crate) manager_id: u32, +} + +/// Optional retained event-machine object, with the native timestamp and +/// index. None emits the native disabled branch's full-width sentinels. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ChannelEvent { + pub(crate) address: u64, + pub(crate) timestamp: u64, + pub(crate) index: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ChannelConfig { + pub(crate) ring_low: u64, + pub(crate) ring_high: u64, + pub(crate) queue_id: u8, + pub(crate) priority: u8, + /// Native descriptor +0x181 bit 0. Mode 3 validates the first queue + /// installation on M5; an already-installed native capture has 0. + pub(crate) install_queue: bool, + /// Native descriptor +0x144, independently matched to the UAT context. + pub(crate) gpu_context: u8, + pub(crate) event: Option, + /// AGXCommandQueue +0x8b8: retained scheduler/control object. + pub(crate) channel_control: u64, + /// Native descriptor +0x2dc bit 0. + pub(crate) update_scheduler: bool, + /// AGXCommandQueue +0x898 object's +0x10. Not the GPU context ID. + pub(crate) scheduler_index: u32, + /// getStampIndex on descriptor +0x170. Not a fixed target constant. + pub(crate) channel_stamp_index: u32, + /// Native descriptor +0x180 bit 0. + pub(crate) update_context: bool, + /// Descriptor +0x58 owner's virtual context value. Distinct from UAT. + pub(crate) owner_context: u32, + /// Native channel-state +0x84 is nonzero. + pub(crate) extended_command: bool, +} + +fn firmware_address(address: u64) -> bool { + address >= 0xffff_fc00_0000_0000 && address & 7 == 0 +} + +fn put32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +pub(crate) fn encode_m5_channel_config( + p: M5ChannelConfig, +) -> Result<[u8; M5_CHANNEL_CONFIG_BYTES], M5ChannelError> { + encode_m5_channel_config_for(p, 2, 2, None, None) +} + +/// The two selectors are independent: native TA/fragment use 0/1 at +0x22 +/// while both retain 2 at +0x5e. Changing both by stage breaks that contract. +/// gpu_state_update retains the separate +0x26 flag/+0x2a value pair; +/// the native TA writer copies descriptor +0xe41/+0xe48 to these fields. +pub(crate) fn encode_m5_channel_config_for( + p: M5ChannelConfig, + data_master: u32, + channel_data_master: u32, + gpu_state_update: Option, + parameter_buffer: Option, +) -> Result<[u8; M5_CHANNEL_CONFIG_BYTES], M5ChannelError> { + if data_master > 2 + || channel_data_master > 2 + || (parameter_buffer.is_some() && data_master != 0) + { + return Err(M5ChannelError::DataMaster); + } + if p.queue_id > 127 { + return Err(M5ChannelError::QueueId); + } + if p.priority > 3 { + return Err(M5ChannelError::Priority); + } + if p.gpu_context == 0 { + return Err(M5ChannelError::Context); + } + if p.ring_low == 0 + || p.ring_low >= (1 << 42) + || p.ring_low & 0x3fff != 0 + || !firmware_address(p.ring_high) + || p.ring_high & 0x3fff != 0 + || !firmware_address(p.channel_control) + { + return Err(M5ChannelError::Address); + } + if let Some(event) = p.event { + if !firmware_address(event.address) { + return Err(M5ChannelError::Address); + } + } + if let Some(event) = parameter_buffer { + if !firmware_address(event.address) { + return Err(M5ChannelError::Address); + } + } + let mut bytes = [0; M5_CHANNEL_CONFIG_BYTES]; + put32(&mut bytes, 0, 15); + put64(&mut bytes, 8, p.ring_low); + put64(&mut bytes, 0x10, p.ring_high); + bytes[0x18..0x1a].copy_from_slice(&u16::from(p.queue_id).to_le_bytes()); + for (offset, value) in [ + (0x1a, u32::from(p.install_queue)), + (0x1e, u32::from(p.priority)), + (0x22, data_master), + (0x32, u32::from(p.gpu_context)), + (0x52, u32::from(p.update_scheduler)), + (0x56, p.scheduler_index), + (0x5a, p.channel_stamp_index), + (0x5e, channel_data_master), + (0x62, u32::from(p.update_context)), + (0x66, p.owner_context), + (0x6a, u32::from(p.extended_command)), + ] { + put32(&mut bytes, offset, value); + } + if let Some(value) = gpu_state_update { + put32(&mut bytes, 0x26, 1); + put64(&mut bytes, 0x2a, value); + } + match p.event { + Some(event) => { + put64(&mut bytes, 0x36, event.address); + put64(&mut bytes, 0x3e, event.timestamp); + put32(&mut bytes, 0x46, event.index); + } + None => { + put64(&mut bytes, 0x3e, u64::MAX); + put32(&mut bytes, 0x46, u32::MAX); + } + } + put64(&mut bytes, 0x4a, p.channel_control); + if let Some(event) = parameter_buffer { + put64(&mut bytes, 0x6e, event.address); + put64(&mut bytes, 0x76, event.timestamp); + put32(&mut bytes, 0x7e, event.hardware_buffer_id); + put32(&mut bytes, 0x82, event.manager_id); + } else { + bytes[0x76..0x86].fill(0xff); + } + Ok(bytes) +} + diff --git a/drivers/gpu/drm/asahi/m5/command.rs b/drivers/gpu/drm/asahi/m5/command.rs new file mode 100644 index 0000000000000..c4555602c781e --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/command.rs @@ -0,0 +1,1233 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +use crate::m5::{render_desc, submission}; + +use render_desc::{ + G17pRenderParameters, RenderBuildError, +}; + +pub(crate) const UAPI_COMMAND_HEADER_SIZE: usize = 8; + +pub(crate) const UAPI_RENDER_SIZE: usize = 272; + +pub(crate) const UAPI_COMPUTE_SIZE: usize = 88; + +pub(crate) const UAPI_G18_COMPUTE_SIZE: usize = 136; + +pub(crate) const COMPUTE_G18_RESOURCES: u32 = 1 << 2; + +/// `drm_asahi_cmd_render_g18`: the render payload plus its 80-byte suffix. +pub(crate) const UAPI_G18_RENDER_SIZE: usize = UAPI_RENDER_SIZE + 80; + +/// `DRM_ASAHI_RENDER_G18_RESOURCES`. Only the G18 adapter may consume and +/// strip it; the shared translator's RENDER_FLAGS keeps refusing it. +pub(crate) const RENDER_G18_RESOURCES: u32 = 1 << 21; + +pub(crate) const UAPI_ATTACHMENT_SIZE: usize = 24; + +pub(crate) const UAPI_MAX_ATTACHMENTS: usize = 16; + +pub(crate) const UAPI_MAX_HARDWARE_COMMANDS: u32 = 64; + +pub(crate) const M5_MAX_RENDER_DIMENSION: u32 = 16384; + +pub(crate) const UAPI_BARRIER_NONE: u16 = 0xffff; + +pub(crate) const USC_WINDOW_SIZE: u64 = 1 << 32; + +pub(crate) const COMPUTE_G17P_ADD3_PROOF: u32 = + kernel::uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF as u32; + +const G17P_ADD3_BUFFER_BYTES: u64 = 64 * 4; + +const COMMAND_RENDER: u16 = 0; + +const COMMAND_COMPUTE: u16 = 1; + +const SET_VERTEX_ATTACHMENTS: u16 = 2; + +const SET_FRAGMENT_ATTACHMENTS: u16 = 3; + +const SET_COMPUTE_ATTACHMENTS: u16 = 4; + +const RENDER_VERTEX_SCRATCH: u32 = 1 << 0; + +const RENDER_PROCESS_EMPTY_TILES: u32 = 1 << 1; + +const RENDER_NO_VERTEX_CLUSTERING: u32 = 1 << 2; + +const RENDER_RSRC_SPEC_HI: u32 = 1 << 4; + +const RENDER_DBIAS_IS_INT: u32 = 1 << 18; + +const RENDER_FLAGS: u32 = RENDER_VERTEX_SCRATCH + | RENDER_PROCESS_EMPTY_TILES + | RENDER_NO_VERTEX_CLUSTERING + | RENDER_RSRC_SPEC_HI + | RENDER_DBIAS_IS_INT + // Private, opt-in matched-workload diagnostic. Ordinary builds reject it. + | if cfg!(g17p_aux_fb_small_probe) { 1 << 31 } else { 0 } + | if cfg!(g17p_mcache_aperture_probe) { 1 << 30 } else { 0 }; + +const SAMPLER_SIZE: u64 = 8; + +const MAX_SAMPLERS: u32 = 1024; + +const PAGE_SIZE: u64 = 0x4000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum UapiTranslateError { + HeaderTruncated, + PayloadTruncated, + PayloadTail, + CommandType, + AttachmentPayloadSize, + AttachmentCount, + AttachmentFields, + AttachmentBarrier, + HardwareCommandCount, + BarrierFuture, + QueueUscAlignment, + QueueUscRange, + AddressRange, + AddressAlignment, + FlagsUnknownBits, + FlagsUnsupported, + Dimensions, + Layers, + UtileDimensions, + Samples, + TilebufferSize, + SamplerCount, + SamplerPair, + HelperProgram, + Add3ProofAttachments, + ProgramOffset, + ZlsFields, + RenderBuild(RenderBuildError), +} + +impl From for UapiTranslateError { + fn from(value: RenderBuildError) -> Self { + Self::RenderBuild(value) + } +} + +impl UapiTranslateError { +} + +fn get_u16(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) +} + +fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) +} + +fn padded_payload(raw: &[u8]) -> Result<[u8; N], UapiTranslateError> { + if raw.len() > N && raw[N..].iter().any(|byte| *byte != 0) { + return Err(UapiTranslateError::PayloadTail); + } + let mut out = [0u8; N]; + let count = core::cmp::min(raw.len(), N); + out[..count].copy_from_slice(&raw[..count]); + Ok(out) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiCommandHeader { + pub(crate) command_type: u16, + pub(crate) size: u16, + pub(crate) render_barrier: u16, + pub(crate) compute_barrier: u16, +} + +impl UapiCommandHeader { + fn parse(raw: &[u8]) -> Result { + if raw.len() < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + Ok(Self { + command_type: get_u16(raw, 0), + size: get_u16(raw, 2), + render_barrier: get_u16(raw, 4), + compute_barrier: get_u16(raw, 6), + }) + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct UapiAttachment { + pub(crate) address: u64, + pub(crate) size: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiAttachmentList { + pub(crate) entries: [UapiAttachment; UAPI_MAX_ATTACHMENTS], + pub(crate) count: u8, +} + +impl UapiAttachmentList { + pub(crate) const EMPTY: Self = Self { + entries: [UapiAttachment { + address: 0, + size: 0, + }; UAPI_MAX_ATTACHMENTS], + count: 0, + }; + + pub(crate) fn as_slice(&self) -> &[UapiAttachment] { + &self.entries[..self.count as usize] + } + + fn parse(raw: &[u8]) -> Result { + if raw.len() % UAPI_ATTACHMENT_SIZE != 0 { + return Err(UapiTranslateError::AttachmentPayloadSize); + } + let count = raw.len() / UAPI_ATTACHMENT_SIZE; + if count > UAPI_MAX_ATTACHMENTS { + return Err(UapiTranslateError::AttachmentCount); + } + let mut out = Self::EMPTY; + for index in 0..count { + let offset = index * UAPI_ATTACHMENT_SIZE; + if get_u32(raw, offset + 0x10) != 0 || get_u32(raw, offset + 0x14) != 0 { + return Err(UapiTranslateError::AttachmentFields); + } + out.entries[index] = UapiAttachment { + address: get_u64(raw, offset), + size: get_u64(raw, offset + 8), + }; + } + out.count = count as u8; + Ok(out) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiHelperProgram { + pub(crate) binary: u32, + pub(crate) config: u32, + pub(crate) data: u64, +} + +fn helper(raw: &[u8], offset: usize) -> UapiHelperProgram { + UapiHelperProgram { + binary: get_u32(raw, offset), + config: get_u32(raw, offset + 4), + data: get_u64(raw, offset + 8), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiProgram { + pub(crate) usc: u32, + pub(crate) resource_spec: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamp { + pub(crate) handle: u32, + pub(crate) offset: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamps { + pub(crate) start: UapiTimestamp, + pub(crate) end: UapiTimestamp, +} + +fn timestamps(raw: &[u8], offset: usize) -> UapiTimestamps { + UapiTimestamps { + start: UapiTimestamp { + handle: get_u32(raw, offset), + offset: get_u32(raw, offset + 4), + }, + end: UapiTimestamp { + handle: get_u32(raw, offset + 8), + offset: get_u32(raw, offset + 12), + }, + } +} + +fn program(raw: &[u8], offset: usize, high_offset: usize) -> UapiProgram { + UapiProgram { + usc: get_u32(raw, offset), + resource_spec: get_u32(raw, offset + 4) as u64 + | (get_u32(raw, high_offset) as u64) << 32, + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiZlsBuffer { + pub(crate) base: u64, + pub(crate) compression_base: u64, + pub(crate) stride: u32, + pub(crate) compression_stride: u32, +} + +fn zls(raw: &[u8], offset: usize) -> UapiZlsBuffer { + UapiZlsBuffer { + base: get_u64(raw, offset), + compression_base: get_u64(raw, offset + 8), + stride: get_u32(raw, offset + 0x10), + compression_stride: get_u32(raw, offset + 0x14), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiRenderCommand { + pub(crate) flags: u32, + pub(crate) depth_dimensions: u32, + pub(crate) vdm_base: u64, + pub(crate) vertex_helper: UapiHelperProgram, + pub(crate) fragment_helper: UapiHelperProgram, + pub(crate) scissor_base: u64, + pub(crate) depth_bias_base: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth: UapiZlsBuffer, + pub(crate) stencil: UapiZlsBuffer, + pub(crate) zls_control: u64, + pub(crate) multisample_control: u64, + pub(crate) sampler_heap: u64, + pub(crate) ppp_control: u32, + pub(crate) width: u16, + pub(crate) height: u16, + pub(crate) layers: u16, + pub(crate) sampler_count: u16, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) samples: u8, + pub(crate) sample_size: u8, + pub(crate) merge_upper_x: u32, + pub(crate) merge_upper_y: u32, + pub(crate) background: UapiProgram, + pub(crate) end_of_tile: UapiProgram, + pub(crate) partial_background: UapiProgram, + pub(crate) partial_end_of_tile: UapiProgram, + pub(crate) depth_clear: u32, + pub(crate) stencil_clear: u32, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, + pub(crate) prior_vdm: u64, + pub(crate) prior_cdm: u64, + /// `drm_asahi_g18_render_resources` words; zero unless flagged. + pub(crate) g18_resources: [u32; 20], +} + +impl UapiRenderCommand { + fn parse(raw: &[u8]) -> Result { + // Legacy payloads are zero-extended exactly as before. A flagged G18 + // suffix must be complete; padding could hide a truncated field. + let has_resources = raw.len() >= 4 && get_u32(raw, 0) & RENDER_G18_RESOURCES != 0; + if has_resources && raw.len() < UAPI_G18_RENDER_SIZE { + return Err(UapiTranslateError::PayloadTruncated); + } + if !has_resources && raw.len() > UAPI_RENDER_SIZE + && raw[UAPI_RENDER_SIZE..].iter().any(|byte| *byte != 0) + { + return Err(UapiTranslateError::PayloadTail); + } + let raw = padded_payload::(raw)?; + let mut g18_resources = [0; 20]; + if has_resources { + for (i, word) in g18_resources.iter_mut().enumerate() { + *word = get_u32(&raw, UAPI_RENDER_SIZE + i * 4); + } + } + Ok(Self { + flags: get_u32(&raw, 0x00), + depth_dimensions: get_u32(&raw, 0x04), + vdm_base: get_u64(&raw, 0x08), + vertex_helper: helper(&raw, 0x10), + fragment_helper: helper(&raw, 0x20), + scissor_base: get_u64(&raw, 0x30), + depth_bias_base: get_u64(&raw, 0x38), + occlusion_query_base: get_u64(&raw, 0x40), + depth: zls(&raw, 0x48), + stencil: zls(&raw, 0x60), + zls_control: get_u64(&raw, 0x78), + multisample_control: get_u64(&raw, 0x80), + sampler_heap: get_u64(&raw, 0x88), + ppp_control: get_u32(&raw, 0x90), + width: get_u16(&raw, 0x94), + height: get_u16(&raw, 0x96), + layers: get_u16(&raw, 0x98), + sampler_count: get_u16(&raw, 0x9a), + utile_width: raw[0x9c], + utile_height: raw[0x9d], + samples: raw[0x9e], + sample_size: raw[0x9f], + merge_upper_x: get_u32(&raw, 0xa0), + merge_upper_y: get_u32(&raw, 0xa4), + background: program(&raw, 0xa8, 0xf8), + end_of_tile: program(&raw, 0xb0, 0xf0), + partial_background: program(&raw, 0xb8, 0xfc), + partial_end_of_tile: program(&raw, 0xc0, 0xf4), + depth_clear: get_u32(&raw, 0xc8), + stencil_clear: get_u32(&raw, 0xcc), + vertex_timestamps: timestamps(&raw, 0xd0), + fragment_timestamps: timestamps(&raw, 0xe0), + prior_vdm: get_u64(&raw, 0x100), + prior_cdm: get_u64(&raw, 0x108), + g18_resources, + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiComputeCommand { + pub(crate) flags: u32, + pub(crate) sampler_count: u32, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) sampler_heap: u64, + pub(crate) helper: UapiHelperProgram, + pub(crate) timestamps: UapiTimestamps, + pub(crate) scs_layout: u64, + pub(crate) g18_resources: [u32; 16], + pub(crate) prior_vdm: u64, + pub(crate) prior_cdm: u64, +} + +impl UapiComputeCommand { + fn parse(raw: &[u8]) -> Result { + // Legacy compute payloads are zero-extended as before. A flagged + // metadata payload must be complete; padding it could hide truncation + // of a valid zero-valued field or reserved word. + let has_resources = raw.len() >= 4 && get_u32(raw, 0) & COMPUTE_G18_RESOURCES != 0; + if has_resources && raw.len() < UAPI_G18_COMPUTE_SIZE { + return Err(UapiTranslateError::PayloadTruncated); + } + if !has_resources && raw.len() > UAPI_COMPUTE_SIZE + && raw[UAPI_COMPUTE_SIZE..].iter().any(|byte| *byte != 0) + { + return Err(UapiTranslateError::PayloadTail); + } + let raw = padded_payload::(raw)?; + let mut g18_resources = [0; 16]; + if has_resources { + for (i, word) in g18_resources.iter_mut().enumerate() { + *word = get_u32(&raw, 72 + i * 4); + } + } + Ok(Self { + flags: get_u32(&raw, 0), + sampler_count: get_u32(&raw, 4), + control_stream_base: get_u64(&raw, 8), + control_stream_end: get_u64(&raw, 0x10), + sampler_heap: get_u64(&raw, 0x18), + helper: helper(&raw, 0x20), + timestamps: timestamps(&raw, 0x30), + scs_layout: get_u64(&raw, 0x40), + g18_resources, + prior_vdm: if has_resources { 0 } else { get_u64(&raw, 0x48) }, + prior_cdm: if has_resources { 0 } else { get_u64(&raw, 0x50) }, + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ParsedHardwareCommand { + Render { + index: u32, + header: UapiCommandHeader, + payload: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + }, + Compute { + index: u32, + header: UapiCommandHeader, + payload: UapiComputeCommand, + attachments: UapiAttachmentList, + }, +} + +pub(crate) struct UapiCommandParser<'a> { + raw: &'a [u8], + offset: usize, + command_index: u32, + hardware_count: u32, + render_count: u16, + compute_count: u16, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + compute_attachments: UapiAttachmentList, +} + +impl<'a> UapiCommandParser<'a> { + pub(crate) const fn new(raw: &'a [u8]) -> Self { + Self { + raw, + offset: 0, + command_index: 0, + hardware_count: 0, + render_count: 0, + compute_count: 0, + vertex_attachments: UapiAttachmentList::EMPTY, + fragment_attachments: UapiAttachmentList::EMPTY, + compute_attachments: UapiAttachmentList::EMPTY, + } + } + + pub(crate) fn finish(&self) -> Result { + if self.offset != self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + if self.hardware_count == 0 || self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + Ok(self.hardware_count) + } + + pub(crate) fn next_hardware( + &mut self, + ) -> Result, UapiTranslateError> { + loop { + if self.offset == self.raw.len() { + return Ok(None); + } + if self.raw.len() - self.offset < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + let header = UapiCommandHeader::parse(&self.raw[self.offset..])?; + self.offset += UAPI_COMMAND_HEADER_SIZE; + let end = self + .offset + .checked_add(header.size as usize) + .ok_or(UapiTranslateError::PayloadTruncated)?; + if end > self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + let payload = &self.raw[self.offset..end]; + self.offset = end; + self.command_index += 1; + + match header.command_type { + COMMAND_RENDER | COMMAND_COMPUTE => { + if header.render_barrier != UAPI_BARRIER_NONE + && header.render_barrier > self.render_count + { + return Err(UapiTranslateError::BarrierFuture); + } + if header.compute_barrier != UAPI_BARRIER_NONE + && header.compute_barrier > self.compute_count + { + return Err(UapiTranslateError::BarrierFuture); + } + self.hardware_count += 1; + if self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + if header.command_type == COMMAND_RENDER { + self.render_count += 1; + return Ok(Some(ParsedHardwareCommand::Render { + index: self.command_index, + header, + payload: UapiRenderCommand::parse(payload)?, + vertex_attachments: self.vertex_attachments, + fragment_attachments: self.fragment_attachments, + })); + } + self.compute_count += 1; + return Ok(Some(ParsedHardwareCommand::Compute { + index: self.command_index, + header, + payload: UapiComputeCommand::parse(payload)?, + attachments: self.compute_attachments, + })); + } + SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS | SET_COMPUTE_ATTACHMENTS => { + if header.render_barrier != UAPI_BARRIER_NONE + || header.compute_barrier != UAPI_BARRIER_NONE + { + return Err(UapiTranslateError::AttachmentBarrier); + } + let attachments = UapiAttachmentList::parse(payload)?; + match header.command_type { + SET_VERTEX_ATTACHMENTS => self.vertex_attachments = attachments, + SET_FRAGMENT_ATTACHMENTS => self.fragment_attachments = attachments, + SET_COMPUTE_ATTACHMENTS => self.compute_attachments = attachments, + _ => unreachable!(), + } + } + _ => return Err(UapiTranslateError::CommandType), + } + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum GpuAccess { + Read, + Write, + ReadWrite, +} + +pub(crate) trait GpuAddressSpace { + fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool; +} + +fn require_range( + address_space: &A, + address: u64, + size: u64, + access: GpuAccess, +) -> Result<(), UapiTranslateError> { + let end = address + .checked_add(size) + .ok_or(UapiTranslateError::AddressRange)?; + if size == 0 || end <= address || !address_space.covers(address, size, access) { + return Err(UapiTranslateError::AddressRange); + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueueUscWindow { + pub(crate) base: u64, + pub(crate) user_start: u64, + pub(crate) user_end: u64, +} + +impl QueueUscWindow { + pub(crate) fn validate(self) -> Result<(), UapiTranslateError> { + if self.base & (USC_WINDOW_SIZE - 1) != 0 { + return Err(UapiTranslateError::QueueUscAlignment); + } + let end = self + .base + .checked_add(USC_WINDOW_SIZE) + .ok_or(UapiTranslateError::QueueUscRange)?; + if self.base < self.user_start || end > self.user_end || end <= self.base { + return Err(UapiTranslateError::QueueUscRange); + } + Ok(()) + } + + fn full_program( + self, + address_space: &A, + tagged_offset: u32, + ) -> Result { + self.validate()?; + if tagged_offset == 0 { + return Err(UapiTranslateError::ProgramOffset); + } + let allocation = self + .base + .checked_add((tagged_offset & !7) as u64) + .ok_or(UapiTranslateError::ProgramOffset)?; + require_range(address_space, allocation, 4, GpuAccess::Read)?; + // Honeykrisp carries resource metadata in the low three UAPI bits, + // but G17P's native 3D command strips those bits before programming + // every BG/EOT/partial USC address. Preserving bit 2 launched four + // bytes into the instruction stream and contradicted all complete + // Metal register-list captures. + Ok(allocation) + } +} + +fn validate_sampler( + address_space: &A, + heap: u64, + count: u32, +) -> Result<(), UapiTranslateError> { + if count > MAX_SAMPLERS { + return Err(UapiTranslateError::SamplerCount); + } + if (heap == 0) != (count == 0) { + return Err(UapiTranslateError::SamplerPair); + } + if count != 0 { + if heap & (SAMPLER_SIZE - 1) != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + heap, + count as u64 * SAMPLER_SIZE, + GpuAccess::Read, + )?; + } + Ok(()) +} + +fn validate_attachments( + address_space: &A, + attachments: &UapiAttachmentList, +) -> Result<(), UapiTranslateError> { + for attachment in attachments.as_slice() { + require_range( + address_space, + attachment.address, + attachment.size, + GpuAccess::Write, + )?; + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderInternalState { + pub(crate) parameters: G17pRenderParameters, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedRenderCommand { + /// Exact logical render prefix assigned by the DRM ioctl dispatcher. + pub(crate) render_order: submission::G17PRenderOrder, + /// Original UAPI render index, retained through admission. + pub(crate) render_barrier: u16, + /// Original UAPI compute index, retained through admission. + pub(crate) compute_barrier: u16, + /// Exact compute prefix resolved at ioctl admission. Later independent + /// compute commands must not become parents of this render. + pub(crate) compute_wait_through: Option, + pub(crate) flags: u32, + pub(crate) vdm_base: u64, + pub(crate) parameters: G17pRenderParameters, + pub(crate) vertex_attachments: UapiAttachmentList, + pub(crate) fragment_attachments: UapiAttachmentList, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, +} + +fn validate_zls( + address_space: &A, + value: UapiZlsBuffer, + layers: u16, + g18_subpage: bool, +) -> Result<(), UapiTranslateError> { + if value.base == 0 + && (value.compression_base != 0 || value.stride != 0 || value.compression_stride != 0) + { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_base == 0 && value.compression_stride != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if !g18_subpage && layers > 1 && value.base != 0 && value.stride == 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.stride != 0 && value.stride & 0x3fff != 1 + && !(g18_subpage && value.stride & 0x3fff == 0) { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_stride & 0x3fff != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.base != 0 { + let layer_stride = if g18_subpage && layers > 1 { + let unit = if value.stride & 1 != 0 { PAGE_SIZE } else { 0x80 }; + ((value.stride as u64 >> 14) + 1) * unit + } else if value.stride == 0 { + 0 + } else { + ((value.stride as u64 >> 14) + 1) * PAGE_SIZE + }; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range(address_space, value.base, size, GpuAccess::ReadWrite)?; + } + if value.compression_base != 0 { + let layer_stride = ((value.compression_stride as u64 >> 14) + 1) * 0x80; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range( + address_space, + value.compression_base, + size, + GpuAccess::ReadWrite, + )?; + } + Ok(()) +} + +/// Bounded M5 D32 clear/store envelope, alongside the original color-only +/// profile. Native 25G76 supplies ff00ff dimensions and 80044 with compression; +/// bits2/6 select compression, while bit19 stores the depth plane. Geometry, +/// stencil absence, and application programs are checked by the M5 caller. +pub(crate) fn validate_m5_reference_depth( + command: &UapiRenderCommand, + address_space: &A, +) -> Result<(), UapiTranslateError> { + let depth = command.depth; + if depth.base == 0 { + // No depth plane. The surface may still carry an inert stencil one, + // which contributes nothing but its compression pair and the shared + // pixel dimensions; a render that transfers it is refused with the + // rest of the stencil profile. + let stencil_only = command.stencil.base != 0 + && command.zls_control + == if command.stencil.compression_base != 0 { 0x110 } else { 0 } + && command.depth_dimensions + == ((u32::from(command.width) - 1) + | ((u32::from(command.height) - 1) << 16)); + return if depth.compression_base == 0 && depth.stride == 0 + && depth.compression_stride == 0 + && (stencil_only + || (command.zls_control == 0 && command.depth_dimensions == 0)) { + Ok(()) + } else { + Err(UapiTranslateError::ZlsFields) + }; + } + let compressed = depth.compression_base != 0; + // The control word's load and store bits are the only part a render + // chooses: bit15 loads the depth plane and bit19 stores it, and a render + // that neither tests nor writes depth sets neither, which is how a + // conformance surface with an unused depth buffer arrives. Everything + // else -- the compression pair in bits 2 and 6, the GPU tiling, the 32F + // format -- is the witnessed profile and stays exact. depth_flags reaches + // register 0x15319 unchanged either way, so the admitted envelope is the + // set of load/store combinations over that fixed remainder. + let compression_bits = if compressed { 0x44 } else { 0 }; + // Bits 25-26 select the depth format: 0 is 32-bit float, 2 is 16-bit + // unorm. The narrow plane was captured on 25G76 alongside the wide one + // and moves nothing else in the register program except the clear value + // and one ISP flag, both of which the caller derives. Its compression + // metadata has no witness, so a compressed 16-bit plane stays out. + let z16 = command.zls_control & (3 << 25) == (2 << 25); + if z16 && compressed { + return Err(UapiTranslateError::ZlsFields); + } + let format_bits = if z16 { 2 << 25 } else { 0 }; + // A stencil plane may be attached without being transferred: a surface + // with a depth-stencil buffer carries one for every render, whether or + // not the render uses it. Its compression pair in bits 4 and 8 describes + // the attached layout and is the only stencil state admitted here -- + // loading or storing the plane (bits 14 and 18) needs its own witnesses, + // as does the stencil unit in the graphics record. + let stencil_compression_bits = + if command.stencil.compression_base != 0 { 0x110 } else { 0 }; + let control_transfers = command.zls_control & 0x88000; + let expected_control = + compression_bits | stencil_compression_bits | format_bits | control_transfers; + let width = u32::from(command.width); + let height = u32::from(command.height); + if !(1..=M5_MAX_RENDER_DIMENSION).contains(&width) + || !(1..=M5_MAX_RENDER_DIMENSION).contains(&height) + || command.layers != 1 || (compressed && (width < 16 || height < 16)) { + return Err(UapiTranslateError::ZlsFields); + } + // AIL tiles are 16 KiB whatever the format: 64x64 at four bytes a pixel, + // 128x64 at two. While both axes fit one tile the level is a whole + // number of them; if either axis is smaller, the level uses a + // power-of-two miptail aligned to a 128-byte cacheline. Single-layer M5 + // surfaces are not page-padded. + let (tile_width, bytes_per_pixel) = if z16 { (128u32, 2u64) } else { (64u32, 4u64) }; + let data_bytes = if width >= tile_width && height >= 64 { + u64::from(width.div_ceil(tile_width)) * u64::from(height.div_ceil(64)) * 0x4000 + } else { + (u64::from(width.next_power_of_two()) + * u64::from(height.next_power_of_two()) * bytes_per_pixel) + .div_ceil(128) * 128 + }; + let metadata_bytes = (u64::from(width.next_power_of_two().div_ceil(16)) + * u64::from(height.next_power_of_two().div_ceil(16)) * 8).div_ceil(128) * 128; + let dimensions = (width - 1) | ((height - 1) << 16); + if command.depth_dimensions != dimensions + || command.zls_control != expected_control + // The clear value is whatever the application asked for; it reaches + // register 0x15301 verbatim and is only consumed when the render + // actually clears the plane. + || (command.depth_clear & 0x7fffffff) > 0x7f800000 + // Image suballocations and small miptails use cacheline alignment; + // the backing VM pages remain independently validated below. + || depth.base & 0x7f != 0 || depth.compression_base & 0x7f != 0 + // Both strides are array strides, and the M5 caller zeroes them + // because native single-layer ZLS does not encode one. Every render + // admitted here is single-layer -- layers != 1 is refused above -- so + // the value that arrives is whatever the userspace packer computed + // and the hardware never sees it: for a surface smaller than a page + // Mesa's (pages - 1) << 14 | 1 underflows to 0xffffc001. Comparing it + // would refuse a valid render over a field that is discarded. The + // backing store is still validated in full below, from the + // dimensions rather than from either stride. + || (!compressed && depth.compression_stride != 0) { + return Err(UapiTranslateError::ZlsFields); + } + // Validate the entire resized surface and metadata before the + // M5 caller removes the unused single-layer array stride fields. + require_range(address_space, depth.base, data_bytes, GpuAccess::ReadWrite)?; + if compressed { + require_range(address_space, depth.compression_base, metadata_bytes, + GpuAccess::ReadWrite)?; + } + Ok(()) +} + +pub(crate) fn translate_render_command( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, +) -> Result { + translate_render_command_with_capacity(command, vertex_attachments, + fragment_attachments, queue, address_space, internal, 32_768, false) +} + +fn translate_render_command_with_capacity( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, + tile_capacity: u64, + g18_subpage: bool, +) -> Result { + if command.flags & !RENDER_FLAGS != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + // Genuinely unimplemented on G17P rather than merely unvalidated: + // `G17pRenderParameters` carries no helper-program fields at all, so + // VERTEX_SCRATCH has nowhere to put the scratch binding, and there is no + // cluster-count control for NO_VERTEX_CLUSTERING. A clean EINVAL beats + // rendering with the request silently dropped. + if command.flags & (RENDER_VERTEX_SCRATCH | RENDER_NO_VERTEX_CLUSTERING) != 0 { + return Err(UapiTranslateError::FlagsUnsupported); + } + // RSRC_SPEC_HI is *declarative*, and honouring it must not mean rejecting. + // The UAPI defines it as "the appended resource-specifier high dwords are + // present", and states its purpose: "This flag lets old kernels report an + // unsupported render payload instead of silently truncating 64-bit G17 + // values." It exists for kernels that do NOT understand the dwords. + // + // This kernel does understand them -- `program()` ORs the high dword in + // unconditionally, with no reference to the flag -- so the flag cannot + // change what we do with a payload that sets it. The old check therefore + // refused payloads it would otherwise have handled identically, protecting + // nothing; it is the same bring-up leftover as the stencil-clear mask. + // + // Honour the documented meaning instead: when the flag is clear the high + // dwords are "not present", so mask them off rather than refuse the + // submission. A userspace built against the 240-byte upstream struct (whose + // payload `padded_payload` zero-fills) and one that clears the flag now + // behave identically, which is what "not present" has to mean. + let resource_spec_mask = if command.flags & RENDER_RSRC_SPEC_HI != 0 { + u64::MAX + } else { + u32::MAX as u64 + }; + if !(1..=16_384).contains(&command.width) || !(1..=16_384).contains(&command.height) { + return Err(UapiTranslateError::Dimensions); + } + if !(1..=2048).contains(&command.layers) { + return Err(UapiTranslateError::Layers); + } + if !matches!( + (command.utile_width, command.utile_height), + (32, 32) | (32, 16) | (16, 16) + ) { + return Err(UapiTranslateError::UtileDimensions); + } + let samples_log2 = match command.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err(UapiTranslateError::Samples), + }; + let utile_bytes = command.sample_size as u64 + * command.utile_width as u64 + * command.utile_height as u64 + * command.samples as u64; + if utile_bytes > tile_capacity { + return Err(UapiTranslateError::TilebufferSize); + } + let blocks_per_utile = utile_bytes.div_ceil(2048); + + if command.vdm_base == 0 || command.vdm_base & 3 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.vdm_base, 4, GpuAccess::Read)?; + + if command.scissor_base == 0 || command.scissor_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.scissor_base, 8, GpuAccess::Read)?; + if command.depth_bias_base != 0 { + if command.depth_bias_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.depth_bias_base, 8, GpuAccess::Read)?; + } + if command.occlusion_query_base != 0 { + if command.occlusion_query_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.occlusion_query_base, + 8, + GpuAccess::Write, + )?; + } + validate_zls(address_space, command.depth, command.layers, g18_subpage)?; + validate_zls(address_space, command.stencil, command.layers, g18_subpage)?; + validate_sampler( + address_space, + command.sampler_heap, + command.sampler_count as u32, + )?; + if command.vertex_helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + || command.fragment_helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + // `isp_bgobjvals` is the *whole* ISP_BGOBJVALS register value, not just + // the stencil clear: the UAPI documents "the bottom 8-bits contain the + // stencil buffer clear value", and every Mesa backend unconditionally + // seeds the upper bits with 0x300 before OR-ing the clear in + // (hk_cmd_draw.c `render->cr.isp_bgobjvals = 0x300;`, + // agx_pipe.c `c->isp_bgobjvals = 0x300;`, d12_queue.c `0x300 | stencil`). + // The legacy queue path passes the field straight through to the firmware + // (queue/render.rs `let load_bgobjvals = cmdbuf.isp_bgobjvals as u64`), so + // rejecting anything above 0xff refused every real render pass while the + // in-tree render selftest -- which leaves the field zero -- was accepted. + // Accept the register value; `build_*_descriptor` re-asserts the 0x300 + // bits with `stencil_clear_value | 0x300`, which is idempotent for the + // value userspace sends and still supplies 0x300 when it sends zero. + + let background = queue.full_program(address_space, command.background.usc)?; + let end_of_tile = queue.full_program(address_space, command.end_of_tile.usc)?; + let partial_background = queue.full_program(address_space, command.partial_background.usc)?; + let partial_end_of_tile = queue.full_program(address_space, command.partial_end_of_tile.usc)?; + validate_attachments(address_space, &vertex_attachments)?; + validate_attachments(address_space, &fragment_attachments)?; + + let utile_config = ((command.utile_width as u64 / 16) << 12) + | ((command.utile_height as u64 / 16) << 14) + | samples_log2; + let mut tile_config = 0x280; + if command.layers > 1 { + tile_config |= 1; + } + if command.flags & RENDER_PROCESS_EMPTY_TILES != 0 { + tile_config |= 0x1_0000; + } + + let mut parameters = internal.parameters; + parameters.width = command.width as u32; + parameters.height = command.height as u32; + parameters.encoder = command.vdm_base; + parameters.layers = command.layers; + parameters.utile_width = command.utile_width; + parameters.utile_height = command.utile_height; + parameters.utile_config = utile_config; + parameters.multisample_control = command.multisample_control; + parameters.ppp_control = command.ppp_control as u64; + parameters.tib_blocks = blocks_per_utile; + parameters.tile_config = tile_config; + parameters.depth_dimensions = command.depth_dimensions as u64; + parameters.depth_buffer = command.depth.base; + parameters.depth_aux_buffer = command.depth.compression_base; + parameters.depth_stride = command.depth.stride as u64; + parameters.depth_aux_stride = command.depth.compression_stride as u64; + parameters.stencil_buffer = command.stencil.base; + parameters.stencil_aux_buffer = command.stencil.compression_base; + parameters.stencil_stride = command.stencil.stride as u64; + parameters.stencil_aux_stride = command.stencil.compression_stride as u64; + parameters.depth_flags = command.zls_control; + // The firmware's internal partial-render phases preserve attached Z/S + // surfaces even when the render-pass boundary is store-only. Mesa already + // computes this distinction for its partial BG/EOT programs, but the + // current 256-byte render UAPI carries only the boundary ZLS word. Derive + // the exact uncompressed enable bits here until the append-only partial + // ZLS UAPI field is plumbed; tiling/compression/format bits remain those + // packed into the original word. + parameters.partial_depth_flags = command.zls_control + | if command.depth.base != 0 { (1 << 15) | (1 << 19) } else { 0 } + | if command.stencil.base != 0 { (1 << 14) | (1 << 18) } else { 0 }; + parameters.occlusion_query_base = command.occlusion_query_base; + parameters.scissor_array = command.scissor_base; + parameters.depth_bias_array = command.depth_bias_base; + parameters.sampler_array = command.sampler_heap; + parameters.sampler_count = command.sampler_count as u32; + parameters.process_empty_tiles = command.flags & RENDER_PROCESS_EMPTY_TILES != 0; + parameters.merge_upper_x_bits = command.merge_upper_x; + parameters.merge_upper_y_bits = command.merge_upper_y; + parameters.depth_clear_value_bits = command.depth_clear; + parameters.stencil_clear_value = command.stencil_clear; + parameters.load_pipeline_bind = g17p_bg_resource_spec( + command.background.resource_spec, + resource_spec_mask, + internal.parameters.bg_resource_prefix, + ); + parameters.load_pipeline = background; + parameters.store_pipeline_bind = command.end_of_tile.resource_spec & resource_spec_mask; + parameters.store_pipeline = end_of_tile; + // These high 17 bits are BG shader pass control, not an address class. + // Native derives main and partial masks from their respective shader + // outputs; they need not match. The default diagnostic prefix is zero, + // preserving both full RT0 output masks and no-output masks from userspace. + parameters.partial_load_pipeline_bind = g17p_bg_resource_spec( + command.partial_background.resource_spec, + resource_spec_mask, + internal.parameters.bg_resource_prefix, + ); + parameters.partial_load_pipeline = partial_background; + parameters.partial_store_pipeline_bind = + command.partial_end_of_tile.resource_spec & resource_spec_mask; + parameters.partial_store_pipeline = partial_end_of_tile; + + Ok(TranslatedRenderCommand { + render_order: submission::G17PRenderOrder::default(), + render_barrier: UAPI_BARRIER_NONE, + compute_barrier: UAPI_BARRIER_NONE, + compute_wait_through: None, + flags: command.flags, + vdm_base: command.vdm_base, + parameters, + vertex_attachments, + fragment_attachments, + vertex_timestamps: command.vertex_timestamps, + fragment_timestamps: command.fragment_timestamps, + }) +} + +fn g17p_bg_resource_spec(resource_spec: u64, present_mask: u64, diagnostic_prefix: u64) -> u64 { + diagnostic_prefix | (resource_spec & present_mask) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedComputeCommand { + /// Monotonic logical index, independent of the hardware timestamp. + pub(crate) compute_sequence: u64, + /// Explicit same-engine prefix; only host-published timestamps need host waits. + pub(crate) compute_wait_through: Option, + pub(crate) render_wait_through: Option, + pub(crate) render_wait_last: Option, + pub(crate) usc_exec_base: u64, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) control_stream_terminator: u64, + pub(crate) sampler_heap: u64, + pub(crate) sampler_count: u32, + pub(crate) attachments: UapiAttachmentList, + pub(crate) g17p_add3_buffers: Option<[u64; 3]>, + pub(crate) timestamps: UapiTimestamps, + pub(crate) scs_layout: u64, + /// G18 only: the command carried `DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE`. + /// [`translate_compute_command`] refuses that bit like any other unknown + /// one and leaves this false; the G18 adapter admits the bit, strips it + /// before translation and sets this afterwards. + pub(crate) g18_texture_state: bool, + /// Validated native descriptor inputs; resource/launch integration is + /// separate, so this field does not by itself authorize scratch use. + pub(crate) g18_resource_descriptor: Option<[u32; 23]>, +} + +pub(crate) fn translate_compute_command( + command: UapiComputeCommand, + attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, +) -> Result { + queue.validate()?; + if command.flags & !COMPUTE_G17P_ADD3_PROOF != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + if command.g18_resources != [0; 16] { + return Err(UapiTranslateError::FlagsUnsupported); + } + if command.control_stream_base & 3 != 0 + || command.control_stream_end & 3 != 0 + || command.control_stream_end <= command.control_stream_base + { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.control_stream_base, + command.control_stream_end - command.control_stream_base, + GpuAccess::Read, + )?; + validate_sampler(address_space, command.sampler_heap, command.sampler_count)?; + if command.helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + let g17p_add3_buffers = if command.flags & COMPUTE_G17P_ADD3_PROOF != 0 { + let entries = attachments.as_slice(); + if entries.len() != 3 + || entries + .iter() + .any(|entry| entry.size < G17P_ADD3_BUFFER_BYTES) + { + return Err(UapiTranslateError::Add3ProofAttachments); + } + require_range( + address_space, + entries[0].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Read, + )?; + require_range( + address_space, + entries[1].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Read, + )?; + require_range( + address_space, + entries[2].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Write, + )?; + Some([entries[0].address, entries[1].address, entries[2].address]) + } else { + validate_attachments(address_space, &attachments)?; + None + }; + Ok(TranslatedComputeCommand { + compute_sequence: 0, + compute_wait_through: None, + render_wait_through: None, + render_wait_last: None, + usc_exec_base: queue.base, + control_stream_base: command.control_stream_base, + control_stream_end: command.control_stream_end, + control_stream_terminator: command.control_stream_end - 4, + sampler_heap: command.sampler_heap, + sampler_count: command.sampler_count, + attachments, + g17p_add3_buffers, + timestamps: command.timestamps, + scs_layout: command.scs_layout, + g18_texture_state: false, + g18_resource_descriptor: None, + }) +} + +impl TranslatedComputeCommand { + +} diff --git a/drivers/gpu/drm/asahi/m5/completion.rs b/drivers/gpu/drm/asahi/m5/completion.rs new file mode 100644 index 0000000000000..b6b1e748df482 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/completion.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Completion ownership for the bounded M5 TA/fragment pair. +//! +//! 25G76 GFX 0x17544 compares scheduler.completed with *scheduler.submitted; +//! 0x1763c unlinks the completed scheduler. The submitted total is host-owned, +//! not a firmware-decremented pending count. 0x17a68/0x17ae0 then write the +//! first stamp pointer and 0x17b98 publishes the corresponding event mask. + +// The 32-bit completion value reserves its low byte. Stop before it wraps; +// storage-ring wrap is independent of this monotonically increasing stamp. +pub(crate) const MAX_GRAPHICS_GENERATION: u32 = 0x00ff_ffff; +pub(crate) const GRAPHICS_POINTER_COUNT: u32 = 0x500; + +pub(crate) const fn graphics_pointer_head(generation: u32) -> u32 { + (1 + generation * 3) % GRAPHICS_POINTER_COUNT +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct M5GraphicsCompletion { + pub(crate) submitted: u32, + pub(crate) completed: u32, + pub(crate) linked: u32, + pub(crate) list: [u64; 2], + pub(crate) stamps: [u32; 2], + pub(crate) signalled: u128, +} + +/// The same native scheduler retirement contract for one compute record. +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct M5ComputeCompletion { + pub(crate) submitted: u32, + pub(crate) completed: u32, + pub(crate) linked: u32, + pub(crate) list: [u64; 2], + pub(crate) stamp: u32, + pub(crate) signalled: u128, +} + +impl M5ComputeCompletion { + pub(crate) fn complete_at(self, list_address: u64, queue_id: u8, stamp: u32) -> bool { + list_address != 0 && queue_id < 128 && stamp != 0 && stamp & 0xff == 0 + && self.submitted == 1 && self.completed == 1 && self.linked == 0 + && self.list == [0, list_address] && self.stamp == stamp + && self.signalled & (1u128 << queue_id) != 0 + } +} + +impl M5GraphicsCompletion { + + pub(crate) fn pair_complete_at(self, list_address: u64, stamp: u32) -> bool { + list_address != 0 + && stamp != 0 && stamp & 0xff == 0 + && self.submitted == 2 + && self.completed == self.submitted + && self.linked == 0 + && self.list == [0, list_address] + && self.stamps == [stamp; 2] + && self.signalled & 0xc00 == 0xc00 + } +} + diff --git a/drivers/gpu/drm/asahi/m5/compute.rs b/drivers/gpu/drm/asahi/m5/compute.rs new file mode 100644 index 0000000000000..71e1384b070bc --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/compute.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8142/G17G C0 J813 compute register program for the 25G76 native profile. +//! +//! The native SKSM payload has selector 3 and its register array begins at +//! +0x40. The 40-write program below is separately captured on M5; it is not +//! the G17P program with a new chip ID. Queue registration, descriptor-owned +//! pointers, shader/CDM construction and completion still belong to the M5 +//! runtime. Encoding these registers does not admit a Linux GPU job. + +pub(crate) const M5_COMPUTE_REGISTER_COUNT: usize = 40; +pub(crate) const M5_COMPUTE_REGISTER_BYTES: usize = 12 * M5_COMPUTE_REGISTER_COUNT; +/// Measured descriptor prefix, not a claim about the host allocation size. +pub(crate) const M5_COMPUTE_DESCRIPTOR_BYTES: usize = 0x1000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5ComputeError { + AddressOverflow, + BufferTooSmall, + CdmTerminatorBeforeStream, +} + +/// Retained objects and per-command state for the measured simple CL path. +/// Samplers, user timestamps, indirect dispatch and extra spill modes are not +/// part of this profile. Pointer ownership belongs to the runtime caller. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ComputeDescriptor { + pub(crate) registers: M5ComputeParameters, + pub(crate) submit_sequence: u64, + /// Host descriptor +0x188, copied to native command +0x10. + pub(crate) scheduler_record: u64, + pub(crate) descriptor_low_alias: u64, + pub(crate) cdm_terminator: u64, + pub(crate) dispatch_stamps: [u64; 2], + pub(crate) event_machine_stamp: u32, + pub(crate) queue_id: u32, + pub(crate) queue_ordinal: u32, + pub(crate) has_event: bool, + pub(crate) submission_index: u32, + pub(crate) status: [u64; 2], + pub(crate) shared_control: u64, + pub(crate) shared_control_enabled: bool, + /// Sixteen bytes copied by the native host from the event machine's + /// +0x70 record. Their bit meanings are not all qualified; keep opaque. + pub(crate) event_metadata: [u8; 16], + pub(crate) zero_page: u64, + /// Host descriptor +0x148; not a fixed target constant. + pub(crate) owner_id: u8, + /// Full host descriptor +0x140. Register context packing uses only its + /// low byte; the descriptor copy at +0xf70 must retain all 32 bits. + pub(crate) descriptor_work_ordinal: u32, +} + +/// Encode all 4096 bytes of the native simple-compute descriptor prefix. +/// The register arrays and their aliases are generated from owned inputs. +/// This function performs no allocation, cache operation or publication. +pub(crate) fn encode_m5_compute_descriptor( + p: M5ComputeDescriptor, + raw: &mut [u8], +) -> Result<(), M5ComputeError> { + if raw.len() < M5_COMPUTE_DESCRIPTOR_BYTES { + return Err(M5ComputeError::BufferTooSmall); + } + if p.cdm_terminator < p.registers.cdm_base { + return Err(M5ComputeError::CdmTerminatorBeforeStream); + } + let registers = build_m5_compute_registers(p.registers)?; + let primary_alias = add(p.descriptor_low_alias, 0x40)?; + let secondary_alias = add(p.descriptor_low_alias, 0x760)?; + let secondary = [ + (0x10099u32, add(p.registers.operand_state_base, 5)?), + (0x10091, add(p.registers.operand_state_base, 0xf000)?), + (0x0a5c1, add(p.registers.operand_state_base, 0xf405)?), + (0x0a5c9, add(p.registers.operand_state_base, 0x1e400)?), + ]; + let raw = &mut raw[..M5_COMPUTE_DESCRIPTOR_BYTES]; + raw.fill(0); + for (offset, value) in [ + (0x00, 3u32), (0x0c, u32::from(p.registers.context_id)), + (0x748, 0x01e0_0028), (0xe68, 0x0030_0004), + (0xf20, (p.registers.dispatch_identity >> 32) as u32), + (0xf28, u32::MAX), (0xf50, p.event_machine_stamp), + (0xf54, p.queue_id), (0xf58, p.queue_ordinal), + (0xf5c, u32::from(p.has_event)), (0xf60, p.submission_index), + (0xf70, p.descriptor_work_ordinal), + ] { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [ + (4, p.submit_sequence), (0x10, p.scheduler_record), + (0x740, primary_alias), (0xe60, secondary_alias), + (0xed8, p.registers.preempt_base), (0xee0, p.cdm_terminator), + (0xf08, 0x0000_0001_5402_4201), + (0xf40, p.dispatch_stamps[0]), (0xf48, p.dispatch_stamps[1]), + (0xf68, p.registers.dispatch_identity & 0xffff_ffff), + (0xf7c, p.status[0]), (0xf84, p.status[1]), + (0xfb2, p.shared_control), (0xfcb, p.zero_page), + ] { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + for (index, count) in [37u16, 39, 39, 40].into_iter().enumerate() { + raw[0x18 + index * 2..0x1a + index * 2].copy_from_slice(&count.to_le_bytes()); + } + for (index, register) in registers.into_iter().enumerate() { + let offset = 0x40 + index * 12; + raw[offset..offset + 4].copy_from_slice(®ister.number.to_le_bytes()); + raw[offset + 4..offset + 12].copy_from_slice(®ister.value.to_le_bytes()); + } + for (index, (number, value)) in secondary.into_iter().enumerate() { + let offset = 0x760 + index * 12; + raw[offset..offset + 4].copy_from_slice(&number.to_le_bytes()); + raw[offset + 4..offset + 12].copy_from_slice(&value.to_le_bytes()); + } + raw[0xfba] = u8::from(p.shared_control_enabled); + raw[0xfbb..0xfcb].copy_from_slice(&p.event_metadata); + raw[0xfd3] = p.owner_id; + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ComputeRegister { + pub(crate) number: u32, + pub(crate) value: u64, +} + +impl M5ComputeRegister { + const fn new(number: u32, value: u64) -> Self { + Self { number, value } + } +} + +/// Command-owned addresses and identities; the runtime must retain their +/// backing and validate its mappings before publishing the complete command. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ComputeParameters { + pub(crate) preempt_base: u64, + pub(crate) cdm_base: u64, + pub(crate) dispatch_identity: u64, + pub(crate) context_id: u8, + pub(crate) work_ordinal: u8, + pub(crate) robustness: u64, + pub(crate) operand_state_base: u64, + /// Owned UMA pool slot, copied from native host descriptor +0x788 by + /// 25G76 generateRegisterList at 0x89e38dc..0x89e391c. This must match + /// shared-control +8 and the populated hardware UMA descriptor slot. + pub(crate) private_pool_hw_id: u64, + /// Native command descriptor +0x530, copied to register 0xd4a1. + pub(crate) descriptor_control_530: u64, + /// Descriptor +0x538; zero selects the native 0x60 default. The native + /// encoder shifts the selected value left 32 bits, discarding high bits. + pub(crate) descriptor_control_538: u64, + /// Native descriptor +0x60a/+0x60b bit 0. Their broader semantics are + /// still under investigation; neither is a fixed M5 device property. + pub(crate) descriptor_flag_60a: bool, + pub(crate) descriptor_flag_60b: bool, +} + +fn add(base: u64, offset: u64) -> Result { + base.checked_add(offset) + .ok_or(M5ComputeError::AddressOverflow) +} + +/// Encode the measured J813 profile, including its M5 control words and +/// enlarged operand-state offsets. Other G17G configurations are not implied. +pub(crate) fn build_m5_compute_registers( + p: M5ComputeParameters, +) -> Result<[M5ComputeRegister; M5_COMPUTE_REGISTER_COUNT], M5ComputeError> { + let context = ((p.context_id as u64) << 8) | p.work_ordinal as u64; + // 25G76 generateRegisterList +0x5d4..0x670. The final 0xa599 write + // restores 0x60 irrespective of the command-specific initial value. + let control_538 = if p.descriptor_control_538 == 0 { + 0x60 + } else { + p.descriptor_control_538 + }; + // 25G76 +0x644..0x66c; additional device-dependent bits used by other + // profiles are outside this measured J813 configuration. + let mode = if p.descriptor_flag_60a { + 0x14c0_8a60 + } else { + 0x10c0_8860 + } | if p.descriptor_flag_60b { 0x80 } else { 0 }; + Ok([ + M5ComputeRegister::new(0x1a510, p.preempt_base), + M5ComputeRegister::new(0x1a420, p.cdm_base), + M5ComputeRegister::new(0x1a4d0, add(p.preempt_base, 0x1480)?), + M5ComputeRegister::new(0x1a4d8, add(p.preempt_base, 0x1488)?), + M5ComputeRegister::new(0x1a4e0, add(p.preempt_base, 0x1490)?), + M5ComputeRegister::new(0x1a4e8, add(p.preempt_base, 0x1498)?), + M5ComputeRegister::new(0x1a440, 0x0000_0001_5402_4201), + M5ComputeRegister::new(0x0d4a1, p.descriptor_control_530), + M5ComputeRegister::new(0x0d411, 0x0000_0002_0000_0001), + M5ComputeRegister::new(0x0a599, (control_538 << 32) | 0x0040_0020), + M5ComputeRegister::new(0x1a458, mode), + M5ComputeRegister::new(0x101d9, 0x1c), + M5ComputeRegister::new(0x1a089, 0), + M5ComputeRegister::new(0x1a091, 0), + M5ComputeRegister::new(0x1a059, 0), + M5ComputeRegister::new(0x1a061, 0), + M5ComputeRegister::new(0x1a0b9, 0), + M5ComputeRegister::new(0x1a0c1, 0), + M5ComputeRegister::new(0x101d1, 0), + M5ComputeRegister::new(0x0d479, 0), + M5ComputeRegister::new(0x1a0e9, 8), + M5ComputeRegister::new(0x1a141, 0), + M5ComputeRegister::new(0x107a1, 0xff_0000), + M5ComputeRegister::new(0x1a540, p.dispatch_identity), + M5ComputeRegister::new(0x014a9, p.dispatch_identity), + M5ComputeRegister::new(0x0a351, p.dispatch_identity), + M5ComputeRegister::new(0x0a731, 0), + M5ComputeRegister::new(0x10201, context), + M5ComputeRegister::new(0x10428, context), + M5ComputeRegister::new(0x14028, p.private_pool_hw_id), + M5ComputeRegister::new(0x14070, p.robustness | 1), + M5ComputeRegister::new(0x10229, add(p.operand_state_base, 0x1e800)?), + M5ComputeRegister::new(0x140a8, add(p.operand_state_base, 0x1f000)?), + M5ComputeRegister::new(0x10099, add(p.operand_state_base, 0x0f405)?), + M5ComputeRegister::new(0x10091, add(p.operand_state_base, 0x1e400)?), + M5ComputeRegister::new(0x0a5c1, add(p.operand_state_base, 0x00005)?), + M5ComputeRegister::new(0x0a5c9, add(p.operand_state_base, 0x0f000)?), + M5ComputeRegister::new(0x0a731, 0), + M5ComputeRegister::new(0x1a440, 0x0000_0001_5402_4209), + M5ComputeRegister::new(0x0a599, 0x0000_0060_0040_0020), + ]) +} + +/// Write only the register array. No descriptor header or publication state +/// is manufactured here, and an invalid request leaves the destination alone. +pub(crate) fn encode_m5_compute_registers( + p: M5ComputeParameters, + bytes: &mut [u8], +) -> Result<(), M5ComputeError> { + if bytes.len() < M5_COMPUTE_REGISTER_BYTES { + return Err(M5ComputeError::BufferTooSmall); + } + let registers = build_m5_compute_registers(p)?; + for (index, register) in registers.iter().enumerate() { + let offset = index * 12; + bytes[offset..offset + 4].copy_from_slice(®ister.number.to_le_bytes()); + bytes[offset + 4..offset + 12].copy_from_slice(®ister.value.to_le_bytes()); + } + Ok(()) +} + diff --git a/drivers/gpu/drm/asahi/m5/drm.rs b/drivers/gpu/drm/asahi/m5/drm.rs new file mode 100644 index 0000000000000..a2a7b9d2679ac --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/drm.rs @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Experimental M5 DRM adapter using the retained M5 firmware session. +//! First-job admission is deliberately separate from general Mesa support. + +use core::{any::Any, ops::Range}; +use kernel::{bindings, c_str, device, macros::vtable, new_mutex, prelude::*, static_lock_class, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + sync::{Arc, Mutex}, uapi, xarray}; +use crate::{alloc, driver::AsahiRegData, file, m5::live_boot::M5Runtime, + m5::resources::M5FirmwareMemory, m5::command, gem, gpu, hw, m5::vm_layout, mmu, queue}; + +pub(crate) type SharedM5Runtime = Arc>; +const USER_END: u64 = (1 << 42) - 2 * mmu::UAT_PGSZ as u64; + +/// Bytes of CDM control stream the witnessed M5 compute profile covers. The +/// captured stream (compute_program::STREAM) is twelve words: one dispatch +/// and the terminator at +44. Nothing longer has native evidence, and a +/// longer one is what a geometry-shader prepass submits. +const M5_COMPUTE_STREAM_BYTES: u64 = 48; +static QUEUE_NAME: &CStr = c_str!("m5_queue"); +static QUEUE_CLASS_KEY: Pin<&'static kernel::sync::LockClassKey> = static_lock_class!(); + +#[cfg(CONFIG_APPLE_CLPC)] +extern "C" { + fn apple_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) + -> core::ffi::c_int; + fn apple_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); + fn apple_clpc_gpu_put(gpu: *mut core::ffi::c_void); +} + +/// The CLPC's GPU input (`apple,clpc` on the GPU node). While the PMP runs, +/// its vote leaves the DRAM controller and the fabric at their lowest states; +/// the CLPC holds them at the top of its maps while M5 work is in flight, +/// and releases them its hold time after the last of it retires. Without a +/// CLPC the GPU runs as before, only slower under the PMP. +#[pin_data(PinnedDrop)] +pub(crate) struct M5ClpcInput { + gpu: *mut core::ffi::c_void, + /// Submissions in flight; the input is active while it is not zero. + #[pin] + in_flight: Mutex, +} + +// SAFETY: the CLPC serializes its input under its own lock, and the counter +// is only changed under `in_flight`. +unsafe impl Send for M5ClpcInput {} +// SAFETY: as above; shared use never changes the reference itself. +unsafe impl Sync for M5ClpcInput {} + +impl M5ClpcInput { + /// Takes the input before the firmware session starts, so that a CLPC + /// which has not probed yet defers the GPU without stranding a session. + pub(crate) fn new(dev: &device::Device) -> Result> { + #[allow(unused_mut)] + let mut gpu: *mut core::ffi::c_void = core::ptr::null_mut(); + #[cfg(CONFIG_APPLE_CLPC)] + { + // SAFETY: a live device and a writable result; C returns a + // reference, or NULL when the node names no CLPC. + let ret = unsafe { apple_clpc_gpu_get(dev.as_raw().cast(), &mut gpu) }; + if ret == EPROBE_DEFER.to_errno() { + return Err(EPROBE_DEFER); + } + if ret < 0 { + dev_warn!(dev, "no CLPC input ({}): GPU work gets no memory floor\n", ret); + gpu = core::ptr::null_mut(); + } + } + #[cfg(not(CONFIG_APPLE_CLPC))] + let _ = dev; + Arc::pin_init(pin_init!(Self { gpu, in_flight <- new_mutex!(0) }), GFP_KERNEL) + } + + fn set_active(&self, _active: bool) { + #[cfg(CONFIG_APPLE_CLPC)] + // SAFETY: NULL is accepted; a non-NULL reference is held until drop. + unsafe { apple_clpc_gpu_set_active(self.gpu, _active) }; + } + + /// Marks one submission in flight until the returned guard drops. + fn busy(&self) -> M5ClpcBusy<'_> { + let mut count = self.in_flight.lock(); + *count += 1; + if *count == 1 { + self.set_active(true); + } + M5ClpcBusy(self) + } +} + +#[pinned_drop] +impl PinnedDrop for M5ClpcInput { + fn drop(self: Pin<&mut Self>) { + #[cfg(CONFIG_APPLE_CLPC)] + // SAFETY: releases exactly the reference taken in new(), clearing the + // input first; NULL is accepted. + unsafe { apple_clpc_gpu_put(self.gpu) }; + } +} + +struct M5ClpcBusy<'a>(&'a M5ClpcInput); + +impl Drop for M5ClpcBusy<'_> { + fn drop(&mut self) { + let mut count = self.0.in_flight.lock(); + *count -= 1; + if *count == 0 { + self.0.set_active(false); + } + } +} + +pub(crate) struct M5DrmGpu { + // A strong reference prevents open DRM files from outliving the session. + runtime: SharedM5Runtime, + id: crate::regs::G17Id, + ids: gpu::SequenceIDs, + clpc: Arc, +} + +impl M5DrmGpu { + pub(crate) fn new(runtime: SharedM5Runtime, id: crate::regs::G17Id, + clpc: Arc) -> Self { + Self { runtime, id, ids: Default::default(), clpc } + } +} + +impl gpu::Gpu for M5DrmGpu { + fn stats_snapshot(&self) -> Option> { None } + fn as_any(&self) -> &dyn Any { self } + fn arc_as_any(self: Arc) -> Arc { self as Arc } + fn manager(&self) -> Option<&dyn gpu::GpuManager> { None } + fn init(&self) -> Result { + if self.is_crashed() { Err(ENODEV) } else { Ok(()) } + } + fn ids(&self) -> &gpu::SequenceIDs { &self.ids } + fn is_crashed(&self) -> bool { self.runtime.lock().is_crashed() } + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.gpu_generation = hw::GpuGen::G17 as u32; + params.gpu_variant = hw::GpuVariant::G as u32; + params.gpu_revision = self.id.gpu_rev as u32; + params.chip_id = 0x8142; + params.num_dies = self.id.num_dies; + params.num_clusters_total = self.id.num_clusters; + params.num_cores_per_cluster = self.id.num_cores; + for (i, mask) in self.id.core_masks.iter().enumerate() { + *params.core_masks.get_mut(i).ok_or(EIO)? = (*mask).into(); + } + // Dynamic M5 clock policy is not implemented. + params.max_frequency_khz = 0; + params.usc_generation = 3; + params.gpu_hal_generation = 300; + // One hardware command per submission. + params.max_commands_per_submission = 1; + Ok(()) + } + fn supports_vm_status(&self) -> bool { true } + /// VM_BIND defers unmaps while accepted work runs, and a submission installs its fence + /// in every output sync, also on failure. + fn tracked_submission(&self) -> bool { true } + fn user_range(&self) -> Range { mmu::UAT_PGSZ as u64..USER_END } + fn base_clock_hz(&self) -> u32 { 24_000_000 } + fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } + fn new_vm(&self, kernel_range: Range) -> Result { + self.runtime.lock().new_user_vm(self.ids.vm.next(), kernel_range) + } + fn new_queue(&self, vm: mmu::Vm, + _ualloc: Arc>, + _ualloc_priv: Arc>, + priority: u32, usc_exec_base: u64) -> Result> { + if priority > 3 { return Err(EINVAL); } + let window = command::QueueUscWindow { + base: usc_exec_base, user_start: vm_layout::DRIVER_BASE, user_end: USER_END, + }; + window.validate().map_err(|_| EINVAL)?; + let queue = M5Queue { + runtime: self.runtime.clone(), vm, window, + fences: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, + clpc: self.clpc.clone(), + }; + Ok(KBox::new(queue, GFP_KERNEL)?) + } + fn map_timestamp_buffer(&self, _bo: gem::ObjectRef, _range: Range) + -> Result { Err(ENOTSUPP) } +} + +struct M5SubmissionFence; +#[vtable] +impl FenceOps for M5SubmissionFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("m5-submit") } +} + +/// Keep the original sync vector: creating a second output list would add an +/// allocation failure after userspace has already supplied its wait targets. +struct OutputPublication { + fence: UserFence, + syncs: KVec, + input_count: usize, + result: Result, +} + +impl Drop for OutputPublication { + fn drop(&mut self) { + if let Err(error) = self.result { self.fence.set_error(error); } + let fence = Fence::from_fence(&self.fence); + for mut sync in self.syncs.drain(self.input_count..) { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj.add_point(chain, &fence, sync.timeline_value); + } else { sync.syncobj.replace_fence(Some(&fence)); } + } + self.fence.signal(); + } +} + +struct UserAddressSpace<'a> { vm: &'a mmu::Vm } +impl command::GpuAddressSpace for UserAddressSpace<'_> { + fn covers(&self, address: u64, size: u64, access: command::GpuAccess) -> bool { + let (read, write) = match access { + command::GpuAccess::Read => (true, false), + command::GpuAccess::Write => (false, true), + command::GpuAccess::ReadWrite => (true, true), + }; + vm_layout::user_range_contains(&self.vm.kernel_range(), address, size, USER_END) + && self.vm.covers_range(address, size, read, write) + } +} + +struct M5Queue { + runtime: SharedM5Runtime, + vm: mmu::Vm, + window: command::QueueUscWindow, + fences: FenceContexts, + clpc: Arc, +} + +impl queue::Queue for M5Queue { + fn submit(&mut self, _reg_data: &AsahiRegData<'_>, _submission_id: u64, + syncs: KVec, + in_sync_count: usize, cmdbuf_raw: &[u8], + _objects: Pin<&xarray::XArray>>) -> Result { + if in_sync_count > syncs.len() { return Err(EINVAL); } + let fence = self.fences.new_fence(0, M5SubmissionFence)?.into(); + let mut publication = OutputPublication { + fence, syncs, input_count: in_sync_count, result: Err(ECANCELED), + }; + let result = (|| { + if self.vm.status().is_some_and(|status| status.get() != 0) { return Err(EIO); } + // Wait outside the runtime lock; another client may own the job + // whose fence satisfies this submission's dependency. + for sync in &publication.syncs[..in_sync_count] { + if let Some(fence) = &sync.fence { + // SAFETY: both calls borrow the strongly held input fence. + let waited = unsafe { bindings::dma_fence_wait_timeout( + fence.raw(), true, kernel::task::MAX_SCHEDULE_TIMEOUT) }; + if waited < 0 { return Err(Error::from_errno(waited as i32)); } + if waited == 0 { return Err(ETIMEDOUT); } + let status = unsafe { bindings::dma_fence_get_status(fence.raw()) }; + if status < 0 { return Err(Error::from_errno(status)); } + } + } + let _vm_job = self.vm.retain_job()?; + let address_space = UserAddressSpace { vm: &self.vm }; + let mut parser = command::UapiCommandParser::new(cmdbuf_raw); + let command = parser.next_hardware().map_err(|_| EINVAL)?.ok_or(EINVAL)?; + if parser.next_hardware().map_err(|_| EINVAL)?.is_some() { + pr_err!("M5 submit admission rejected: more than one hardware command\n"); + return Err(ENOTSUPP); + } + parser.finish().map_err(|_| EINVAL)?; + if let command::ParsedHardwareCommand::Compute { payload, attachments, .. } = command { + let timestamps = [payload.timestamps.start, payload.timestamps.end]; + if payload.flags != 0 || payload.sampler_heap != 0 || payload.sampler_count != 0 + || timestamps.iter().any(|t| t.handle != 0 || t.offset != 0) { + pr_err!("M5 compute admission rejected flags={:#x} sampler_heap={:#x} samplers={} timestamps={}\n", + payload.flags, payload.sampler_heap, payload.sampler_count, + timestamps.iter().any(|t| t.handle != 0 || t.offset != 0)); + return Err(ENOTSUPP); + } + // The witnessed M5 compute profile is one dispatch: the + // captured control stream is twelve words, a single dispatch + // followed by the terminator at +44, and the descriptor + // carries exactly one dispatch identity, one submission index + // and one pair of dispatch stamps. A longer stream -- every + // geometry-shader and transform-feedback prepass emits four or + // five dispatches -- is accepted by firmware, linked into the + // job list and never retired, so the submitting ioctl times + // out after forty seconds and quarantines the VM. Refuse it + // here instead: the client drops the batch and the session + // survives, exactly as it does for an out-of-profile render. + let stream_bytes = payload.control_stream_end + .saturating_sub(payload.control_stream_base); + if stream_bytes > M5_COMPUTE_STREAM_BYTES { + pr_err!("M5 compute admission rejected stream_bytes={} (profile admits one dispatch, {})\n", + stream_bytes, M5_COMPUTE_STREAM_BYTES); + return Err(ENOTSUPP); + } + let translated = command::translate_compute_command(payload, attachments, + self.window, &address_space).map_err(|_| EINVAL)?; + let _busy = self.clpc.busy(); + return self.runtime.lock().submit_application_compute(&self.vm, &translated); + } + let command::ParsedHardwareCommand::Render { + payload, vertex_attachments, fragment_attachments, .. + } = command else { + pr_err!("M5 submit admission rejected: neither a render nor a compute command\n"); + return Err(ENOTSUPP); + }; + admit_reference_render(&payload).map_err(|error| { + pr_err!("M5 render admission rejected size={}x{} layers={} utiles={}x{} samples={}\n", + payload.width, payload.height, payload.layers, + payload.utile_width, payload.utile_height, payload.samples); + error + })?; + // An attached stencil plane is admitted only while it is inert: + // its buffers are range-checked with the depth ones and its + // registers are written from the same parameter block, but a + // render that loads or stores it, or that runs the stencil unit, + // is outside the witnessed profile. + if payload.zls_control & 0x44000 != 0 { + pr_err!("M5 render admission rejected stencil transfer control={:#x}\n", + payload.zls_control); + return Err(ENOTSUPP); + } + command::validate_m5_reference_depth(&payload, &address_space) + .map_err(|error| { + pr_err!("M5 depth admission rejected {:?}: size={}x{} dimensions={:#x} control={:#x} stride={:#x} meta_stride={:#x} base={:#x} meta={:#x} clear={:#x}\n", + error, payload.width, payload.height, payload.depth_dimensions, + payload.zls_control, payload.depth.stride, payload.depth.compression_stride, + payload.depth.base, payload.depth.compression_base, payload.depth_clear); + ENOTSUPP + })?; + let (parameters, mut controls) = M5FirmwareMemory::reference_render_profile(); + let mut translated = command::translate_render_command(payload, + vertex_attachments, fragment_attachments, self.window, &address_space, + command::RenderInternalState { parameters }).map_err(|_| EINVAL)?; + // Native single-layer M5 ZLS does not encode an array stride. + // The full data/metadata ranges were validated above. + translated.parameters.depth_stride = 0; + translated.parameters.depth_aux_stride = 0; + // Native 25G76 indexed draws use 15021=0x10c000. The reference + // triangle's low bit causes fragment recovery with larger + // indexed overdraw, even after the tiling stage completes. + // Bit18 selects the narrow depth plane: the same controlled + // render against a Depth16Unorm attachment publishes 0x14c001 + // where the 32-bit one publishes 0x10c001, and that bit is the + // only difference in the whole register program beyond the ZLS + // format field and the integer clear value. + let z16 = payload.zls_control & (3 << 25) == (2 << 25); + translated.parameters.aux_fb_flags = + if z16 { 0x14c000 } else { 0x10c000 }; + // M5's second store-program pair refers to the same EOT program + // in the admitted native profile, with caller-owned program VAs. + controls.store_pipeline_2 = translated.parameters.store_pipeline; + controls.store_pipeline_2_bind = translated.parameters.store_pipeline_bind; + controls.partial_store_pipeline_2 = translated.parameters.partial_store_pipeline; + controls.partial_store_pipeline_2_bind = translated.parameters.partial_store_pipeline_bind; + let _busy = self.clpc.busy(); + self.runtime.lock().submit_application(&self.vm, translated.parameters, + controls, 1, [0x7d4000, 0x934000])?; + Ok(()) + })(); + // The runtime records and quarantines a failed published VM before + // this output fence is installed/signalled by the publication guard. + publication.result = result; + result + } +} + +/// Until the M5 controls are derived for other render modes, admit only the +/// reference render-state envelope. This is an experimental first-job ABI +/// path, not an assertion that arbitrary Vulkan programs are supported. +fn admit_reference_render(command: &command::UapiRenderCommand) -> Result { + let timestamps = [command.vertex_timestamps.start, command.vertex_timestamps.end, + command.fragment_timestamps.start, command.fragment_timestamps.end]; + if !(1..=command::M5_MAX_RENDER_DIMENSION).contains(&u32::from(command.width)) + || !(1..=command::M5_MAX_RENDER_DIMENSION).contains(&u32::from(command.height)) + || command.layers != 1 + || command.utile_width != 32 || command.utile_height != 32 || command.samples != 1 + || command.sampler_heap != 0 || command.sampler_count != 0 + || command.occlusion_query_base != 0 + || timestamps.iter().any(|timestamp| timestamp.handle != 0 || timestamp.offset != 0) { + return Err(ENOTSUPP); + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/m5/fw_msg.rs b/drivers/gpu/drm/asahi/m5/fw_msg.rs new file mode 100644 index 0000000000000..368ed4cc36f34 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/fw_msg.rs @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +const INITDATA_TAG: u64 = 0x0081_0000_0000_0000; + +const INITDATA_ADDRESS_MASK: u64 = 0x0000_0fff_ffff_ffff; + +const INITDATA_ADDRESS_PREFIX: u64 = 0xffff_f000_0000_0000; + +/// `AGXArmFirmware::powerTransitionEnd(AGFIFirmwareRole, bool)` closes the GPU +/// power-on transition by sending this note to EACH firmware role. +/// +/// KEXT_PROVEN, 25G83 @0xfffffe00088936a4. Both send sites are identical: +/// +/// ldr x0, [x19, #6616] / [x19, #6672] ; the two role objects +/// ldr x8, [obj + 0x8a8] ; sendAsyncNoteToFirmware +/// mov x1, #0x89000000000000 ; tag 0x89, payload 0 +/// mov w2, #0x1 ; gated=true +/// blraa x8, x16 +/// +/// MEASURED on hardware (m1n1 hv capture 20260904-001852Z): it is the LAST +/// EP-0x21 message of bring-up, sent once per role right after the initdata +/// handshake completes and 11.4 seconds before the first work doorbell: +/// +/// 57.95ms role0 -> ep=0x21 0x0089000000000000 +/// 59.31ms role1 -> ep=0x21 0x0089000000000000 +/// ... +/// 11425.12ms role0 -> ep=0x21 0x0083000000000000 <- first doorbell +/// +/// This driver has never sent it. Its EP-0x21 vocabulary was 0x81 initdata, +/// 0x83 doorbell, 0x84 device control and 0x86 recovery inspection only. +const POWER_TRANSITION_END_TAG: u64 = 0x0089_0000_0000_0000; + +/// Encode the role's power-transition-end note. The payload is zero in every +/// captured instance and the kext materialises the bare tag with a single +/// `movz x1, #0x89, lsl #48`. +pub(crate) const fn encode_power_transition_end() -> u64 { + POWER_TRANSITION_END_TAG +} + +/// Refusals made by the evidence-backed codec. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum WireError { + AddressOutOfRange, +} + +/// Encode the G17 init-data handoff exactly as the recovered host does. +/// +/// The host instruction truncates to 44 bits. Linux instead rejects an +/// address unless it is already in low-44-bit or firmware-canonical form. +pub(crate) const fn encode_initdata_doorbell(firmware_va: u64) -> Result { + let high = firmware_va & !INITDATA_ADDRESS_MASK; + if high != 0 && high != INITDATA_ADDRESS_PREFIX { + return Err(WireError::AddressOutOfRange); + } + + Ok(INITDATA_TAG | (firmware_va & INITDATA_ADDRESS_MASK)) +} diff --git a/drivers/gpu/drm/asahi/m5/initdata.rs b/drivers/gpu/drm/asahi/m5/initdata.rs new file mode 100644 index 0000000000000..f2ad6afea017f --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/initdata.rs @@ -0,0 +1,1630 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +use core::result::Result; + +/// Root size for the primary (`gfx-asc`) instance. +pub(crate) const ROOT_SIZE_PRIMARY: usize = 0x0b8; + +/// Root size for the secondary (`gfx1-asc`) instance; the extra 0x10 bytes are +/// the two secondary-only pointers at `+0xb8`/`+0xc0`. +pub(crate) const ROOT_SIZE_SECONDARY: usize = 0x0c8; + +pub(crate) const MAIN_CONFIG_SIZE: usize = 0x600; + +pub(crate) const REGION_A_SIZE: usize = 0x4000; + +pub(crate) const REGION_C_SIZE: usize = 0x1000; + +pub(crate) const STATUS_BLOCK_SIZE: usize = 0x80; + +pub(crate) const HW_DATA_BUNDLE_MIN_ALLOC: usize = 0x10000; + +/// The native placement additionally hosts the twelve work rings and both main +/// configuration objects inside the same cluster, which then spans this much +/// (see [`WORK_RING_BUNDLE_OFFSETS`], [`NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET`]). +pub(crate) const NATIVE_SHARED_CLUSTER_SIZE: usize = 0x28000; + +/// The same tuple as the little-endian u64 the firmware compares in one go. +pub(crate) const INITDATA_VERSION_WORD: u64 = 0x0c8a_a322_0396_04c0; + +/// Target/build-specific root encoding. This is a serialization choice, not +/// permission to start firmware or reuse the other target's hardware tables. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RootProfile { + J700G17P, + /// J813 Mac17,3 / 25G76, RTKit-3255.160.4.release. Both roots were captured + /// before INBOX1 publication. Independently confirmed by the loaded GFX + /// version comparison at +0x1fa5c and GFX1 at +0x21140. + J813G17G25G76, + /// J873 Mac18,5 / T8152 / G18G / 26A428, RTKit_release-3514.0.15 + /// (armfw_g18g-b000 on GFX, armfw1_g18g-b010 on GFX1). No capture of a + /// native root exists; the encoding is source-constructed from the 26A428 + /// kext and firmware (see the "J873 / G18G" section below). Selecting it + /// changes nothing for the other two profiles. + J873G18G26A428, +} + +impl RootProfile { + pub(crate) const fn version_word(self) -> u64 { + match self { + Self::J700G17P => INITDATA_VERSION_WORD, + Self::J813G17G25G76 => 0x0c8a_a322_039e_04c0, + Self::J873G18G26A428 => G18G_26A428_INITDATA_VERSION_WORD, + } + } + + /// Bytes of the root object for `role`. J700/J813 keep the historical + /// 0xb8/0xc8 split. G18G sizes both roots to 0xc8 because both firmware + /// images copy 0xc8 bytes from each root (FW_PROVEN b000 0x21664-0x216a8). + pub(crate) const fn root_size(self, role: InstanceRole) -> usize { + match self { + Self::J873G18G26A428 => G18G_26A428_ROOT_SIZE, + Self::J700G17P | Self::J813G17G25G76 => root_size(role), + } + } + + /// Opcode of the one opening device-control record staged at main + /// `+0x4c0`. G18G renumbered the opening handler: 0x16 is now an + /// unrelated handler and 0x15 reaches the opening (FW_PROVEN jump table + /// 0xfffffc0000006c60, handler b000 0x3ed0). The secondary keeps 0x2a. + pub(crate) const fn control_opening_opcode(self, role: InstanceRole) -> u32 { + match (self, role) { + (Self::J873G18G26A428, InstanceRole::Primary) => { + G18G_26A428_CONTROL_OPENING_OPCODE_PRIMARY + } + (Self::J873G18G26A428, InstanceRole::Secondary) => { + G18G_26A428_CONTROL_OPENING_OPCODE_SECONDARY + } + (_, InstanceRole::Primary) => CONTROL_OPENING_OPCODE_PRIMARY, + (_, InstanceRole::Secondary) => CONTROL_OPENING_OPCODE_SECONDARY, + } + } + + /// Bundle offsets of the five views published at main `+0x254`. + pub(crate) const fn bundle_view_offsets(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => G18G_26A428_BUNDLE_VIEW_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_OFFSETS, + } + } + + /// Extents each view must fit inside the bundle allocation. + pub(crate) const fn bundle_view_extents(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => G18G_26A428_BUNDLE_VIEW_EXTENTS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_EXTENTS, + } + } + + /// Bundle offset named twice by main `+0x08`/`+0x10` (the empty BRN + /// table and the 0x88 block, which the allocator places together). + pub(crate) const fn repeated_region_bundle_offset(self) -> usize { + match self { + Self::J873G18G26A428 => G18G_26A428_SHARED_LAYOUT.block_88, + Self::J700G17P | Self::J813G17G25G76 => MAIN_REPEATED_REGION_BUNDLE_OFFSET, + } + } + +} + +const INITDATA_ADDR_MASK: u64 = 0x0000_0fff_ffff_ffff; + +const INITDATA_ADDR_PREFIX: u64 = 0xffff_f000_0000_0000; + +const ROOT_VERSION: usize = 0x00; + +const ROOT_REGION_A: usize = 0x08; + +const ROOT_MAIN_CONFIG: usize = 0x18; + +const ROOT_REGION_C: usize = 0x20; + +const ROOT_INSTANCE_KIND: usize = 0x28; + +const ROOT_CONSTANT_2C: usize = 0x2c; + +const ROOT_PAGE_SIZE: usize = 0x30; + +const ROOT_PAGE_BITS: usize = 0x32; + +const ROOT_LEVEL_COUNT: usize = 0x33; + +const ROOT_LEVELS: usize = 0x34; + +/// Root `+0xa8`: Status A pointer. (The M5-derived model misused this slot as +/// a role pointer.) +const ROOT_STATUS_A: usize = 0xa8; + +/// Root `+0xb0`: Status B pointer. Real for the primary, zero for the +/// secondary. (The M5-derived model misused it as a cross-role pointer.) +const ROOT_STATUS_B: usize = 0xb0; + +const ROOT_SECONDARY_EXTRA_0: usize = 0xb8; + +const ROOT_SECONDARY_EXTRA_1: usize = 0xc0; + +pub(crate) const UAT_PAGE_SIZE: u16 = 0x4000; + +pub(crate) const UAT_PAGE_BITS: u8 = 14; + +pub(crate) const UAT_LEVEL_COUNT: usize = 3; + +/// `(index_shift, entry_count)` for the three levels below the root, +/// confirmed against a live table walk. +pub(crate) const UAT_LEVELS: [(u8, u16); UAT_LEVEL_COUNT] = [(36, 64), (25, 2048), (14, 2048)]; + +pub(crate) const UAT_LEVEL_DESC_SIZE: usize = 0x20; + +/// Level descriptor `+0x10`: physical mask, identical in all three. +pub(crate) const UAT_LEVEL_PHYS_MASK: u64 = 0x0000_03ff_ffff_c000; + +const MAIN_HW_DATA: usize = 0x00; + +/// The same repeated-region address appears at `+0x08` and `+0x10`; it is the +/// bundle base plus [`MAIN_REPEATED_REGION_BUNDLE_OFFSET`], not the bundle +/// base itself. It reads zero at handoff (firmware writes into it), so the +/// host only has to provide it mapped. +const MAIN_REPEATED_0: usize = 0x08; + +const MAIN_REPEATED_1: usize = 0x10; + +pub(crate) const MAIN_REPEATED_REGION_BUNDLE_OFFSET: usize = 0xc500; + +const MAIN_CHANNEL_TABLE: usize = 0x20; + +/// Main `+0x254`: five unaligned u64 views into the hardware-data bundle. +const MAIN_BUNDLE_VIEWS: usize = 0x254; + +/// Main `+0x2d0`: six qwords of region views. +const MAIN_REGION_VIEWS: usize = 0x2d0; + +/// Main `+0x300`: u32 4, secondary only. +const MAIN_SECONDARY_KIND: usize = 0x300; + +const MAIN_SECONDARY_KIND_VALUE: u32 = 4; + +/// Main `+0x3e0`: u32 0xff, primary only. +const MAIN_PRIMARY_MASK: usize = 0x3e0; + +const MAIN_PRIMARY_MASK_VALUE: u32 = 0xff; + +const MAIN_PRIMARY_SCALAR_32C: usize = 0x32c; + +const MAIN_PRIMARY_SCALAR_344: usize = 0x344; + +/// Main `+0x471`: unaligned u64, secondary only, pointing *into* bundle view 2 +/// at [`SECONDARY_EXTRA_VIEW_OFFSET`] (not a fresh allocation — the M5-derived +/// model had this at `+0x481` backed by an invented 0x12688-byte buffer; both +/// were wrong). +const MAIN_SECONDARY_EXTRA_PTR: usize = 0x471; + +pub(crate) const SECONDARY_EXTRA_VIEW_INDEX: usize = 2; + +pub(crate) const SECONDARY_EXTRA_VIEW_OFFSET: usize = 0xe40; + +/// Main `+0x4c0` is both a scalar and a place: the leading u32 of the +/// device-control ring, which is embedded in the main configuration object at +/// exactly this offset on both instances. Writing the opening opcode here *is* +/// staging the opening device-control record (records are 0x40 bytes; fields +/// past the opcode are zero). The control channel's producer counter must then +/// be set to the role-specific value returned by +/// [`control_opening_producer`] before the descriptor is handed over. +const MAIN_CONTROL_RING: usize = 0x4c0; + +pub(crate) const CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x16; + +pub(crate) const CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; + +/// Bundle offsets of the five views named at main `+0x254`. They are views of +/// the common bundle, not independent objects: view 0 overlaps the tail of +/// the hardware-data object, and views 3 and 4 overlap each other by all but +/// 0x80 bytes. +pub(crate) const BUNDLE_VIEW_OFFSETS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; + +/// Logical extents of the five views. +pub(crate) const BUNDLE_VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; + +/// The context-0 mappings of the PB descriptor and UMA page-pool descriptor +/// tables published at main `+0x2d8` and `+0x2e8`, respectively. The matching +/// firmware-high mappings are published at `+0x2e0` and `+0x2f0`. +pub(crate) const REGION_VIEW_LOW_ADDRS: [u64; 2] = [0x70_0183_8000, 0x70_0184_0000]; + +pub(crate) const CHANNEL_TABLE_ENTRIES: usize = 17; + +pub(crate) const CHANNEL_ENTRY_SIZE: usize = 0x20; + +/// Channel entry layout: three *separate* 32-bit state-counter addresses at +/// `+0x00`/`+0x08`/`+0x10` (the counters themselves sit at 0x10 spacing in +/// their state block — do not collapse them into one state object), then the +/// ring base at `+0x18`. +pub(crate) const CHANNEL_ENTRY_STATE_COUNT: usize = 3; + +pub(crate) const CHANNEL_STATE_SPACING: usize = 0x10; + +/// Entries 0..11 are ordered TA_0, 3D_0, CL_0, TA_1, ... CL_3. +pub(crate) const WORK_CHANNEL_COUNT: usize = 12; + +/// Entry 12 is device control. It continues the compact control-state grid +/// ([`WORK_STATE_GRID_OFFSETS`] has 13 entries, one more than there are work +/// channels), and its ring is embedded at main `+0x4c0`. The M5-derived model +/// refused this index, claiming a "cursor pair at +0x1a0/+0x1a8"; that was a +/// misread — those offsets are exactly this entry's state-counter addresses 0 +/// and 1. +pub(crate) const CONTROL_CHANNEL_INDEX: usize = 12; + +/// Entries 13 and 14 are report channels with split state objects: their +/// state addresses alias off status A rather than the compact grid, and the +/// firmware peer of each host-owned counter is at `+0x20` within its object. +pub(crate) const REPORT_CHANNEL_INDICES: [usize; 2] = [13, 14]; + +/// Entry 15 is partial: only its first state address, no ring. Entry 16 is +/// empty. Filling them like the rest declares channels that do not exist. +pub(crate) const PARTIAL_CHANNEL_INDEX: usize = 15; + +/// Offsets of the channel state blocks (channels 0..=12) inside the compact +/// state grid. Native state is *not* in channel-table order: within each +/// workload group the work queues sit at descending offsets, and the control +/// channel's block follows at `+0x300`. Preserving these address identities is +/// part of the descriptor ABI. +pub(crate) const WORK_STATE_GRID_OFFSETS: [usize; 13] = [ + 0x0c0, 0x1c0, 0x2c0, // TA_0, 3D_0, CL_0 + 0x080, 0x180, 0x280, // TA_1, 3D_1, CL_1 + 0x040, 0x140, 0x240, // TA_2, 3D_2, CL_2 + 0x000, 0x100, 0x200, // TA_3, 3D_3, CL_3 + 0x300, // device control +]; + +/// The primary status-B object runs from there to status A: `0xeb00` bytes. +pub(crate) const PRIMARY_STATUS_B_OBJECT_SIZE: usize = 0xeb00; + +/// State-counter addresses of report channels 13/14, as offsets from status A. +pub(crate) const REPORT_STATE_STATUS_A_OFFSETS: [[usize; 3]; 2] = + [[0x00040, 0x002c0, 0x00080], [0x00240, 0xa6ac0, 0x00280]]; + +/// Ring bases of report channels 13/14, as offsets from status A. +pub(crate) const REPORT_RING_STATUS_A_OFFSETS: [usize; 2] = [0x04ac0, 0xafac0]; + +/// Entry 15's sole state address, as an offset from status A. +pub(crate) const PARTIAL_STATE_STATUS_A_OFFSET: usize = 0x2d2c0; + +/// Native bundle offsets of the twelve work rings (0x1800 each), in +/// channel-table order. These extend past [`HW_DATA_BUNDLE_MIN_ALLOC`]; a host +/// using this placement sizes the cluster [`NATIVE_SHARED_CLUSTER_SIZE`]. +pub(crate) const WORK_RING_BUNDLE_OFFSETS: [usize; 12] = [ + 0x10dc0, 0x16dc0, 0x1cdc0, 0x0f5c0, 0x155c0, 0x1b5c0, 0x0ddc0, 0x13dc0, 0x19dc0, 0x0c5c0, + 0x125c0, 0x185c0, +]; + +/// Native bundle offsets of the two main configuration objects. +pub(crate) const NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET: usize = 0x1e5c0; + +pub(crate) const NATIVE_SECONDARY_MAIN_BUNDLE_OFFSET: usize = 0x22a80; + +pub(crate) const HW_REGISTER_MAP_ENTRY_SIZE: usize = 0x28; + +pub(crate) const HW_REGISTER_MAP_SLOTS: usize = 53; + +/// Two GPU mappings of one present completion-queue backing. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmCompletionAliases { + pub(crate) low: u64, + pub(crate) high: u64, +} + +pub(crate) const M5_25G76_COMPLETION_QUEUE_COUNT: usize = 4; + +pub(crate) const M5_25G76_COMPLETION_BACKING_SIZE: usize = 0x800 * 0x40; + +/// M5 publishes all four descriptors at hardware-data +0x2614, with ordinals +/// 0,1,2,3. Each names low/high mappings of its own retained 128-KiB backing. +/// The two native pre-handoff captures agree on every descriptor byte. +/// Construction only: this does not allocate, map, publish, or poll a queue. +pub(crate) fn encode_m5_25g76_completion_table( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_COMPLETION_QUEUE_COUNT * 0x20 { + return Err(InitdataError::BufferSize); + } + let extent = M5_25G76_COMPLETION_BACKING_SIZE as u64; + // Validate every range before mutating the output. Different queues must + // not alias each other in either address space. + for (i, alias) in aliases.iter().enumerate() { + for address in [alias.low, alias.high] { + required_address(address)?; + let end = address.checked_add(extent).ok_or(InitdataError::AddressOutOfRange)?; + required_address(end - 1)?; + } + for prior in &aliases[..i] { + if (alias.low < prior.low + extent && prior.low < alias.low + extent) + || (alias.high < prior.high + extent && prior.high < alias.high + extent) + { + return Err(InitdataError::PairAliasedObject); + } + } + } + out.fill(0); + for (ordinal, alias) in aliases.iter().enumerate() { + let offset = ordinal * 0x20; + put_u64(out, offset, alias.low); + put_u64(out, offset + 8, alias.high); + put_u32(out, offset + 0x10, 0x800); + put_u32(out, offset + 0x14, 0x40); + put_u32(out, offset + 0x18, ordinal as u32); + } + Ok(()) +} + +/// Status block `+0x04` holds u32 1 pre-init. `+0x10`/`+0x14` are *post-ack +/// firmware state* and must be zero at handoff. +pub(crate) const STATUS_BLOCK_ONE_OFFSET: usize = 0x04; + +pub(crate) const PRIMARY_STATUS_B_FWCTL_STATE: usize = 0x48e0; + +pub(crate) const PRIMARY_STATUS_B_FWCTL_RING: usize = 0x48e8; + +pub(crate) const PRIMARY_STATUS_B_CONFIG_HEADER: usize = 0xe434; + +pub(crate) const PRIMARY_STATUS_B_CONFIG_OFFSET: usize = 0xe440; + +/// The secondary root sits exactly this far above the primary root; the +/// secondary's initialisation message carries the primary's address plus this +/// delta in the same field, so the pair is not placed freely. +pub(crate) const SECONDARY_ROOT_DELTA: u64 = 0x8000; + +/// Physical firmware instance encoded at root `+0x28`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum InstanceRole { + /// `gfx-asc`: scheduler, device control, and all work channels. + Primary = 0, + /// `gfx1-asc`: control-only peer. Linux starts it after primary RTKit is ready. + Secondary = 1, +} + +pub(crate) const fn root_size(role: InstanceRole) -> usize { + match role { + InstanceRole::Primary => ROOT_SIZE_PRIMARY, + InstanceRole::Secondary => ROOT_SIZE_SECONDARY, + } +} + +/// GPU virtual addresses of the objects both instances share. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedAddresses { + /// Shared 16 KiB region A (root `+0x08`). + pub(crate) region_a: u64, + /// Shared region C (root `+0x20`), 0x1000 bytes. + pub(crate) region_c: u64, + /// Base of the contiguous hardware-data bundle. The hardware-data object + /// sits at the base; the five main-config views, the repeated region, and + /// the secondary's `+0x471` pointer are all derived offsets into it. + pub(crate) hw_data_bundle: u64, + /// Size of that one allocation; must be at least + /// [`HW_DATA_BUNDLE_MIN_ALLOC`]. + pub(crate) hw_data_bundle_alloc: usize, +} + +/// GPU virtual addresses of one instance's private objects. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct InstanceAddresses { + /// This instance's root; its address rides the endpoint-0x20 doorbell. + pub(crate) root: u64, + pub(crate) main_config: u64, + pub(crate) status_a: u64, + /// Required for the primary; must be zero for the secondary (whose root + /// still has the field, written zero). + pub(crate) status_b: u64, + /// Root `+0xb8`/`+0xc0`. Required for the secondary; the primary root is + /// only 0xb8 bytes and cannot even hold them, so they must be zero there. + pub(crate) secondary_extras: [u64; 2], +} + +/// One channel-table entry: three state-counter addresses, then the ring base. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ChannelEntry { + pub(crate) states: [u64; 3], + pub(crate) ring: u64, +} + +impl ChannelEntry { + pub(crate) const EMPTY: ChannelEntry = ChannelEntry { + states: [0; 3], + ring: 0, + }; +} + +/// The primary's global region views at main `+0x2d0`: a globally retained, +/// initially zero HWDSID accounting table (256 pairs of u32), followed by +/// the high halves of two low/high alias pairs (the low halves +/// are the fixed context-0 addresses in +/// [`REGION_VIEW_LOW_ADDRS`]). Each pair must map one physical page from both +/// address spaces — an invariant of the UAT mapping layer that cannot be +/// checked from the addresses alone. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrimaryRegionViews { + pub(crate) sentinel_high: u64, + pub(crate) alias_high: [u64; 2], +} + +/// M5 25G76 descriptor fields +0x10 and +0x14 are independent. In slot 39 +/// native publishes 0x10000 and 0x8000 respectively. The older RegisterWindow +/// encoder duplicates one size and must not serialize this target's table. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RegisterWindow { + pub(crate) slot: usize, + pub(crate) phys: u64, + pub(crate) device_va: u64, + pub(crate) size_10: u32, + pub(crate) size_14: u32, + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +impl M5RegisterWindow { + /// Actual M5 PTE walks cover size_10 rounded to 16 KiB, including the + /// four pages of slot 39. size_14 is independently serialized firmware + /// metadata and must not truncate that mapping to two pages. + pub(crate) fn mapped_pages(&self) -> Result<(u64, u64, u64), InitdataError> { + let mask = UAT_PAGE_SIZE as u64 - 1; + if self.size_10 == 0 || self.phys & mask != self.device_va & mask { + return Err(InitdataError::RegisterWindow); + } + let physical = self.phys & !mask; + let virtual_address = self.device_va & !mask; + let end = self.device_va.checked_add(self.size_10 as u64) + .and_then(|value| value.checked_add(mask)) + .ok_or(InitdataError::AddressOutOfRange)? & !mask; + let span = end.checked_sub(virtual_address).ok_or(InitdataError::AddressOutOfRange)?; + physical.checked_add(span).ok_or(InitdataError::AddressOutOfRange)?; + Ok((physical, virtual_address, span)) + } +} + +/// Source-constructed register descriptors from both identical pre-handoff +/// J813/25G76 captures. These are firmware virtual mappings, not a request to +/// access or power the devices. No runtime currently consumes this profile. +pub(crate) const M5_25G76_REGISTER_WINDOWS: [M5RegisterWindow; 22] = { + const fn w(slot: usize, phys: u64, va: u64, size_10: u32, size_14: u32, + unk_18: u64, flag: u32) -> M5RegisterWindow { + M5RegisterWindow { slot, phys, device_va: 0xffff_fc21_8000_0000 + va, + size_10, size_14, unk_18, flag } + } + [ + w(0, 0x381014000, 0x000000, 0x04000, 0x04000, 0, 2), + w(3, 0x220104000, 0x008000, 0x18000, 0x18000, 0, 2), + w(8, 0x3a1350000, 0x028000, 0x04000, 0x04000, 0, 2), + w(9, 0x3803d0000, 0x030000, 0x04000, 0x04000, 0, 2), + w(10, 0x3803c0000, 0x038000, 0x04000, 0x04000, 0, 0), + w(12, 0x42165c000, 0x040000, 0x04000, 0x04000, 0, 2), + w(14, 0x380280000, 0x048000, 0x08000, 0x08000, 0, 0), + w(15, 0x38c840000, 0x058000, 0x24000, 0x24000, 0, 2), + w(17, 0x580000000, 0x080000, 0x21400, 0x21400, 0, 2), + w(22, 0x581000000, 0x0a8000, 0x08000, 0x08000, 0, 2), + w(24, 0x580ec0000, 0x0b8000, 0x10000, 0x10000, 0, 2), + w(26, 0x580d04000, 0x0d0000, 0x08000, 0x08000, 0xd04000, 2), + w(27, 0x580d0d000, 0x0e1000, 0x01000, 0x01000, 0xd0d000, 2), + w(28, 0x580d50000, 0x0e8000, 0x10000, 0x10000, 0xd50000, 2), + w(29, 0x580d10000, 0x100000, 0x04000, 0x04000, 0xd10000, 2), + w(31, 0x580d40000, 0x108000, 0x04000, 0x04000, 0xd40000, 2), + w(32, 0x580d60000, 0x110000, 0x08000, 0x08000, 0xd60000, 2), + w(35, 0x580e00000, 0x120000, 0x04000, 0x04000, 0xe00000, 2), + w(39, 0x580e08000, 0x128000, 0x10000, 0x08000, 0, 2), + w(41, 0x580e33800, 0x143800, 0x04000, 0x04000, 0, 2), + w(42, 0x580e35000, 0x151000, 0x04000, 0x04000, 0, 2), + w(50, 0x580014400, 0x160400, 0x00400, 0x00400, 0x14400, 0), + ] +}; + +pub(crate) fn encode_m5_25g76_register_map(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for window in M5_25G76_REGISTER_WINDOWS { + let base = window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, window.phys); + put_u64(out, base + 8, window.device_va); + put_u32(out, base + 0x10, window.size_10); + put_u32(out, base + 0x14, window.size_14); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + for slot in [2, 5, 6, 7, 30, 33, 34, 37, 38, 40, 43, 46, 47, 48, 49, 52] { + put_u32(out, slot * HW_REGISTER_MAP_ENTRY_SIZE + 0x20, 2); + } + Ok(()) +} + +pub(crate) const M5_25G76_HW_DATA_SIZE: usize = 0x2708; + +pub(crate) const M5_25G76_SHARED_PREFIX_SIZE: usize = 0xc500; + +pub(crate) const M5_25G76_AON_PTD_SIZE: usize = 0xe4; + +/// J813/25G76 AON PTD backing, captured twice before each role's handoff and +/// unchanged after a passing Metal workload. The three recorded words are +/// scoped to this platform's telemetry description; they are not generic +/// M5 constants. The allocation and GPU binding must remain caller-owned. +pub(crate) fn encode_m5_25g76_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_25G76_AON_PTD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, 0x6730_5452); + put_u32(out, 4, 0x001d_0001); + put_u32(out, 8, 0x0000_1401); + Ok(()) +} + +const M5_25G76_STATUS_CONFIG_WORDS: &[(usize, u32)] = &[ + (0x010, 0x0000001f), (0x014, 0x3e0a3d71), (0x018, 0x401ae148), + (0x01c, 0x00000028), (0x020, 0x00000001), (0x0c8, 0x00002af8), + (0x0cc, 0x00001f40), (0x0d8, 0x41980000), (0x0dc, 0x3d0f5c28), + (0x0f0, 0x000026ac), (0x0f4, 0x000000c8), (0x108, 0x42c80000), + (0x10c, 0x43c80000), (0x110, 0x000000c8), (0x128, 0x00000001), + (0x12c, 0x00005eb0), (0x130, 0x3e800000), (0x134, 0x3f99999a), + (0x158, 0x0000000e), (0x1c0, 0x00000001), (0x1c4, 0x00000001), + (0x1c8, 0x00000002), (0x1cc, 0x00000001), (0x1d0, 0x00000001), + (0x1d4, 0x00000001), (0x1d8, 0x00000001), (0x588, 0x00000001), + (0x58c, 0x0000000a), (0x590, 0x00000dac), (0x594, 0x000003e8), + (0x598, 0x00000bb8), (0x59c, 0x000002bc), (0x5cc, 0x00000001), + (0x5d0, 0x00000004), (0x5dc, 0x00001f40), (0x5e0, 0x000000c8), + (0x5e4, 0x00000fa0), (0x5e8, 0x0000012c), (0x5ec, 0x000007d0), + (0x5f0, 0x0000012c), (0x5f4, 0x000003e8), (0x5f8, 0x0000012c), + (0x600, 0x00000001), (0x604, 0x41700000), (0x608, 0x40a00000), + (0x60c, 0x00000020), (0x610, 0x000004b0), (0x620, 0x0000044c), +]; + +/// Build the primary status object before either firmware role is started. +/// Secondary root+0xb8 aliases status+0xe440. GFX1 copies that pointer to +/// firmware global 0x176248 at text+0x211d4 and later writes config+0x670, +/// +0x678 and +0x680 at text+0x1af94..0x1afc4. Those are firmware-owned +/// runtime state, not platform constants, and must start at zero even when +/// a primary pre-handoff capture (taken after secondary startup) has values. +pub(crate) fn encode_m5_25g76_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != PRIMARY_STATUS_B_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, PRIMARY_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, PRIMARY_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, PRIMARY_STATUS_B_CONFIG_HEADER, 1); + for &(offset, value) in M5_25G76_STATUS_CONFIG_WORDS { + put_u32(out, PRIMARY_STATUS_B_CONFIG_OFFSET + offset, value); + } + Ok(()) +} + +/// Shared private arena observed before primary initialization on J813. +/// The status and secondary-grid pointers name overlapping views of this +/// one backing. This extent is a measured mapped arena, not a claim about +/// the sizes of the native host's individual suballocations. +pub(crate) const M5_25G76_PRIVATE_CLUSTER_SIZE: usize = 0x178000; + +pub(crate) fn encode_m5_25g76_private_cluster( + fwctl_state: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_PRIVATE_CLUSTER_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + let fwctl_ring = fwctl_state.checked_add(0x40).ok_or(InitdataError::AddressOutOfRange)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_m5_25g76_primary_status_b(fwctl_state, fwctl_ring, &mut out[0x340..0xee40])?; + for offset in [0xee40, 0xc3100] { + encode_status_block(&mut out[offset..offset + STATUS_BLOCK_SIZE])?; + } + encode_secondary_root_extra_1_for_profile(RootProfile::J813G17G25G76, + &mut out[0x177400..0x177480])?; + // The main-config encoders carry one measured opening record (0x16 / + // 0x2a). Publish exactly one producer entry for each control channel. + // All work-channel and report counters remain zero. + put_u32(out, 0x320, 1); + put_u32(out, 0x1773e0, 1); + Ok(()) +} + +/// Caller-owned bindings in the power/performance policy allocation. +/// The host stores mach_absolute_time() at policy+0x651c. Supply the live +/// platform counter in its native timebase, never a captured timestamp. +#[derive(Clone, Copy)] +pub(crate) struct M5PolicyBindings { + pub state: u64, + pub aon_ptd: u64, + pub boot_timestamp: u64, +} + +/// Pre-handoff J813/25G76 scalars, identical across two cold boots after +/// removing owned pointers and the native boot timestamp. The repeated +/// 512-entry u16 table is encoded separately. Calibration fields whose +/// meanings remain unknown are scoped to this measured ten-core target. +const M5_25G76_POLICY_WORDS: &[(usize, u32)] = &[ + (0x3d98, 0xffffffff), (0x3db0, 0xffffffff), (0x5244, 0x00030d40), + (0x5248, 0x00030d40), (0x5250, 0x00000004), (0x5254, 0x3f800000), + (0x526c, 0x00000001), (0x5270, 0x00000001), (0x5280, 0x00000064), + (0x5284, 0x00000001), (0x5288, 0x000004b0), (0x5290, 0x00000064), + (0x52a0, 0x00000004), (0x52a4, 0x3f800000), (0x52ac, 0x00000001), + (0x52b0, 0x00000001), (0x52c0, 0x00000064), (0x52c4, 0x00000001), + (0x52c8, 0x000004b0), (0x52d0, 0x00000064), (0x5300, 0x3f828f5c), + (0x5304, 0x3f828f5c), (0x5308, 0x3f828f5c), (0x530c, 0x3f828f5c), + (0x5310, 0x3f828f5c), (0x5314, 0x3f828f5c), (0x5318, 0x3f828f5c), + (0x531c, 0x3f828f5c), (0x5320, 0x3f828f5c), (0x5324, 0x3f828f5c), + (0x5328, 0x3f828f5c), (0x532c, 0x3f828f5c), (0x5330, 0x3f828f5c), + (0x5bf8, 0x00000271), (0x5c04, 0x3f7f2e9f), (0x5c0c, 0x3b516154), + (0x5c14, 0x3ca59586), (0x5c20, 0x46bd6000), (0x5c24, 0x40a90fe4), + (0x5c2c, 0xbd4ac5b4), (0x5c30, 0x00000028), (0x5c34, 0x000004b0), + (0x5c38, 0x000004b0), (0x5c44, 0x00005eb0), (0x5c78, 0x000004b0), + (0x5cb0, 0x00000008), (0x5cbc, 0x3f000000), (0x5cc4, 0x3f000000), + (0x5ccc, 0x3f99999a), (0x5cd8, 0x47800000), (0x5cdc, 0x3e800000), + (0x5ce8, 0x00000028), (0x5cec, 0x000004b0), (0x5cfc, 0x00005eb0), + (0x5d08, 0x00000010), (0x5d10, 0x0005dc00), (0x5d68, 0x0000004a), + (0x5d70, 0x0000005a), (0x5d74, 0x00000032), (0x5d78, 0x00000006), + (0x5d80, 0x00000006), (0x5d84, 0x00000001), (0x5d8c, 0x3f4ccccd), + (0x5d90, 0x3f7eb852), (0x5d94, 0x3e4ccccd), (0x5d98, 0x3ba3d70a), + (0x5d9c, 0x3f144b3c), (0x5da0, 0x3f144b3c), (0x5da8, 0x42be0000), + (0x5dac, 0x4029eb85), (0x5db0, 0x4029eb85), (0x5db4, 0x4139435e), + (0x5db8, 0x00000064), (0x5dbc, 0x000004b0), (0x5dc0, 0x00000064), + (0x5dcc, 0x0000004a), (0x5dd4, 0x00000001), (0x5e00, 0x00000064), + (0x5e20, 0x00000401), (0x5e24, 0x00006400), (0x5e2c, 0x001f4000), + (0x5e30, 0x0000c800), (0x5e34, 0x000fa000), (0x5e38, 0x00012c00), + (0x5e3c, 0x0007d000), (0x5e40, 0x00012c00), (0x5e44, 0x0003e800), + (0x5e48, 0x00012c00), (0x5e4c, 0x00000100), (0x5ea4, 0x00000001), + (0x5ed0, 0x00000001), (0x74d8, 0x47800000), (0x74dc, 0x42200000), + (0x74e0, 0x44960000), (0x74e4, 0x0000062a), (0x74f8, 0x00000028), + (0x7500, 0x42c80000), (0x7504, 0x000004b0), (0x750c, 0x3f4ccccd), + (0x7510, 0x3e4ccccd), (0x7550, 0xdcdcfbfa), (0x7558, 0x00001f40), + (0x7560, 0x00000004), (0x7574, 0x3f800000), (0x757c, 0x3d0f5c28), + (0x7588, 0x47800000), (0x758c, 0x41980000), (0x7598, 0x00000028), + (0x759c, 0x000004b0), (0x75a0, 0x000004b0), (0x75ac, 0x00002af8), + (0x75e0, 0x000004b0), (0x77f8, 0x00000002), (0x77fc, 0x42000006), + (0x7a10, 0xb8520000), (0x7a14, 0x00003f7e), (0x7a18, 0xd70a0000), + (0x7a1c, 0x00003ba3), (0x7a24, 0x000043c8), (0x7a30, 0x00004780), + (0x7a34, 0x000042c8), (0x7a3c, 0x0384bac8), (0x7a40, 0x03e80000), + (0x7a44, 0x03e80000), (0x7a4c, 0x80000000), (0x7a50, 0x26ac453b), + (0x7a84, 0x03e80000), (0x7aa0, 0x00c80000), (0x7b58, 0x000004b0), + (0x7b6c, 0x00000001), (0x7ba0, 0x00000101), (0x7ba4, 0x00000200), + (0x7bbc, 0x00023f00), (0x7bc0, 0x00000100), (0x7bc4, 0x00000100), + (0x7bdc, 0x00024000), (0x7be0, 0x00000100), (0x7be4, 0x00000100), + (0x7bfc, 0x00024100), (0x7c48, 0x00000001), (0x7c50, 0x00005eb0), + (0x7c54, 0x00005eb0), (0x7c58, 0x00005eb0), (0x7c68, 0x3f800000), + (0x7c6c, 0x00000004), (0x7c70, 0x00000008), (0x7c74, 0x0002ee00), + (0x7d44, 0x0000007c), (0x7d50, 0x3f77bdef), (0x7d58, 0x3d042108), + (0x7d60, 0x3e0a3d71), (0x7d6c, 0x47800000), (0x7d70, 0x401ae148), + (0x7d7c, 0x00000028), (0x7d80, 0x000004b0), (0x7d84, 0x000004b0), + (0x7d8c, 0x46bd6000), (0x7d90, 0x00005eb0), (0x7d9c, 0x000000f8), + (0x7da4, 0x005ad200), (0x7dc4, 0x000004b0), (0x7e04, 0x00000001), + (0x7ed4, 0x00000032), (0x7ed8, 0x00000001), (0x7ee0, 0x3f638e39), + (0x7ee4, 0x3f2aaaab), (0x7ee8, 0x3de38e39), (0x7eec, 0x3eaaaaab), + (0x7ef0, 0xbecccccd), (0x7ef4, 0xbf4ccccd), (0x7efc, 0x47800000), + (0x7f00, 0xc0a00000), (0x7f04, 0xc0a00000), (0x7f0c, 0x00000064), + (0x7f10, 0x000004b0), (0x7f14, 0x00000064), (0x7f1c, 0x45fa0000), + (0x7f20, 0x00000578), (0x7f28, 0x0000000e), (0x7f2c, 0x00000048), + (0x7f30, 0x00000018), (0x7f34, 0x001a5e00), (0x7f3c, 0x0008ca00), + (0x7f48, 0x47800000), (0x7f70, 0x00001f40), (0x7f74, 0x00000004), + (0x7f78, 0x000009c4), (0x7f7c, 0x00000258), (0x7f80, 0x00000002), + (0x7f84, 0x00000004), (0x7f88, 0x00000032), (0x7fb8, 0x47800000), + (0x7fc8, 0x00000028), (0x7fcc, 0x000004b0), (0x8098, 0x00000001), + (0x80ac, 0x0000000a), (0x80b8, 0x00000001), (0x8174, 0xac000000), + (0x8178, 0xe800000d), (0x817c, 0xb8000003), (0x8180, 0xb000000b), + (0x8184, 0xb0000004), (0x8188, 0xbc000004), (0x818c, 0x20000002), + (0x8198, 0x00000001), (0x81b4, 0x00000004), (0x81c0, 0x3f780000), + (0x81c8, 0x3d000000), (0x81d0, 0x40a00000), (0x81dc, 0x47800000), + (0x81e0, 0x41700000), (0x81ec, 0x0000044c), (0x81f0, 0x000004b0), + (0x81f4, 0x000004b0), (0x8200, 0x000004b0), (0x8234, 0x000004b0), + (0x827c, 0x00000064), (0x8290, 0xdcdcfbfa), (0x8298, 0x0000007d), + (0x829c, 0x00000001), (0x8ee0, 0xdcdcfbfb), (0x8ee8, 0x00000001), + (0xaeec, 0x41c80000), (0xaef0, 0x41c80000), (0xb6a8, 0x67305452), + (0xb6e8, 0x00000001), (0xb728, 0x5243464c), (0xb72c, 0x4758464c), + (0xb738, 0x47584c50), (0xb748, 0x00000001), (0xb7a8, 0x00000001), + (0xb7b4, 0x42b73333), (0xb7f4, 0x449b8000), (0xb834, 0x42bc0000), + (0xb8e0, 0x0000186a), +]; + +/// Initialize the six allocations following M5 hardware-data. Preserve the +/// hardware block and subsequent channel resources. No firmware is started. +pub(crate) fn encode_m5_25g76_bundle_policy( + bindings: M5PolicyBindings, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() < M5_25G76_SHARED_PREFIX_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(bindings.state)?; + required_address(bindings.aon_ptd)?; + let state_end = bindings.state.checked_add(STATUS_BLOCK_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + let aon_end = bindings.aon_ptd.checked_add(M5_25G76_AON_PTD_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(state_end - 1)?; + required_address(aon_end - 1)?; + if bindings.state < aon_end && bindings.aon_ptd < state_end { + return Err(InitdataError::PairAliasedObject); + } + out[0x2740..M5_25G76_SHARED_PREFIX_SIZE].fill(0); + for &(offset, value) in M5_25G76_POLICY_WORDS { + put_u32(out, offset, value); + } + for index in 0..512 { + put_u16(out, 0x6ed7 + index * 2, 0x7fff); + } + put_u64(out, 0x8ed8, bindings.state); + put_u64(out, 0xb6a0, bindings.aon_ptd); + put_u64(out, 0xb75c, bindings.boot_timestamp); + Ok(()) +} + +/// Recorded platform words outside the explicit register, performance and +/// owned-pointer fields. Meanings not yet derived are deliberately kept as +/// target-scoped measured values. Never insert runtime completion counters +/// or captured allocation addresses here. Reference hardware-data SHA-256: +/// 6f9ac0c4664f898a0ddb4e5df9ad160a23c1b52d3fd750750779cbdebe446be9. +const M5_25G76_HW_PLATFORM_WORDS: &[(usize, u32)] = &[ + (0x0004, 0x0000006f), (0x0008, 0xffc00000), (0x0014, 0x00000010), + (0x001c, 0x00000010), (0x0020, 0xffff8000), (0x0024, 0x000002ff), + (0x0028, 0x81400000), (0x002c, 0xfffffc21), (0x00e0, 0x00002008), + (0x00e8, 0x20080000), (0x00f4, 0x00002008), (0x00f8, 0x000024cb), + (0x00fc, 0xc98a2cfa), (0x0100, 0xf4f624cb), (0x0104, 0x1877e917), + (0x0108, 0x38d924cb), (0x010c, 0xbdab0000), (0x02d8, 0xfff82000), + (0x02dc, 0xd32d2cdb), (0x02e0, 0xf5002000), (0x02e4, 0x21dae926), + (0x02e8, 0x38b62000), (0x02ec, 0xc7420008), (0x03e0, 0x00007fe0), + (0x03e8, 0x7fe00000), (0x03f4, 0x00007fe0), (0x03f8, 0x00008000), + (0x0400, 0x80000000), (0x040c, 0x00008000), (0x05d8, 0x4b232645), + (0x05dc, 0x00000e98), (0x05e0, 0xd5a2ea5f), (0x05e4, 0x40003fff), + (0x05e8, 0xca5e4000), (0x05ec, 0x4000f5a2), (0x0e90, 0x00008142), + (0x0e94, 0x00000002), (0x0ea0, 0x00000001), (0x0ea4, 0x00000001), + (0x0eb8, 0x00000001), (0x0ec0, 0x00000001), (0x0ec8, 0x00000001), + (0x0ed0, 0x00005dc0), (0x0ed4, 0x00000001), (0x0ed8, 0x00000008), + (0x0ee0, 0x00000001), (0x0ee4, 0x00000001), (0x0ee8, 0x00000001), + (0x0f04, 0x0000001f), (0x0f24, 0x07800004), (0x0f28, 0x00000006), + (0x0f34, 0x00000001), (0x0f38, 0x00000001), (0x0f4c, 0x00000031), + (0x0f6c, 0x00000001), (0x0f88, 0x0000000a), (0x0f8c, 0x00000001), + (0x0fac, 0x00000001), (0x0fb8, 0x00000020), (0x0fbc, 0x00000005), + (0x0fc0, 0x0000000a), (0x0fc4, 0x0000000c), (0x2544, 0x00000004), + (0x2548, 0x00000006), (0x254c, 0x00000003), (0x2550, 0x00000007), + (0x2554, 0x00000007), (0x2568, 0x0000000a), (0x2570, 0x00000001), + (0x258c, 0x00180000), (0x2594, 0x00000001), (0x2598, 0x00000001), + (0x259c, 0x00000001), (0x25a4, 0x00000001), (0x25a8, 0x00000001), + (0x25ac, 0x00000001), (0x25b4, 0xffffffff), (0x25b8, 0xffffffff), + (0x25bc, 0xffffffff), (0x25c0, 0xffffffff), (0x25c4, 0xffffffff), + (0x25c8, 0xffffffff), (0x25cc, 0xffffffff), (0x25d0, 0xffffffff), + (0x25d4, 0xffffffff), (0x25d8, 0xffffffff), (0x25dc, 0xffffffff), + (0x25e0, 0xffffffff), (0x25f4, 0x00000001), (0x2600, 0x00000001), + (0x2608, 0x00000001), (0x2610, 0x00000100), (0x26b4, 0x00000001), + (0x26b8, 0x00000001), (0x26e0, 0x000003e8), (0x26e8, 0x00000001), +]; + +/// Reconstruct the M5 reference hardware-data block with caller-owned queue +/// and QoS bindings. Performance values are measured on the ten-core J813 +/// reference, not derived from ADT and not a generic M5 power configuration. +/// Both pre-handoff firmware roles supplied identical reference bytes. +/// This encoder grants no runtime admission and performs no MMIO accesses. +pub(crate) fn encode_m5_25g76_hw_data( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + qos_high: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(qos_high)?; + let qos_end = qos_high.checked_add(0xe40).ok_or(InitdataError::AddressOutOfRange)?; + required_address(qos_end - 1)?; + let mut completion = [0; 0x80]; + encode_m5_25g76_completion_table(aliases, &mut completion)?; + for alias in aliases { + if qos_high < alias.high + M5_25G76_COMPLETION_BACKING_SIZE as u64 + && alias.high < qos_end { + return Err(InitdataError::PairAliasedObject); + } + } + + out.fill(0); + for &(offset, value) in M5_25G76_HW_PLATFORM_WORDS { + put_u32(out, offset, value); + } + encode_m5_25g76_register_map(&mut out[0x640..0xe88])?; + out[0x2614..0x2694].copy_from_slice(&completion); + put_u64(out, 0x26ac, qos_high); + + // Native includes index zero plus twelve operating points. Each voltage + // occupies all sixteen lanes of its 0x40-byte state row. + const FREQUENCY: [u32; 13] = [0, 338, 486, 636, 796, 888, 988, 1084, 1182, 1278, 1374, 1470, 1578]; + const CORE_VOLTAGE: [u32; 13] = [125, 615, 630, 660, 705, 735, 755, 800, 830, 865, 905, 945, 1010]; + const MEMORY_VOLTAGE: [u32; 13] = [770, 770, 770, 770, 770, 770, 770, 800, 830, 865, 905, 945, 1010]; + const RELATIVE_A: [u32; 13] = [0, 17, 23, 30, 42, 50, 58, 71, 82, 85, 89, 93, 100]; + const RELATIVE_B: [u32; 13] = [0, 0, 11, 24, 36, 44, 52, 60, 68, 75, 83, 91, 100]; + for index in 0..13 { + put_u32(out, 0xfc8 + index * 4, FREQUENCY[index]); + put_u32(out, 0x1848 + index * 4, 0x3f828f5c); + put_u32(out, 0x18c8 + index * 4, RELATIVE_A[index]); + put_u32(out, 0x1908 + index * 4, RELATIVE_B[index]); + for lane in 0..16 { + put_u32(out, 0x1008 + index * 0x40 + lane * 4, CORE_VOLTAGE[index]); + put_u32(out, 0x1408 + index * 0x40 + lane * 4, MEMORY_VOLTAGE[index]); + } + } + for index in 0..16 { + put_u32(out, 0x19c8 + index * 4, index as u32); + } + Ok(()) +} + +/// Fail-closed validation errors for the grounded layout. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum InitdataError { + BufferSize, + AddressOutOfRange, + NullRequiredAddress, + RoleSpecificAddress, + BundleAllocationTooSmall, + ChannelTableShape, + RegisterWindow, + PairAliasedObject, +} + +fn address_representable(address: u64) -> bool { + let high = address & !INITDATA_ADDR_MASK; + high == 0 || high == INITDATA_ADDR_PREFIX +} + +fn required_address(address: u64) -> Result<(), InitdataError> { + if address == 0 { + Err(InitdataError::NullRequiredAddress) + } else if !address_representable(address) { + Err(InitdataError::AddressOutOfRange) + } else { + Ok(()) + } +} + +fn put_u16(bytes: &mut [u8], offset: usize, value: u16) { + bytes[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn write_runs(bytes: &mut [u8], runs: &[(usize, &[u8])]) { + for (offset, data) in runs { + bytes[*offset..*offset + data.len()].copy_from_slice(data); + } +} + +fn validate_shared(shared: &SharedAddresses) -> Result<(), InitdataError> { + required_address(shared.region_a)?; + required_address(shared.region_c)?; + required_address(shared.hw_data_bundle)?; + if shared.hw_data_bundle_alloc < HW_DATA_BUNDLE_MIN_ALLOC { + return Err(InitdataError::BundleAllocationTooSmall); + } + Ok(()) +} + +fn validate_instance(role: InstanceRole, inst: &InstanceAddresses) -> Result<(), InitdataError> { + required_address(inst.root)?; + required_address(inst.main_config)?; + required_address(inst.status_a)?; + match role { + InstanceRole::Primary => { + // The primary root has no +0xb8/+0xc0 fields at all; carrying + // secondary extras there is a role-confusion bug at the caller. + if inst.secondary_extras != [0, 0] { + return Err(InitdataError::RoleSpecificAddress); + } + required_address(inst.status_b)?; + } + InstanceRole::Secondary => { + // Every hardware-booting host hands the secondary a zero status-B; + // a nonzero value there is ungrounded, so fail closed on it. + if inst.status_b != 0 { + return Err(InitdataError::RoleSpecificAddress); + } + for extra in inst.secondary_extras { + required_address(extra)?; + } + } + } + Ok(()) +} + +/// Encode one 0x20-byte UAT level descriptor. Every field is derivable from +/// the level's index shift and entry count; the index mask +/// `(entries - 1) << shift` reproduces all three captured masks exactly. +fn encode_uat_level_descriptor(bytes: &mut [u8], offset: usize, shift: u8, entries: u16) { + bytes[offset] = 8; + bytes[offset + 1] = UAT_PAGE_BITS; + bytes[offset + 2] = UAT_PAGE_BITS; + bytes[offset + 3] = shift; + put_u16(bytes, offset + 0x04, entries); + put_u16(bytes, offset + 0x06, UAT_PAGE_SIZE); + put_u64(bytes, offset + 0x08, 1); + put_u64(bytes, offset + 0x10, UAT_LEVEL_PHYS_MASK); + put_u64(bytes, offset + 0x18, (entries as u64 - 1) << shift); +} + +/// Encode the common root layout with an explicitly selected firmware build. +/// The existing runtime continues to call the J700-only wrapper above. +pub(crate) fn encode_root_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + out: &mut [u8], +) -> Result<(), InitdataError> { + // Profile-sized: identical to root_size(role) for J700/J813. + if out.len() != profile.root_size(role) { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + + out.fill(0); + put_u64(out, ROOT_VERSION, profile.version_word()); + put_u64(out, ROOT_REGION_A, shared.region_a); + // ROOT_ZERO_10 deliberately left zero. + put_u64(out, ROOT_MAIN_CONFIG, inst.main_config); + put_u64(out, ROOT_REGION_C, shared.region_c); + put_u32(out, ROOT_INSTANCE_KIND, role as u32); + put_u32(out, ROOT_CONSTANT_2C, 1); + put_u16(out, ROOT_PAGE_SIZE, UAT_PAGE_SIZE); + out[ROOT_PAGE_BITS] = UAT_PAGE_BITS; + out[ROOT_LEVEL_COUNT] = UAT_LEVEL_COUNT as u8; + for (index, (shift, entries)) in UAT_LEVELS.iter().enumerate() { + encode_uat_level_descriptor( + out, + ROOT_LEVELS + index * UAT_LEVEL_DESC_SIZE, + *shift, + *entries, + ); + } + put_u64(out, ROOT_STATUS_A, inst.status_a); + put_u64(out, ROOT_STATUS_B, inst.status_b); + if role == InstanceRole::Secondary { + put_u64(out, ROOT_SECONDARY_EXTRA_0, inst.secondary_extras[0]); + put_u64(out, ROOT_SECONDARY_EXTRA_1, inst.secondary_extras[1]); + } + Ok(()) +} + +/// Derive the native channel table for one instance. +/// +/// `state_grid` is this instance's compact channel-state block (0x40-stride, +/// permuted per [`WORK_STATE_GRID_OFFSETS`]); `status_a` anchors the trailing +/// report/partial channels; the ring for entry 12 is `main_config + 0x4c0`; +/// work rings for the primary sit in the bundle at +/// [`WORK_RING_BUNDLE_OFFSETS`] (which requires the +/// [`NATIVE_SHARED_CLUSTER_SIZE`] cluster). The secondary leaves entries +/// 0..11 empty: it is a control-only peer with no work channels. +pub(crate) fn derive_channel_table( + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = [ChannelEntry::EMPTY; CHANNEL_TABLE_ENTRIES]; + let grid_states = |index: usize| { + let base = state_grid + WORK_STATE_GRID_OFFSETS[index] as u64; + [ + base, + base + CHANNEL_STATE_SPACING as u64, + base + 2 * CHANNEL_STATE_SPACING as u64, + ] + }; + if role == InstanceRole::Primary { + for index in 0..WORK_CHANNEL_COUNT { + table[index] = ChannelEntry { + states: grid_states(index), + ring: bundle + WORK_RING_BUNDLE_OFFSETS[index] as u64, + }; + } + } + table[CONTROL_CHANNEL_INDEX] = ChannelEntry { + states: grid_states(CONTROL_CHANNEL_INDEX), + ring: main_config + MAIN_CONTROL_RING as u64, + }; + for (slot, index) in REPORT_CHANNEL_INDICES.iter().enumerate() { + let mut states = [0u64; 3]; + for (word, offset) in states.iter_mut().zip(REPORT_STATE_STATUS_A_OFFSETS[slot]) { + *word = status_a + offset as u64; + } + table[*index] = ChannelEntry { + states, + ring: status_a + REPORT_RING_STATUS_A_OFFSETS[slot] as u64, + }; + } + table[PARTIAL_CHANNEL_INDEX] = ChannelEntry { + states: [status_a + PARTIAL_STATE_STATUS_A_OFFSET as u64, 0, 0], + ring: 0, + }; + table +} + +fn validate_populated_entry(entry: &ChannelEntry) -> Result<(), InitdataError> { + // A populated channel needs all three state-counter addresses — a null + // producer (state 2) on a populated entry makes firmware fault on address + // zero — and a real ring. + for state in entry.states { + if state == 0 { + return Err(InitdataError::ChannelTableShape); + } + if !address_representable(state) { + return Err(InitdataError::AddressOutOfRange); + } + } + required_address(entry.ring) +} + +fn validate_channel_table( + role: InstanceRole, + main_config: u64, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], +) -> Result<(), InitdataError> { + for (index, entry) in channels.iter().enumerate() { + match index { + 0..=11 => match role { + InstanceRole::Primary => validate_populated_entry(entry)?, + // The secondary has no work channels at all. + InstanceRole::Secondary => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + }, + 12 => { + validate_populated_entry(entry)?; + // Device control is embedded in the main object, at exactly + // main + 0x4c0 on both instances. + if entry.ring != main_config + MAIN_CONTROL_RING as u64 { + return Err(InitdataError::ChannelTableShape); + } + } + 13 | 14 => validate_populated_entry(entry)?, + 15 => { + // Partial: a first state address and nothing else. + required_address(entry.states[0])?; + if entry.states[1] != 0 || entry.states[2] != 0 || entry.ring != 0 { + return Err(InitdataError::ChannelTableShape); + } + } + _ => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + } + } + Ok(()) +} + +/// The two complete M5/25G76 pre-handoff main configurations match the common +/// layout except primary +0x32c/+0x344 and secondary +0x300. Keep these +/// measured scalars build-specific; their meanings are not inferred here. +pub(crate) fn encode_main_config_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], + region_views: Option<&PrimaryRegionViews>, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != MAIN_CONFIG_SIZE { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + validate_channel_table(role, inst.main_config, channels)?; + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + required_address(views.sentinel_high)?; + for high in views.alias_high { + required_address(high)?; + } + } + (InstanceRole::Secondary, None) => {} + _ => return Err(InitdataError::RoleSpecificAddress), + } + // Every view must fit inside the one bundle allocation. (The last two end + // exactly at 0xf000, inside the 0x10000 minimum.) The view table is the + // profile's: J700/J813 keep BUNDLE_VIEW_OFFSETS/EXTENTS unchanged. + let view_offsets = profile.bundle_view_offsets(); + for (offset, extent) in view_offsets.iter().zip(profile.bundle_view_extents()) { + if offset + extent > shared.hw_data_bundle_alloc { + return Err(InitdataError::BundleAllocationTooSmall); + } + } + + out.fill(0); + // Firmware log enable. The emitter at A000 0xfffffc0000021fa0 reads this + // word and returns immediately when it is zero; every buffer the facility + // needs is already allocated and pointed at by the addresses below. The + // records are plain ASCII. Shipping macOS provisions the whole facility + // and then throws the messages away (drainFirmwareLogRing is `bti c; ret`), + // which is why the captured value is zero. It stays zero here. + let bundle = shared.hw_data_bundle; + put_u64(out, MAIN_HW_DATA, bundle); + let repeated = bundle + profile.repeated_region_bundle_offset() as u64; + put_u64(out, MAIN_REPEATED_0, repeated); + put_u64(out, MAIN_REPEATED_1, repeated); + + for (index, entry) in channels.iter().enumerate() { + let base = MAIN_CHANNEL_TABLE + index * CHANNEL_ENTRY_SIZE; + for (word, state) in entry.states.iter().enumerate() { + put_u64(out, base + word * 8, *state); + } + put_u64(out, base + CHANNEL_ENTRY_STATE_COUNT * 8, entry.ring); + } + + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + // The five bare pointers are views into the one bundle + // allocation, at fixed offsets. Unaligned stores at +0x254. + for (index, offset) in view_offsets.iter().enumerate() { + put_u64(out, MAIN_BUNDLE_VIEWS + index * 8, bundle + *offset as u64); + } + // High-only sentinel followed by two low/high region-view pairs. + put_u64(out, MAIN_REGION_VIEWS, views.sentinel_high); + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x10, views.alias_high[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u64(out, MAIN_REGION_VIEWS + 0x20, views.alias_high[1]); + if profile == RootProfile::J700G17P { + put_u32(out, MAIN_PRIMARY_SCALAR_32C, 1); + put_u32(out, MAIN_PRIMARY_SCALAR_344, 0xabcd_abcd); + } + // KEXT_PROVEN for G18G too (0xfffffe0008c31780: main+0x3e0=0xff). + put_u32(out, MAIN_PRIMARY_MASK, MAIN_PRIMARY_MASK_VALUE); + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + (InstanceRole::Secondary, _) => { + // The secondary keeps the two low context-0 values but carries no + // view or region addresses; the values are therefore not + // properties of any region the host allocates. + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u32(out, MAIN_SECONDARY_KIND, match profile { + RootProfile::J700G17P => MAIN_SECONDARY_KIND_VALUE, + RootProfile::J813G17G25G76 => 0xc, + // INFERRED: 26A428 stores accel+0x10a74 here (25G76: + // accel+0xed30, measured 0xc on J813). Value not read on J873. + RootProfile::J873G18G26A428 => G18G_26A428_MAIN_SECONDARY_KIND, + }); + if profile == RootProfile::J873G18G26A428 { + // KEXT_PROVEN (26A428 initFirmwareSharedData, g18g_key.c:4464): + // role 1 main+0x481 = FW VA of the perf/policy allocation. + // 25G76 stored it at +0x471 and also wrote an accelerator + // pair at +0x479/+0x481, which 26A428 no longer writes. + put_u64(out, G18G_26A428_MAIN_SECONDARY_POLICY_PTR, + bundle + G18G_26A428_SHARED_LAYOUT.policy as u64); + } else { + // One additional unaligned pointer *into* primary view 2, landing + // five bytes before that view's first populated run. + let extra = bundle + + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] as u64 + + SECONDARY_EXTRA_VIEW_OFFSET as u64; + put_u64(out, MAIN_SECONDARY_EXTRA_PTR, extra); + } + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + _ => unreachable!(), + } + Ok(()) +} + +/// Encode target-specific pre-handoff platform policy. M5's native policy is +/// kept separate from the experimental T8140 watchdog and idle overrides. +pub(crate) fn encode_region_c_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + if profile == RootProfile::J873G18G26A428 { + // Derived from the J813 bytes below through the proven +4 shift. + return encode_g18g_26a428_region_c(out); + } + out.fill(0); + write_runs(out, REGION_C_RUNS); + if profile == RootProfile::J813G17G25G76 { + // Scalars measured before each ASC received its root on two cold + // J813 / 25G76 boots. These are a scoped reference configuration, + // not generic M5 calibration or a power-management qualification. + // The final pair moved by 0x20; leaving the T8140 pair in place would + // initialize unrelated M5 fields. + for (offset, value) in [ + (0x084, 0x0000_5eb0), + (0x088, 0x0000_04b0), + (0x08c, 0x0000_04b0), + (0x098, 0x0000_04b0), + (0x7c4, 0x0000_04e2), + (0x7e0, 0x3f33_3333), + (0x9a8, 0x0000_0333), + (0xe38, 0), + (0xe3c, 0), + (0xe58, 0x0000_0300), + (0xe5c, 0x0000_0100), + ] { + put_u32(out, offset, value); + } + return Ok(()); + } + // regionC+0x9cc is the KSM progress-watchdog threshold for data master 2 + // (compute); the firmware copies it to 0x1778d0 and its handler starts + // `cbz w20 -> ret`, so 0 disables the watchdog. We ship 0x32 (50) here and + // leave the other data master's slot at 0 -- i.e. we arm the watchdog on + // exactly the queue we submit to. When work cannot progress for any other + // reason this turns a stall into a self-sustaining recovery loop. + for (offset, value) in [ + (0x7c, 1), + (0x80, 1), + (0x9c, 0x3000_001d), + (0xa0, 0x3100_0002), + ] { + put_u32(out, offset, value); + } + Ok(()) +} + +/// Encode one 0x80-byte status block in its pre-init state: u32 1 at `+0x04`, +/// everything else zero. `+0x10`/`+0x14` are post-acknowledgement firmware +/// state and must be zero at handoff. +pub(crate) fn encode_status_block(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, STATUS_BLOCK_ONE_OFFSET, 1); + Ok(()) +} + +pub(crate) fn encode_secondary_root_extra_1_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + write_runs(out, SECONDARY_ROOT_EXTRA_1_RUNS); + // G18G: KEXT_PROVEN identical store geometry for this 0x60 object + // (26A428 vs 25G76 R_60 stores, all at the same offsets); the J813 values + // are carried unchanged (INFERRED, listed in G18G_26A428_UNPROVEN). + if matches!(profile, RootProfile::J813G17G25G76 | RootProfile::J873G18G26A428) { + put_u32(out, 0x10, 2); + } + Ok(()) +} + +/// Recorded T8140 region-C platform tuning constants (values known, meanings +/// not; generated from the hardware-validated `g17p_initdata.py` +/// `REGION_C_CONSTANTS`). 12 sparse runs, 180 bytes total. The final run +/// initializes the byte preceding the host power-assert counter and the +/// counter itself to their native cold value of zero. +pub(crate) const REGION_C_RUNS: &[(usize, &[u8])] = &[ + (0x0024, &[0xb8, 0x0b, 0x00, 0x00]), + ( + 0x0030, + &[ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x0054, + &[ + 0xff, 0xff, 0x28, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, + ], + ), + (0x0078, &[0x01, 0x00, 0x00, 0x00]), + ( + 0x0084, + &[ + 0x4e, 0x25, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, + ], + ), + (0x0098, &[0xe8, 0x03, 0x00, 0x00]), + ( + 0x07c4, + &[ + 0xd0, 0x07, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0xcd, 0xcc, 0x4c, 0x3f, 0xcd, 0xcc, + 0x4c, 0x3e, 0x66, 0x66, 0x66, 0x3f, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x80, 0x3e, + 0x9a, 0x99, 0x19, 0x3f, 0x66, 0x66, 0x66, 0x3f, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, + ], + ), + ( + 0x0998, + &[ + 0x28, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x09b8, + &[ + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + ], + ), + (0x0e1c, &[0x00, 0x01, 0x00, 0x00]), + (0x0e38, &[0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00]), + (0x0e50, &[0x00, 0x00, 0x00, 0x00]), +]; + +pub(crate) const SECONDARY_ROOT_EXTRA_1_RUNS: &[(usize, &[u8])] = &[ + (0x14, &[0x01]), + (0x2c, &[0x01]), + (0x30, &[0x01]), + (0x40, &[0x01]), + (0x4c, &[0x01]), + (0x50, &[0x6a, 0x18]), +]; + +/// Root `+0x00`: u16 {0x04c0, 0x83a0, 0x4339, 0x0c8b}. FW_PROVEN: b000 +/// 0x2164c-0x21660 and b010 0x294f0-0x29504 compare root+0 with this word and +/// abort with "Host and FW Interface is mismatched" otherwise. KEXT_PROVEN: +/// AGXArmFirmware::initFirmwareData (0xfffffe0008c31f6c) writes it to both +/// roots. +pub(crate) const G18G_26A428_INITDATA_VERSION_WORD: u64 = 0x0c8b_4339_83a0_04c0; + +/// Both roots are 0xc8 bytes. FW_PROVEN: each image copies 0xc8 bytes of its +/// root (b000 0x21664-0x216a8) and requires root+0x2c != 0 (b000 0x216ac). +/// Root field offsets are unchanged from 25G76 (KEXT_PROVEN g18g_key.c:3748). +pub(crate) const G18G_26A428_ROOT_SIZE: usize = 0xc8; + +/// `AGXFirmware::allocFirmwareData` allocation-size table, 24 u64. +/// KEXT_PROVEN: 25G76 __const 0xfffffe0007139820. Index names are the +/// `KX_*` constants below. +pub(crate) const KEXT_ALLOCATION_SIZES_25G76: [usize; 24] = [ + 0x2708, 0x0c18, 0x1048, 0x0e10, 0x6a30, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe440, 0x06a8, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e78, 0x00e4, +]; + +/// KEXT_PROVEN: 26A428 __const 0xfffffe000714f0d8. Only entries 0 (hw_data), +/// 4 (perf/policy), 12 (status B), 13 (status config) and 22 (region C) +/// differ from 25G76. +pub(crate) const KEXT_ALLOCATION_SIZES_26A428: [usize; 24] = [ + 0x2848, 0x0c18, 0x1048, 0x0e10, 0x7800, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe730, 0x06d0, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e80, 0x00e4, +]; + +pub(crate) const KX_HW_DATA: usize = 0; + +pub(crate) const KX_BLOCK_C18: usize = 1; + +pub(crate) const KX_BLOCK_1048: usize = 2; + +pub(crate) const KX_BLOCK_E10: usize = 3; + +pub(crate) const KX_POLICY: usize = 4; + +pub(crate) const KX_BLOCK_68: usize = 5; + +pub(crate) const KX_BLOCK_800: usize = 6; + +pub(crate) const KX_BLOCK_88: usize = 7; + +pub(crate) const KX_DM_RING: usize = 8; + +pub(crate) const KX_MAIN_CONFIG: usize = 9; + +pub(crate) const KX_CONTROL_RING: usize = 10; + +pub(crate) const KX_REGION_C: usize = 22; + +/// `getSizeOfFWBRNTable` returns 0 in both kexts (KEXT_PROVEN +/// 0xfffffe00089ee74c / 0xfffffe0008c507e4). +pub(crate) const KEXT_BRN_TABLE_SIZE: usize = 0; + +/// Both shared allocations are rounded to one 16 KiB page (allocateSharedData). +pub(crate) const KEXT_ALLOCATION_PAGE: usize = 0x4000; + +pub(crate) const SHARED_ALLOCATION_ENTRIES: usize = 25; + +const fn round_up_usize(value: usize, alignment: usize) -> usize { + (value + alignment - 1) & !(alignment - 1) +} + +/// The allocateSharedData packing rule (KEXT_PROVEN 0xfffffe0008c038d8): +/// each next entry starts at `(previous offset + previous size + 0x3f) & !0x3f`. +pub(crate) const fn next_packed_allocation(offset: usize, size: usize) -> usize { + round_up_usize(offset + size, 0x40) +} + +/// Offsets of `sizes` packed by [`next_packed_allocation`], the first at 0. +pub(crate) const fn packed_allocation_offsets(sizes: &[usize; N]) -> [usize; N] { + let mut out = [0usize; N]; + let mut index = 1; + while index < N { + out[index] = next_packed_allocation(out[index - 1], sizes[index - 1]); + index += 1; + } + out +} + +/// The 25 entries of shared allocation #1 in allocation order: hw_data, the +/// 0xc18/0x1048/0xe10 blocks, perf/policy, the 0x68/0x800 blocks, the BRN +/// table, the 0x88 block, twelve data-master rings, then main config and +/// device-control ring for role 0 and for role 1 (KEXT_PROVEN descriptor +/// build, 25G76 g17g_key.c:1224-1310 / 26A428 g18g_key.c:1236-1320). +pub(crate) const fn shared_allocation_sizes(table: &[usize; 24]) -> [usize; SHARED_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; SHARED_ALLOCATION_ENTRIES]; + sizes[0] = table[KX_HW_DATA]; + sizes[1] = table[KX_BLOCK_C18]; + sizes[2] = table[KX_BLOCK_1048]; + sizes[3] = table[KX_BLOCK_E10]; + sizes[4] = table[KX_POLICY]; + sizes[5] = table[KX_BLOCK_68]; + sizes[6] = table[KX_BLOCK_800]; + sizes[7] = KEXT_BRN_TABLE_SIZE; + sizes[8] = table[KX_BLOCK_88]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + sizes[9 + ring] = table[KX_DM_RING]; + ring += 1; + } + sizes[21] = table[KX_MAIN_CONFIG]; + sizes[22] = table[KX_CONTROL_RING]; + sizes[23] = table[KX_MAIN_CONFIG]; + sizes[24] = table[KX_CONTROL_RING]; + sizes +} + +/// Offsets (from the start of the shared allocation #1 backing) of every +/// object in it. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedClusterLayout { + pub(crate) hw_data: usize, + pub(crate) block_c18: usize, + pub(crate) block_1048: usize, + pub(crate) block_e10: usize, + pub(crate) policy: usize, + pub(crate) block_68: usize, + pub(crate) block_800: usize, + pub(crate) brn_table: usize, + pub(crate) block_88: usize, + /// Data-master rings in allocation order (see [`WORK_CHANNEL_DM_INDEX`]). + pub(crate) dm_rings: [usize; WORK_CHANNEL_COUNT], + pub(crate) main_config: [usize; 2], + pub(crate) control_ring: [usize; 2], + pub(crate) end: usize, + pub(crate) total: usize, +} + +pub(crate) const fn shared_cluster_layout(table: &[usize; 24]) -> SharedClusterLayout { + let sizes = shared_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_rings = [0usize; WORK_CHANNEL_COUNT]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + dm_rings[ring] = offsets[9 + ring]; + ring += 1; + } + let end = offsets[24] + sizes[24]; + SharedClusterLayout { + hw_data: offsets[0], + block_c18: offsets[1], + block_1048: offsets[2], + block_e10: offsets[3], + policy: offsets[4], + block_68: offsets[5], + block_800: offsets[6], + brn_table: offsets[7], + block_88: offsets[8], + dm_rings, + main_config: [offsets[21], offsets[23]], + control_ring: [offsets[22], offsets[24]], + end, + total: round_up_usize(end, KEXT_ALLOCATION_PAGE), + } +} + +/// The 26A428 layouts: hw_data 0x2848 at 0, blocks at 0x2880/0x34c0/0x4540, +/// perf/policy 0x7800 at 0x5380, 0x68/0x800 at 0xcb80/0xcc00, BRN and 0x88 at +/// 0xd400, rings from 0xd4c0, main r0 0x1f4c0, control r0 0x1f980, main r1 +/// 0x23980, control r1 0x23e40; status B 0x340, status config 0xea80, status +/// A 0xf180/0xc3440, tail 0x177700/0x177740/0x1777c0. Totals 0x28000 and +/// 0x178000, as in 25G76. +pub(crate) const G18G_26A428_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&KEXT_ALLOCATION_SIZES_26A428); + +/// Main `+0x254` views (KEXT_PROVEN 26A428 initFirmwareSharedData: the FW VAs +/// of the 0xc18, 0x1048, 0xe10, 0x68 and 0x800 blocks, this+0x310..0x338). +pub(crate) const G18G_26A428_BUNDLE_VIEW_OFFSETS: [usize; 5] = [ + G18G_26A428_SHARED_LAYOUT.block_c18, + G18G_26A428_SHARED_LAYOUT.block_1048, + G18G_26A428_SHARED_LAYOUT.block_e10, + G18G_26A428_SHARED_LAYOUT.block_68, + G18G_26A428_SHARED_LAYOUT.block_800, +]; + +pub(crate) const G18G_26A428_BUNDLE_VIEW_EXTENTS: [usize; 5] = [ + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_C18], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_1048], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_E10], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_68], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_800], +]; + +/// Role-1 main `+0x481`: FW VA of the perf/policy allocation (KEXT_PROVEN). +pub(crate) const G18G_26A428_MAIN_SECONDARY_POLICY_PTR: usize = 0x481; + +/// Role-1 main `+0x300`. INFERRED: the J813 value (26A428 source accel+0x10a74). +pub(crate) const G18G_26A428_MAIN_SECONDARY_KIND: u32 = 0xc; + +/// J873 ADT gpu-num-perf-states. ADT_PROVEN. +pub(crate) const J873_GPU_NUM_PERF_STATES: u32 = 13; + +/// Region-C ADT tunables. KEXT_PROVEN: +0x7c8 = accel+0x106b0 +/// (gpu-idleoff-standby-timer; J813 +0x7c4 carried its ADT 1250) and +0x7e4 = +/// accel+0x106cc (gpu-idleoff-min_conf); ADT_PROVEN values 1500 and 0.8f. +/// +0x9c = this+0xf14 * 100 (25G76 +0x98 = this+0xf40 * 100 = 1200 on J813): +/// INFERRED to be the perf-state count again. +pub(crate) const G18G_26A428_REGION_C_ADT_WORDS: [(usize, u32); 3] = [ + (0x7c8, 1500), + (0x7e4, 0x3f4c_cccd), + (0x09c, J873_GPU_NUM_PERF_STATES * 100), +]; + +/// Stable region-C inputs from the same eight initialization snapshots. +/// +0x38 is the new 26A428 feature bit, no longer a guessed zero. +pub(crate) const G18G_26A428_REGION_C_CAPTURED_WORDS: [(usize, u32); 7] = [ + (0x0038, 1), (0x0088, 0x00008f16), (0x008c, 0x00000514), + (0x0090, 0x00000514), (0x09b0, 0x00000200), + (0x09dc, 0x00000032), (0x0e0c, 0x000000fa), +]; + +/// Encode the G18G region C: the J813 bytes, the 25G76 fields from +0x38 on +/// moved up by 4 (FW_PROVEN), the J873 ADT tunables and stable native J873 +/// inputs. The new +0x38 feature bit (accel+0x6b9 bit 2) is captured as one. +pub(crate) fn encode_g18g_26a428_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + encode_region_c_for_profile(RootProfile::J813G17G25G76, out)?; + // The J813 object never reaches its 0xe78 end; shifting in place keeps the + // tail zero and needs no second 4 KiB buffer on the kernel stack. + let shifted = KEXT_ALLOCATION_SIZES_25G76[KX_REGION_C]; + if out[shifted..].iter().any(|byte| *byte != 0) { + return Err(InitdataError::BufferSize); + } + out.copy_within(0x38..shifted, 0x3c); + out[0x38..0x3c].fill(0); + for (offset, value) in G18G_26A428_REGION_C_ADT_WORDS { + put_u32(out, offset, value); + } + for (offset, value) in G18G_26A428_REGION_C_CAPTURED_WORDS { + put_u32(out, offset, value); + } + // Idle power-off policy word (Region C +0x30). The firmware loads + // it on every cold wake, which undoes the IdlePowerOff(0) sent once at boot. + // 0 keeps the scheduler idle event from powering the cores down between + // jobs. Measured on the M6: 8.6 percent less time per op. + put_u32(out, 0x30, 0); + Ok(()) +} + +/// FW_PROVEN (jump table 0xfffffc0000006c60, 56 entries from opcode 4): the +/// opening/region-C reload handler (b000 0x3ed0) is reached by 0x15, 0x2a +/// and 0x30. +pub(crate) const G18G_26A428_CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x15; + +pub(crate) const G18G_26A428_CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; diff --git a/drivers/gpu/drm/asahi/m5/live_boot.rs b/drivers/gpu/drm/asahi/m5/live_boot.rs new file mode 100644 index 0000000000000..89bf0b08c7124 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/live_boot.rs @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5-specific initialization and retained application submission session. +//! DRM admission remains in the device adapter; no G17P manager is selected. + +use super::provider::{validate_provider, ExpectedProvider, G17PAscwrapV6Role, + G17PMailboxProvider, G17PNoRtkitBuffer}; +use crate::{driver::AsahiDrmDevice, m5::resources::M5FirmwareMemory, + m5::fw_msg, mmu, regs}; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use crate::m5::completion::{graphics_pointer_head, MAX_GRAPHICS_GENERATION}; + +/// Application jobs are polled for retirement. A 1 ms step put up to a +/// millisecond of idle GPU after every job, several per video frame. +/// (The first render job keeps 1 ms steps; see submit_application.) +const APPLICATION_POLL_US: i64 = 50; +const APPLICATION_POLLS_PER_MS: u32 = 1000 / APPLICATION_POLL_US as u32; +use kernel::{device::Core, macros::vtable, platform, prelude::*, + soc::apple::rtkit, sync::{aref::ARef, Arc}, time::{delay::fsleep, Delta}}; + +struct M5RoleState { + dev: ARef, + role: usize, + acked: AtomicBool, + crashed: AtomicBool, + events: AtomicU32, +} + +struct M5RtkitOps; + +#[vtable] +impl rtkit::Operations for M5RtkitOps { + type Data = Arc; + type Buffer = G17PNoRtkitBuffer; + + fn recv_message_early( + data: ::Borrowed<'_>, + endpoint: u8, + message: u64, + ) -> bool { + // Both native M5 roles reply with exactly 0x0009000000000000. + if endpoint == 0x20 && message == 0x0009_0000_0000_0000 { + data.acked.store(true, Ordering::Release); + true + } else { + false + } + } + + fn recv_message( + data: ::Borrowed<'_>, + endpoint: u8, + message: u64, + ) { + let ordinal = data.events.fetch_add(1, Ordering::Relaxed); + if ordinal < 8 { + dev_dbg!(data.dev.as_ref(), "M5 message role={} ep={:#x} payload={:#x}\n", + data.role, endpoint, message); + } + } + + fn crashed( + data: ::Borrowed<'_>, + _crashlog: Option<&[u8]>, + ) { + data.crashed.store(true, Ordering::Release); + dev_err!(data.dev.as_ref(), "M5 firmware crashed role={}\n", data.role); + } +} + +struct M5Role { + provider: G17PMailboxProvider, + rtkit: rtkit::RtKit, + state: Arc, + started: bool, +} + +struct M5Session { + roles: [M5Role; 2], + memory: Option>, + uat: Arc, + /// The GPU mappings of both firmware crash log buffers. + _crash_views: [mmu::KernelMapping; 2], + registers: regs::G17Resources, + dev: ARef, +} + +impl M5Session { + fn stop_cpus(&mut self) -> Result { + let mut failure = None; + // Reverse the measured M5 GFX1/GFX start order. + for index in [0, 1] { + if self.roles[index].started { + match self.roles[index].provider.stop_cpu() { + Ok(()) => self.roles[index].started = false, + Err(error) => failure = Some(error), + } + } + } + failure.map_or(Ok(()), Err) + } + + fn initialize(&mut self) -> Result { + self.registers.m5_match_pmgr_boot_control()?; + for index in [1, 0] { + let role = &mut self.roles[index]; + role.provider.start_cpu()?; + role.started = true; + Pin::new(&mut role.rtkit).boot()?; + dev_dbg!(self.dev.as_ref(), "M5_INIT role={} rtkit-ready\n", index); + } + // These helpers operate only on the proven shared dual-root layout; + // the Neo-specific contexts, metrics and manager remain unselected. + let roots = self.uat.adopt_firmware_root_entries()?; + let timestamp: u64; + // SAFETY: Read-only architectural counter, available on this target. + unsafe { core::arch::asm!("mrs {value}, cntvct_el0", value = out(reg) timestamp, + options(nomem, nostack, preserves_flags)); } + self.memory = Some(M5FirmwareMemory::new_with_uat( + &self.dev, self.uat.clone(), timestamp)?); + self.uat.confirm_firmware_root_entries(roots)?; + self.uat.mirror_secondary_root()?; + dev_dbg!(self.dev.as_ref(), "M5_INIT retained memory graph mapped\n"); + self.registers.enable_g17_gating()?; + + // M5's own trace publishes and ACKs GFX1 first (133/137), then GFX + // (160/164). The Neo's opposite ordering is not used here. + for index in [1, 0] { + let address = self.memory.as_ref().ok_or(EIO)?.root(index == 1); + let message = fw_msg::encode_initdata_doorbell(address).map_err(|_| EINVAL)?; + let role = &mut self.roles[index]; + Pin::new(&mut role.rtkit).start_endpoint(0x20)?; + Pin::new(&mut role.rtkit).start_endpoint(0x21)?; + Pin::new(&mut role.rtkit).send_message(0x20, message)?; + dev_dbg!(self.dev.as_ref(), "M5_INIT role={} published root={:#x}\n", index, address); + for _ in 0..200 { + if self.memory.as_mut().ok_or(EIO)?.drain_reports(&self.dev)? { + return Err(EIO); + } + if role.state.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if role.state.acked.load(Ordering::Acquire) { + break; + } + fsleep(Delta::from_millis(10)); + } + if !role.state.acked.load(Ordering::Acquire) { + return Err(ETIMEDOUT); + } + dev_dbg!(self.dev.as_ref(), "M5_INIT role={} initdata-ack\n", index); + } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + self.retire_openings() + } + + fn install_graphics_queues(&mut self) -> Result { + let memory = self.memory.as_mut().ok_or(EIO)?; + dev_dbg!(self.dev.as_ref(), "M5_GRAPHICS install before counters={:x?} registration={:x?}\n", + memory.graphics_status()?, memory.graphics_installation_status()?); + memory.publish_graphics_install()?; + Pin::new(&mut self.roles[0].rtkit).send_message(0x21, 0x0083_0000_0000_000a)?; + for poll in 0..1000 { + if memory.drain_reports(&self.dev)? { return Err(EIO); } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + let (outer, inner, _) = memory.graphics_status()?; + if outer == [2; 3] && inner == [[1; 3]; 2] + && memory.graphics_queues_installed()? { + dev_dbg!(self.dev.as_ref(), "M5_GRAPHICS installed TA/fragment queues; registration={:x?}; no draws published\n", + memory.graphics_installation_status()?); + return Ok(()); + } + if [0, 100].contains(&poll) { + dev_dbg!(self.dev.as_ref(), "M5_GRAPHICS install poll={} counters={:x?} registration={:x?}\n", + poll, memory.graphics_status()?, memory.graphics_installation_status()?); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.dev.as_ref(), "M5_GRAPHICS install timeout counters={:x?} registration={:x?}\n", + memory.graphics_status()?, memory.graphics_installation_status()?); + Err(ETIMEDOUT) + } + + fn retire_idle_policy(&mut self) -> Result { + let memory = self.memory.as_mut().ok_or(EIO)?; + // Retire the idle-policy request. + memory.publish_idle_wake()?; + Pin::new(&mut self.roles[0].rtkit).send_message(0x21, 0x0084_0000_0000_0011)?; + for _ in 0..1000 { + if memory.drain_reports(&self.dev)? { return Err(EIO); } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + dev_err!(self.dev.as_ref(), "M5: firmware crashed retiring the idle policy\n"); + return Err(EIO); + } + if memory.control_counters()? == [[2; 3], [1; 3]] { + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.dev.as_ref(), "M5: idle policy not retired, counters={:?}\n", + memory.control_counters()?); + Err(ETIMEDOUT) + } + + fn retire_openings(&mut self) -> Result { + let mut counters = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + dev_dbg!(self.dev.as_ref(), "M5_OPEN before power-transition-end {:?}\n", counters); + // M5's own cold trace sends one 0x89 to GFX, then GFX1 (events + // 167/171), immediately after the two initdata ACKs. No other control + // record or work item is published in this bounded experiment. + for index in [0, 1] { + Pin::new(&mut self.roles[index].rtkit) + .send_message(0x21, fw_msg::encode_power_transition_end())?; + } + for _ in 0..1000 { + if self.memory.as_mut().ok_or(EIO)?.drain_reports(&self.dev)? { + return Err(EIO); + } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + counters = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + if counters == [[1; 3]; 2] { + dev_dbg!(self.dev.as_ref(), "M5_OPEN both native control openings retired {:?}\n", counters); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.dev.as_ref(), "M5_OPEN control openings did not retire {:?}\n", counters); + Err(ETIMEDOUT) + } + +} + +/// Sizes of the GFX and GFX1 `__DATA` segments the 25G76 firmware runs with. +const DATA_SIZES: [u64; 2] = [0x134000, 0x130000]; +/// The firmware's crash log buffer, at this offset of each `__DATA` segment. +const CRASH_LOG_OFFSET: u64 = 0xa0000; +const CRASH_LOG_SIZE: usize = 0x8000; + +/// Creates the UAT both firmware instances share, as the A18 Pro's: each instance publishes +/// its own kernel root, the secondary one 0x40000 into `pagetables`. Each instance asks for +/// its crash log at the GPU address of `__DATA` + 0xa0000 truncated to 40 bits, which is +/// mapped here, before either instance starts. +fn new_uat(dev: &AsahiDrmDevice) -> Result<(Arc, [mmu::KernelMapping; 2])> { + let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let mut crash_logs = [0u64; 2]; + for (index, name) in [c"gfx-data", c"gfx1-data"].into_iter().enumerate() { + let region = node.reserved_mem_region_to_resource_byname(name)?; + if region.size() != DATA_SIZES[index] || region.start() & (mmu::UAT_PGSZ as u64 - 1) != 0 + { + return Err(EINVAL); + } + crash_logs[index] = region.start() + CRASH_LOG_OFFSET; + } + let uat = Arc::new( + mmu::Uat::new( + dev.into(), + mmu::UatGeometry { ias: 42, oas: 42 }, + false, + mmu::UatFirmware::PublishedRoots { secondary_root_offset: Some(0x40000) }, + mmu::UatQuirks::DEFAULT, + )?, + GFP_KERNEL, + )?; + let map = |pa: u64| { + uat.kernel_lower_vm().map_io( + dev.as_ref().as_ref(), + pa & ((1u64 << 40) - 1), + pa.try_into()?, + CRASH_LOG_SIZE, + mmu::PROT_FW_SHARED_RW, + ) + }; + let views = [map(crash_logs[0])?, map(crash_logs[1])?]; + Ok((uat, views)) +} + +pub(crate) struct M5Runtime { + session: Option>, + application: bool, + faulted: bool, + application_generation: u32, + application_compute_generation: u32, +} + +impl M5Runtime { + fn prepare( + pdev: &platform::Device>, + dev: ARef, + registers: regs::G17Resources, + ) -> Result { + let gfx = G17PMailboxProvider::get(pdev.as_ref(), 0)?; + let gfx1 = G17PMailboxProvider::get(pdev.as_ref(), 1)?; + for (provider, expected) in [ + (&gfx, ExpectedProvider { role: G17PAscwrapV6Role::Gfx, + wrapper: (0x582600000, 0x88000), iop_vbar: (0x582050000, 8) }), + (&gfx1, ExpectedProvider { role: G17PAscwrapV6Role::Gfx1, + wrapper: (0x582e00000, 0x88000), iop_vbar: (0x582850000, 8) }), + ] { + validate_provider(provider, expected)?; + provider.require_safe_cpu_lifecycle()?; + } + let (uat, crash_views) = new_uat(&dev)?; + let make_role = |index, provider| -> Result { + let state = Arc::new(M5RoleState { dev: dev.clone(), role: index, + acked: AtomicBool::new(false), crashed: AtomicBool::new(false), + events: AtomicU32::new(0) }, GFP_KERNEL)?; + let rtkit = rtkit::RtKit::::new_with_firmware_crashlog( + dev.as_ref().as_ref(), None, index, state.clone())?; + Ok(M5Role { provider, rtkit, state, started: false }) + }; + let session = KBox::new(M5Session { + roles: [make_role(0, gfx)?, make_role(1, gfx1)?], + memory: None, uat, _crash_views: crash_views, registers, dev, + }, GFP_KERNEL)?; + // From this point every path returns the retained platform owner, + // even when firmware rejects the graph. Failed probe unwinding must + // never release UAT or transports beneath a running firmware CPU. + // Keep callback code resident even if a failed stop retains owners. + // This explicit RAM-boot experiment is released by reboot only. + unsafe { kernel::bindings::__module_get((*kernel::module::this_module::()).as_ptr()); } + Ok(Self { session: Some(session), application: false, faulted: false, + application_generation: 0, application_compute_generation: 0 }) + } + + /// Boot without a native program or job graph and leave both processors + /// running for explicit application submissions. This does not register + /// a render node or claim that Mesa command translation is complete. + pub(crate) fn start_application( + pdev: &platform::Device>, dev: ARef, registers: regs::G17Resources, + ) -> Result { + // Periodic power/submission traces need continuous draining while + // the session idles. Recovery bit 9 emits only on a fault, which the + // synchronous submit loop drains before retaining a failed graph. + if crate::module_parameters::m5_fw_trace.value() & !0x200 != 0 { + return Err(EINVAL); + } + let mut owner = Self::prepare(pdev, dev, registers)?; + owner.application = true; + let result = (|| { + let session = owner.session.as_mut().ok_or(EIO)?; + session.initialize()?; + session.retire_idle_policy()?; + Ok(()) + })(); + if let Err(error) = result { + owner.faulted = true; + return Err(error); + } + dev_info!(pdev.as_ref(), "M5: GPU firmware 25G76 running\n"); + Ok(owner) + } + + pub(crate) fn new_user_vm(&self, id: u64, kernel_range: core::ops::Range) -> Result { + if !self.application || self.is_crashed() { return Err(ENODEV); } + crate::m5::vm_layout::compact_range(&kernel_range).ok_or(EINVAL)?; + self.session.as_ref().ok_or(EIO)?.uat.new_vm(id, kernel_range)?.with_status() + } + + pub(crate) fn is_crashed(&self) -> bool { + self.faulted || self.session.as_ref().is_none_or(|session| + session.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire))) + } + + /// The adapter serializes this call and validates the translated command + /// plus every user mapping before entry. Return success only after paired + /// queue retirement, scheduler unlink, stamps and firmware notifications. + /// A failed published job retains and quarantines its VM until reboot. + #[allow(clippy::too_many_arguments)] + pub(crate) fn submit_application( + &mut self, vm: &mmu::Vm, parameters: crate::m5::render_desc::G17pRenderParameters, + controls: crate::m5::render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], + ) -> Result { + if !self.application || self.is_crashed() { return Err(ENODEV); } + // Reuse retired ring storage while keeping completion stamps unique. + if self.application_generation >= MAX_GRAPHICS_GENERATION { return Err(EBUSY); } + let generation = self.application_generation + 1; + let session = self.session.as_deref_mut().ok_or(EIO)?; + if generation == 1 { + session.memory.as_mut().ok_or(EIO)?.initialize_application_graphics( + &session.dev, vm, parameters, controls, clusters, usc_required)?; + } + // From installation onward, even a failed mailbox send can have made + // memory visible to firmware. Never unwind or recycle the job graph. + self.application_generation = generation; + let result = (|| { + if generation == 1 { session.install_graphics_queues()?; } + let memory = session.memory.as_mut().ok_or(EIO)?; + if generation == 1 { + memory.publish_graphics_job()?; + } else { + memory.publish_next_application_job(&session.dev, vm, parameters, controls, + clusters, usc_required, generation)?; + } + Pin::new(&mut session.roles[0].rtkit).send_message(0x21, 0x0083_0000_0000_000a)?; + for _ in 0..10000 * APPLICATION_POLLS_PER_MS { + if memory.drain_reports(&session.dev)? + || session.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + let (outer, inner, _) = memory.graphics_status()?; + let completion = memory.graphics_completion()?; + if outer == [memory.application_outer_head(); 3] + && inner == [[graphics_pointer_head(generation); 3]; 2] + && M5FirmwareMemory::graphics_pair_complete(completion, generation) { + dev_dbg!(session.dev.as_ref(), + "M5_APPLICATION generation={} completed; both queues retired\n", + generation); + return Ok(completion); + } + // The first job also installs the queues; polling it this + // often met a firmware recovery on its fragment queue. + fsleep(Delta::from_micros(if generation == 1 { 1000 } else { APPLICATION_POLL_US })); + } + Err(ETIMEDOUT) + })(); + if let Err(error) = result { + if let Some(memory) = session.memory.as_mut() { + for _ in 0..4 { let _ = memory.drain_reports(&session.dev); } + dev_err!(session.dev.as_ref(), + "M5_APPLICATION failed graph counters={:x?} completion={:x?}\n", + memory.graphics_status(), memory.graphics_completion()); + } + self.faulted = true; + if let Some(status) = vm.status() { status.record(error); } + if let Some(memory) = &session.memory { memory.quarantine_graphics(); } + // A successful stop request alone does not prove a halt. Keep + // the full session even after this best-effort stop succeeds. + let stopped = session.stop_cpus(); + dev_err!(session.dev.as_ref(), + "M5_APPLICATION submission failed {:?}; stop={:?}; VM and firmware owners retained\n", + error, stopped); + } + result + } + + /// Execute a compute command in the already retained application VM. + /// Installation, dispatch and retirement are separate firmware transitions. + /// The first submission installs queue 12 and dispatches generation 1; + /// every later one reuses that owner and dispatches the next generation, + /// as submit_application() does for the graphics pair. + pub(crate) fn submit_application_compute(&mut self, vm: &mmu::Vm, + command: &crate::m5::command::TranslatedComputeCommand) -> Result { + if !self.application || self.is_crashed() || self.application_generation == 0 { + return Err(ENOTSUPP); + } + if self.application_compute_generation >= MAX_GRAPHICS_GENERATION { + return Err(EBUSY); + } + let generation = self.application_compute_generation + 1; + let session = self.session.as_deref_mut().ok_or(EIO)?; + if generation == 1 { + session.memory.as_mut().ok_or(EIO)?.initialize_application_compute( + &session.dev, vm, command, self.application_generation)?; + } else { + session.memory.as_mut().ok_or(EIO)?.prepare_next_application_compute( + &session.dev, vm, command, generation)?; + } + // The owner is retained from installation onward. Even a failed mailbox + // send can follow firmware seeing the new producer, so never unwind. + self.application_compute_generation = generation; + let result = (|| { + let memory = session.memory.as_mut().ok_or(EIO)?; + // Installation is published once; the dispatch record every time. + let stages: &[bool] = if generation == 1 { &[false, true] } else { &[true] }; + for &dispatch in stages { + memory.publish_application_compute(dispatch, generation)?; + Pin::new(&mut session.roles[0].rtkit) + .send_message(0x21, 0x0083_0000_0000_000a)?; + let mut retired = false; + for _ in 0..if dispatch { 10000 } else { 1000 } * APPLICATION_POLLS_PER_MS { + if memory.drain_reports(&session.dev)? + || session.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + let (outer, _, _) = memory.graphics_status()?; + if outer == [memory.application_outer_head(); 3] + && if dispatch { memory.application_compute_complete(generation)? } + else { memory.application_compute_installed()? } { + retired = true; + break; + } + fsleep(Delta::from_micros(APPLICATION_POLL_US)); + } + if !retired { return Err(ETIMEDOUT); } + dev_dbg!(session.dev.as_ref(), + "M5_APPLICATION_COMPUTE {} generation={} queue=12 outer={} status={:x?}\n", + if dispatch { "completed" } else { "installed" }, generation, + memory.application_outer_head(), memory.application_compute_status()); + } + Ok(()) + })(); + if let Err(error) = result { + self.faulted = true; + if let Some(status) = vm.status() { status.record(error); } + if let Some(memory) = session.memory.as_mut() { + for _ in 0..4 { let _ = memory.drain_reports(&session.dev); } + memory.quarantine_graphics(); + dev_err!(session.dev.as_ref(), + "M5_APPLICATION_COMPUTE failed {:?}; outer/graphics={:x?} compute={:x?}\n", + error, memory.graphics_status(), memory.application_compute_status()); + } + let stopped = session.stop_cpus(); + dev_err!(session.dev.as_ref(), + "M5_APPLICATION_COMPUTE stop={:?}; VM and owners retained until reboot\n", stopped); + } + result + } +} + +impl Drop for M5Runtime { + fn drop(&mut self) { + if let Some(mut session) = self.session.take() { + if let Some(memory) = &session.memory { memory.quarantine_graphics(); } + let stopped = session.stop_cpus(); + dev_dbg!(session.dev.as_ref(), + "M5 session release requested; stop={:?}; halt unqualified, owners retained until reboot\n", + stopped); + core::mem::forget(session); + } + } +} diff --git a/drivers/gpu/drm/asahi/m5/mapped.rs b/drivers/gpu/drm/asahi/m5/mapped.rs new file mode 100644 index 0000000000000..2ccc37b82531b --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/mapped.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Kernel-owned, write-combined GEM objects mapped at one GPU address. + +use core::ops::Range; +use kernel::prelude::*; + +use crate::{driver::AsahiDrmDevice, gem, mmu}; + +/// A zeroed, write-combined kernel object and its owning GPU mapping. +pub(crate) struct MappedObject { + _mapping: mmu::KernelMapping, + object: gem::ObjectRef, + iova: u64, + /// Size of the object in bytes, as requested. + pub(crate) logical_size: usize, + /// The VM of the owning mapping. + home: u64, +} + +impl MappedObject { + fn new_object(dev: &AsahiDrmDevice, size: usize) -> Result { + let mut object = gem::new_kernel_object_wc(dev, size)?; + let len = object.size(); + let vmap = object.vmap()?; + // SAFETY: the vmap covers the whole page-rounded allocation. + unsafe { core::ptr::write_bytes(vmap.addr.cast::(), 0, len) }; + Ok(object) + } + + fn from_mapping( + object: gem::ObjectRef, vm: &mmu::Vm, mapping: mmu::KernelMapping, size: usize, + ) -> Self { + let iova = mapping.iova(); + Self { _mapping: mapping, object, iova, logical_size: size, home: vm.id() } + } + + /// Maps a new object of `size` bytes at `address`. + pub(crate) fn new_wc_at( + dev: &AsahiDrmDevice, vm: &mmu::Vm, address: u64, size: usize, prot: mmu::Prot, + ) -> Result { + let mut object = Self::new_object(dev, size)?; + let mapping = object.map_at(vm, address, prot, false)?; + Ok(Self::from_mapping(object, vm, mapping, size)) + } + + /// Maps a new object of `size` bytes at a free address in `range`. + pub(crate) fn new_wc_in_range( + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: Range, size: usize, alignment: u64, + prot: mmu::Prot, + ) -> Result { + let mut object = Self::new_object(dev, size)?; + let mapping = object.map_into_range(vm, range, alignment, prot, true)?; + Ok(Self::from_mapping(object, vm, mapping, size)) + } + + /// As [`Self::new_wc_in_range`], followed by `guard_pages` unmapped pages. + pub(crate) fn new_wc_in_range_with_guards( + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: Range, size: usize, alignment: u64, + prot: mmu::Prot, guard_pages: usize, + ) -> Result { + let object = Self::new_object(dev, size)?; + let guard = guard_pages.checked_mul(mmu::UAT_PGSZ).ok_or(EOVERFLOW)?; + let mapping = vm.map_in_range_with_guard_size( + &object.gem, 0..object.size(), alignment, range, prot, guard, + )?; + Ok(Self::from_mapping(object, vm, mapping, size)) + } + + /// Maps this object again at `address` of `vm`. + pub(crate) fn map_alias_at( + &mut self, vm: &mmu::Vm, address: u64, prot: mmu::Prot, + ) -> Result { + self.object.map_at(vm, address, prot, false) + } + + /// Maps this object at its own address in another client VM, followed by `guard_pages` + /// unmapped pages. In the VM of the owning mapping, where the object already is at that + /// address, there is nothing to map. + pub(crate) fn map_alias_at_with_guards( + &mut self, vm: &mmu::Vm, address: u64, prot: mmu::Prot, guard_pages: usize, + ) -> Result> { + if vm.id() == self.home { + return if address == self.iova { Ok(None) } else { Err(EINVAL) }; + } + let guard = guard_pages.checked_mul(mmu::UAT_PGSZ).ok_or(EOVERFLOW)?; + vm.map_at_with_guard_size(address, self.object.size(), &self.object.gem, prot, guard) + .map(Some) + } + + /// GPU address of the object. + pub(crate) fn iova(&self) -> u64 { + self.iova + } + + /// Runs `f` on the CPU view of the object's `size` bytes. + pub(crate) fn with_bytes_mut(&mut self, f: impl FnOnce(&mut [u8]) -> Result) -> Result { + let size = self.logical_size; + let vmap = self.object.vmap()?; + // SAFETY: the vmap covers the whole allocation, which is at least `size` bytes, and + // `&mut self` excludes other CPU views made through this object. + let bytes = unsafe { core::slice::from_raw_parts_mut(vmap.addr.cast::(), size) }; + f(bytes) + } + + fn offset(&self, address: u64, width: usize) -> Result { + let offset: usize = address.checked_sub(self.iova).ok_or(EINVAL)?.try_into()?; + let end = offset.checked_add(width).ok_or(EINVAL)?; + if end > self.logical_size { + return Err(EINVAL); + } + Ok(offset) + } + + /// Reads the little-endian word at GPU address `address` (any alignment). + pub(crate) fn read_u32(&mut self, address: u64) -> Result { + let offset = self.offset(address, 4)?; + self.with_bytes_mut(|bytes| { + Ok(u32::from_le_bytes(bytes[offset..offset + 4].try_into().map_err(|_| EINVAL)?)) + }) + } + + /// Reads the little-endian double word at GPU address `address` (any alignment). + pub(crate) fn read_u64(&mut self, address: u64) -> Result { + let offset = self.offset(address, 8)?; + self.with_bytes_mut(|bytes| { + Ok(u64::from_le_bytes(bytes[offset..offset + 8].try_into().map_err(|_| EINVAL)?)) + }) + } + + /// Writes the naturally aligned little-endian word at GPU address `address` with one store. + pub(crate) fn write_u32(&mut self, address: u64, value: u32) -> Result { + let offset = self.offset(address, 4)?; + if offset % 4 != 0 { + return Err(EINVAL); + } + self.with_bytes_mut(|bytes| { + // SAFETY: checked bounds and natural alignment. + unsafe { + core::ptr::write_volatile(bytes.as_mut_ptr().add(offset).cast::(), value.to_le()) + }; + Ok(()) + }) + } +} diff --git a/drivers/gpu/drm/asahi/m5/mod.rs b/drivers/gpu/drm/asahi/m5/mod.rs new file mode 100644 index 0000000000000..96beaac4391f4 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/mod.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple M5 (T8142, G17G) GPU support for the 25G76 firmware (RTKit-3255.160.4). +//! +//! The M5 has two GPU coprocessors, GFX and GFX1, as the A18 Pro does. The boot loader (aura) +//! loads their firmware, checks both loaded TEXT images against the 25G76 build and says so +//! in `/chosen/aurora,m5-gpu-firmware-profile`. The firmware interface is HAL300 and differs +//! from the 26A428 and 25G83 builds the shared G17 code runs, so this backend builds its own +//! initialization graph ([`initdata`], [`resources`]) and talks to the firmware over its own +//! channels ([`channel`], [`sksm`], [`completion`]). +//! +//! # Plugging into the shared driver +//! +//! - `ProbeConfig::M5` (`apple,agx-t8142`) calls [`probe`], which starts both coprocessors +//! ([`live_boot`]) and returns a [`Gpu`] implementing [`crate::gpu::Gpu`]. +//! - The UAT is the shared [`crate::mmu::Uat`] with published roots, as on the A18 Pro, plus +//! [`crate::mmu::M5ExecutionContext`] for application contexts. User VMs carry a status and +//! a job lifetime, and the backend uses the tracked submission model of `file.rs`. +//! - The UAPI is the shared one. The backend reports `VM_STATUS` (with its mirror) and +//! `SOFT_FAULTS`, and one hardware command per submission. +//! +//! # Submission +//! +//! One render pair (TA and fragment queues) and one compute queue serve every client: the +//! queues are installed by the first render and handed from client VM to client VM, with +//! the driver's pools aliased at the same addresses in each. A submission is published, +//! then polled to retirement before the ioctl returns, so there is one job in flight. +//! Renders and compute are admitted only within the state the firmware programs have been +//! verified with ([`render`], [`compute`]); anything else is refused with `ENOTSUPP` +//! before it reaches the firmware. +//! +//! # Failures +//! +//! There is no recovery yet. A job that fails or does not retire marks its VM, stops both +//! coprocessors and retains every firmware-visible object until reboot; the device is lost +//! for all clients. The module stays loaded for the same reason. + +mod channel; +mod command; +mod completion; +mod compute; +mod drm; +mod fw_msg; +mod initdata; +mod live_boot; +mod mapped; +mod provider; +mod render; +mod render_desc; +mod render_target; +mod resources; +mod sksm; +mod submission; +mod vm_layout; + +use kernel::{device::Core, of, platform, prelude::*, sync::Arc}; + +use crate::{driver::AsahiDrmDevice, regs}; + +pub(crate) use drm::M5DrmGpu as Gpu; + +/// The firmware profile aura publishes in `/chosen/aurora,m5-gpu-firmware-profile` after it +/// checked both loaded TEXT images against the exact 25G76 build. +const FIRMWARE_PROFILE: &[u8] = b"M5-RTKit-3255.160.4.release\0"; + +/// Starts both firmware instances and returns the GPU, ready for DRM registration. +pub(crate) fn probe( + pdev: &platform::Device>, + dev: &AsahiDrmDevice, + res: regs::G17Resources, +) -> Result> { + let chosen = of::chosen().ok_or(ENODEV)?; + let profile: KVec = chosen.get_property(c"aurora,m5-gpu-firmware-profile")?; + if profile.as_slice() != FIRMWARE_PROFILE { + dev_err!(pdev.as_ref(), "M5: GPU firmware profile not published or not 25G76\n"); + return Err(ENODEV); + } + let id = res.get_g17_id()?; + let cores: u32 = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + // M5 GPUs have 10 cores, or 8 with two fused off (MacBook Air). + if id.family != 0xb || id.num_dies != 1 || !matches!(cores, 8 | 10) { + dev_err!(pdev.as_ref(), "M5: unexpected GPU family {:#x}, {} dies, {} cores\n", + id.family, id.num_dies, cores); + return Err(ENODEV); + } + // Before the session: a CLPC that has not probed yet defers the GPU. + let clpc = drm::M5ClpcInput::new(pdev.as_ref())?; + let runtime = live_boot::M5Runtime::start_application(pdev, dev.into(), res)?; + let runtime = Arc::pin_init(kernel::new_mutex!(runtime), GFP_KERNEL)?; + Ok(Arc::new(Gpu::new(runtime, id, clpc), GFP_KERNEL)?) +} diff --git a/drivers/gpu/drm/asahi/m5/provider.rs b/drivers/gpu/drm/asahi/m5/provider.rs new file mode 100644 index 0000000000000..d54a591f85448 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/provider.rs @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The GFX and GFX1 ASCWrap v6 mailbox providers, as seen by the M5 firmware session. + +use core::mem::MaybeUninit; + +use kernel::{ + bindings, + error::{from_err_ptr, to_result}, + io::{IoSysMap, Region}, + prelude::*, + soc::apple::rtkit, +}; + +use crate::mmu; + +const G17P_CRASHLOG_MAX_BYTES: usize = 256 * 1024; + +const G17P_RTKIT_UAT_DVA_MASK: u64 = (1u64 << 40) - 1; + +/// Module-local, read-only view of one existing ASCWrap mailbox provider. +/// +/// The C provider owns the mapping and all lifecycle transitions. This view +/// neither maps an alias nor changes callback ownership; it only borrows the +/// provider's established mailbox-register pointer for two control reads. +pub(super) struct G17PMailboxProvider { + mbox: *mut bindings::apple_mbox, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) enum G17PAscwrapV6Role { + Gfx, + Gfx1, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) struct G17PAscwrapV6Lifecycle { + role: G17PAscwrapV6Role, + wrapper: (u64, u64), + iop_vbar: (u64, u64), + missing: u32, +} + +impl G17PMailboxProvider { + + pub(super) fn get(dev: &kernel::device::Device, index: usize) -> Result { + // SAFETY: `dev` is live and apple_mbox_get() establishes a device link + // to keep the resolved provider alive for this consumer. + let mbox = unsafe { + from_err_ptr(bindings::apple_mbox_get( + dev.as_raw(), + index.try_into()?, + ))? + }; + Ok(Self { mbox }) + } + + pub(super) fn lifecycle(&self) -> Result { + let mut raw = MaybeUninit::::uninit(); + // SAFETY: `self.mbox` is the live provider returned by + // apple_mbox_get() and `raw` points to writable storage. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_lifecycle(self.mbox, raw.as_mut_ptr()) + })?; + // SAFETY: the successful C call initialized every field. + let raw = unsafe { raw.assume_init() }; + let role = match raw.role as u32 { + 1 => G17PAscwrapV6Role::Gfx, + 2 => G17PAscwrapV6Role::Gfx1, + _ => return Err(EINVAL), + }; + Ok(G17PAscwrapV6Lifecycle { + role, + wrapper: (raw.wrapper_start, raw.wrapper_size), + iop_vbar: (raw.iop_vbar_start, raw.iop_vbar_size), + missing: raw.missing, + }) + } + + pub(super) fn require_safe_cpu_lifecycle(&self) -> Result { + // SAFETY: this is a capability check on the live provider pointer. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(self.mbox) + }) + } + + pub(super) fn start_cpu(&self) -> Result { + // SAFETY: lifecycle serialization and MMIO ownership remain in C. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_start_cpu(self.mbox) }) + } + + pub(super) fn stop_cpu(&self) -> Result { + // SAFETY: lifecycle serialization and MMIO ownership remain in C. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }) + } + +} + +unsafe impl Send for G17PMailboxProvider {} + +unsafe impl Sync for G17PMailboxProvider {} + +pub(super) const fn g17p_rtkit_uat_address(dva: u64) -> u64 { + dva & G17P_RTKIT_UAT_DVA_MASK +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(super) enum G17PProcessor { + Gfx = 0, + Gfx1 = 1, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) struct G17PFirmwareCrashBuffer { + dva: u64, + size: usize, +} + +impl G17PProcessor { + + pub(super) const fn firmware_crash_buffer(self) -> G17PFirmwareCrashBuffer { + // Both endpoint-1 buffer requests are stable across the retained + // G17P captures. Message 0x0010810001e1c000 decodes to the primary + // DVA below and size 0x8000; GFX1 advertises the second DVA with the + // same size. + match self { + Self::Gfx => G17PFirmwareCrashBuffer { + dva: 0x100_01e1_c000, + size: 0x8000, + }, + Self::Gfx1 => G17PFirmwareCrashBuffer { + dva: 0x100_01f5_0000, + size: 0x8000, + }, + } + } +} + +pub(super) const fn bounded_crashlog_size(size: usize) -> usize { + if size < G17P_CRASHLOG_MAX_BYTES { + size + } else { + G17P_CRASHLOG_MAX_BYTES + } +} + +const _: () = { + assert!(bounded_crashlog_size(0) == 0); + assert!(bounded_crashlog_size(4096) == 4096); + assert!(bounded_crashlog_size(G17P_CRASHLOG_MAX_BYTES + 1) == G17P_CRASHLOG_MAX_BYTES); + let gfx = G17PProcessor::Gfx.firmware_crash_buffer(); + let gfx1 = G17PProcessor::Gfx1.firmware_crash_buffer(); + assert!(gfx.dva == 0x100_01e1_c000 && gfx1.dva == 0x100_01f5_0000); + assert!(g17p_rtkit_uat_address(gfx.dva) == 0x01e1_c000); + assert!(g17p_rtkit_uat_address(gfx1.dva) == 0x01f5_0000); + assert!(gfx.size == 0x8000 && gfx1.size == 0x8000); + assert!(gfx.dva & (mmu::UAT_PGSZ as u64 - 1) == 0); + assert!(gfx1.dva & (mmu::UAT_PGSZ as u64 - 1) == 0); + assert!(gfx.dva + gfx.size as u64 <= 1u64 << 42); + assert!(gfx1.dva + gfx1.size as u64 <= 1u64 << 42); +}; + +/// Active G17P boot does not ask Rust to back system-endpoint buffers. The +/// generic RTKit core uses its coherent-DMA fallback when these callbacks are +/// absent. +pub(super) struct G17PNoRtkitBuffer; + +impl rtkit::Buffer for G17PNoRtkitBuffer { + fn iova(&self) -> Result { + Err(ENOTSUPP) + } + + fn buf(&mut self) -> Result> { + Err(ENOTSUPP) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) struct ExpectedProvider { + pub(super) role: G17PAscwrapV6Role, + pub(super) wrapper: (u64, u64), + pub(super) iop_vbar: (u64, u64), +} + +pub(super) fn validate_provider( + provider: &G17PMailboxProvider, + expected: ExpectedProvider, +) -> Result { + let lifecycle = provider.lifecycle()?; + if lifecycle.role != expected.role + || lifecycle.wrapper != expected.wrapper + || lifecycle.iop_vbar != expected.iop_vbar + { + return Err(EINVAL); + } + + Ok(lifecycle) +} diff --git a/drivers/gpu/drm/asahi/m5/render.rs b/drivers/gpu/drm/asahi/m5/render.rs new file mode 100644 index 0000000000000..22bede0991343 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/render.rs @@ -0,0 +1,508 @@ +// SPDX-License-Identifier: MIT +//! M5/G17G graphics register construction from owned render resources. +//! +//! Common resource lowering is shared with G17P. Register order, tile packing +//! and per-stage controls follow the native 25G76 M5 reference. This codec is +//! not a claim that Linux TA/3D submission or presentation is qualified. + +use crate::m5::render_desc::{ + self, G17pRenderParameters, RegisterWrite, RenderBuildError, RenderDescriptorMetadata, + RenderDescriptorObjects, +}; + +pub(crate) const TA_REGISTER_COUNT: usize = 82; +pub(crate) const FRAGMENT_REGISTER_COUNT: usize = 90; + +/// Encode the TA parameter-manager metrics slot in the compact GART window. +/// The native 25G76 slot at 0x1000080040 produces register 1c910 = 0x80041: +/// the low bit enables the record; the remaining bits address its 64 bytes. +pub(crate) fn parameter_buffer_metrics_record( + scene_metrics: u64, +) -> Result { + let offset = scene_metrics.checked_sub(0x10_0000_0000) + .ok_or(M5RenderBuildError::OwnershipMismatch)?; + if offset >= (1u64 << 32) || offset & 0x3f != 0 { + return Err(M5RenderBuildError::OwnershipMismatch); + } + Ok(offset | 1) +} + +/// Initial M5 PB slot with fresh ring cursors. The owning job chooses the +/// hardware slot separately and retains the page-list mapping. +pub(crate) fn initial_parameter_buffer_descriptor( + page_list: u64, + page_count: u32, +) -> Result<[u32; 4], M5RenderBuildError> { + if page_list == 0 || page_list >= (1u64 << 42) || page_list & 0x7f != 0 + || page_count == 0 || page_count >= (1 << 22) { + return Err(M5RenderBuildError::OwnershipMismatch); + } + // 25G76 halLoadPBDesc, 89de844..89de8dc. The descriptor has four + // dwords; +0's low flags and both cursors start at zero. Native's + // untagged user address is translated into its PB segment encoding. + let segment = ((page_list + 0x70_0000_0000) & 0x70_0000_0000) >> 8; + Ok([((page_list >> 4) as u32) & !7, + (segment as u32 & 0xf000_0000) | page_count, 0, 0]) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5RenderBuildError { + Common(RenderBuildError), + SharedRegisterLayout, + TileCountOutOfRange, + OverlappingTileFlags, + DispatchBitAlreadySet, + InvalidContext, + OwnershipMismatch, + InvalidUscState, +} + +impl From for M5RenderBuildError { + fn from(error: RenderBuildError) -> Self { + Self::Common(error) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderStageControls { + /// Native host descriptor +0x1120 (TA) / +0x7e8 (fragment). + pub(crate) resource_flags: u64, + /// Explicit per-stage d419/d429 input, independent of the work key. + pub(crate) resource_control: u64, + /// Native +0x1128 / +0x7f0; a zero source selects the native 0x60 fallback. + pub(crate) ptc_max: u32, + pub(crate) work_key: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderControls { + pub(crate) ta: M5RenderStageControls, + pub(crate) fragment: M5RenderStageControls, + pub(crate) ta_tile_control: u64, + /// Base value for the two 1c898 writes. Bit zero is supplied by the codec. + pub(crate) ta_dispatch_control: u64, + pub(crate) fragment_raster_control: u64, + /// Non-geometry bits for 16068. Never includes tile counts, utile or layers. + pub(crate) fragment_tile_flags: u64, + /// M5 adds this second pair of store-program bindings at 153b1/153b9. + pub(crate) store_pipeline_2_bind: u64, + pub(crate) store_pipeline_2: u64, + pub(crate) partial_store_pipeline_2_bind: u64, + pub(crate) partial_store_pipeline_2: u64, + pub(crate) partial_tile_flags: u64, +} + +/// Per-submission USC private-memory accounting, separate from user status +/// registers and queue completion stamps. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderUscState { + pub(crate) control: u64, + /// AGXUMADataRec +0x50. Native allocatePagesEnd derives this from pool + /// sizes and 4-KiB allocation units; it is a byte count, not a GPU VA. + pub(crate) required_bytes: u64, + /// Capacity actually backed by this job's retained USC allocation pages. + pub(crate) allocated_bytes: u64, + /// A 64-byte slot in the firmware-visible statistics ring. + pub(crate) statistics: u64, +} + +impl M5RenderUscState { + fn valid(self) -> bool { + let firmware_range = |address: u64, size: u64| { + address >= 0xffff_fc00_0000_0000 + && address & 0x3f == 0 + && address.checked_add(size).is_some() + }; + firmware_range(self.control, 0x80) + && firmware_range(self.statistics, 0x40) + && self.required_bytes != 0 + && self.required_bytes <= self.allocated_bytes + && self.allocated_bytes & 0xfff == 0 + } +} + +/// Fill the native USC fields only after the caller owns the pool and the +/// statistics slot. Other runtime fields still need queue/link publication. +/// Validate both stages before modifying either descriptor. +pub(crate) fn apply_usc_submission_state( + ta: &mut [u8], + fragment: &mut [u8], + ta_state: M5RenderUscState, + fragment_state: M5RenderUscState, +) -> Result<(), M5RenderBuildError> { + if ta.len() < render_desc::TA_DESCRIPTOR_SIZE + || fragment.len() < render_desc::FRAGMENT_DESCRIPTOR_SIZE + { + return Err(RenderBuildError::BufferTooSmall.into()); + } + if !ta_state.valid() || !fragment_state.valid() { + return Err(M5RenderBuildError::InvalidUscState); + } + // 25G76 AGXTAChannelSKSM::submitBuffer: 8a244c4..8a244ec. + // AGX3DChannelSKSM::submitBuffer takes the corresponding value from + // host +0xba0 = AGXUMADataRec +0x50 at 895b70c..895b718. + for (raw, state, control, required, statistics) in [ + (ta, ta_state, 0x934, 0x93d, 0x945), + (fragment, fragment_state, 0x21ce, 0x21d7, 0x21df), + ] { + put_u64(raw, control, state.control); + put_u64(raw, required, state.required_bytes); + put_u64(raw, statistics, state.statistics); + } + Ok(()) +} + +const EMPTY_REGISTER: RegisterWrite = RegisterWrite { + number: 0, + value: 0, +}; + +struct RegisterList { + values: [RegisterWrite; N], + count: usize, +} + +impl RegisterList { + fn new() -> Self { + Self { + values: [EMPTY_REGISTER; N], + count: 0, + } + } + + fn push(&mut self, number: u32, value: u64) -> Result<(), M5RenderBuildError> { + if self.count == N { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + self.values[self.count] = RegisterWrite { number, value }; + self.count += 1; + Ok(()) + } + + fn finish(self) -> Result<[RegisterWrite; N], M5RenderBuildError> { + if self.count != N { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + Ok(self.values) + } +} + +fn value(registers: &[RegisterWrite], number: u32) -> Result { + registers + .iter() + .find(|r| r.number == number) + .map(|r| r.value) + .ok_or(M5RenderBuildError::SharedRegisterLayout) +} + +const fn ptc_control(maximum: u32) -> u64 { + let maximum = if maximum == 0 { 0x60 } else { maximum }; + ((maximum as u64) << 32) | 0x400020 +} + +fn tile_state(p: &G17pRenderParameters, flags: u64) -> Result { + let tiles_x = (p.width as u64 + 31) / 32; + let tiles_y = (p.height as u64 + 31) / 32; + if !(1..=1024).contains(&tiles_x) || !(1..=1024).contains(&tiles_y) { + return Err(M5RenderBuildError::TileCountOutOfRange); + } + // 25G76 fragment generator: bfi at 89dcb98, 89dcbb8 and 89dcbbc. + let geometry_mask = (0xfffff_u64 << 42) | (0xf_u64 << 38) | (1_u64 << 32); + if flags & geometry_mask != 0 { + return Err(M5RenderBuildError::OverlappingTileFlags); + } + Ok(flags + | ((tiles_x - 1) << 42) + | ((tiles_y - 1) << 52) + | ((p.utile_config & 0xf000) << 26) + | if p.layers > 1 { 1_u64 << 32 } else { 0 }) +} + +fn tile_blocks(pixels: u32) -> u64 { + let tiles = (pixels as u64 + 31) / 32; + let first = (((tiles + 3) / 4) + 3) & !3; + // Native TA host +1208/+120c supplies 0x40200c for the reference: + // boundaries 12, 8, 4 in ten-bit fields, versus G17P's nine-bit fields. + (3 * first) | ((2 * first) << 10) | (first << 20) +} + +pub(crate) fn build_ta_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], M5RenderBuildError> { + // M5's captured register program addresses heap metadata directly. + // G17P's layered path now uses a separate layer table in these slots. + let mut common_parameters = *p; + common_parameters.layermeta = p.heapmeta; + let common = render_desc::build_ta_registers(&common_parameters)?; + if controls.ta_dispatch_control & 1 != 0 { + return Err(M5RenderBuildError::DispatchBitAlreadySet); + } + // Validate the M5 tile fields even when only constructing the TA half. + tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterList::new(); + for mut register in common { + match register.number { + 0x0a309 | 0x0a5a1 | 0x0d419 => continue, + 0x1c8f8 => { + out.push(0x0d499, controls.ta.resource_flags)?; + out.push(0x0d419, controls.ta.resource_control)?; + out.push(0x0a5a1, ptc_control(controls.ta.ptc_max))?; + register.value = controls.ta_tile_control; + } + 0x10171 => { + for (number, source) in [(0x1ca50, 0x101b9), (0x1ca58, 0x10149), (0x1ca60, 0x10141)] + { + out.push(number, value(&common, source)?)?; + } + out.push(0x1ca70, tile_blocks(p.width))?; + out.push(0x1ca78, tile_blocks(p.height))?; + out.push(0x1ca68, value(&common, 0x1c081)?)?; + } + 0x1c069 => register.value = tile_blocks(p.width), + 0x1c071 => register.value = tile_blocks(p.height), + 0x1c898 => { + if register.value > 1 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + register.value |= controls.ta_dispatch_control; + } + 0x10799 => out.push(0x1a149, 0)?, + 0x10209 => { + out.push(0x0a739, 0)?; + register.value = controls.ta.work_key; + } + 0x1c9f0 | 0x14320 => register.value = controls.ta.work_key, + 0x14308 => register.value = p.usc_flist_hardware_buffer_id as i32 as i64 as u64, + _ => {} + } + out.push(register.number, register.value)?; + if register.number == 0x01740 { + out.push(0x0a739, 0)?; + } + } + out.finish() +} + +pub(crate) fn build_fragment_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], M5RenderBuildError> { + // M5's captured register program addresses heap metadata directly. + // G17P's layered path now uses a separate layer table in these slots. + let mut common_parameters = *p; + common_parameters.layermeta = p.heapmeta; + let common = render_desc::build_fragment_registers(&common_parameters)?; + let geometry = tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterList::new(); + let mut dispatch_writes = 0; + let mut tile_writes = 0; + for mut register in common { + match register.number { + 0x0a301 | 0x15109 | 0x15101 | 0x160a8 | 0x0a5a9 | 0x0d429 => continue, + 0x100b1 => register.number = 0x16469, + 0x100d9 => register.number = 0x16471, + 0x100b8 => register.number = 0x16138, + 0x10791 => register.value = controls.fragment_raster_control, + 0x15321 => { + out.push(0x0d491, controls.fragment.resource_flags)?; + out.push(0x0d429, controls.fragment.resource_control)?; + out.push(0x0a5a9, ptc_control(controls.fragment.ptc_max))?; + } + 0x16068 => { + tile_writes += 1; + if tile_writes == 1 { + register.value = geometry; + } + } + 0x16020 => { + dispatch_writes += 1; + if dispatch_writes == 3 { + if register.value != 0 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + continue; + } + } + 0x160e0 => out.push(0x1a151, 0)?, + 0x10211 => { + out.push(0x0a741, 0)?; + register.value = controls.fragment.work_key; + } + 0x10420 => register.value = controls.fragment.work_key, + 0x14048 => register.value = p.usc_flist_hardware_buffer_id as i32 as i64 as u64, + _ => {} + } + out.push(register.number, register.value)?; + if register.number == 0x15381 { + out.push(0x153b1, controls.store_pipeline_2_bind)?; + out.push(0x153b9, controls.store_pipeline_2)?; + } + if register.number == 0x01731 { + out.push(0x0a741, 0)?; + } + } + if dispatch_writes != 3 || tile_writes != 2 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + out.finish() +} + +fn partial_registers( + common: &[RegisterWrite], + insert_after: u32, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; N], M5RenderBuildError> { + let mut out = RegisterList::new(); + for register in common { + let bits = if register.number == 0x10039 { + controls.partial_tile_flags + } else { + 0 + }; + out.push(register.number, register.value | bits)?; + if register.number == insert_after { + out.push(0x153b1, controls.partial_store_pipeline_2_bind)?; + out.push(0x153b9, controls.partial_store_pipeline_2)?; + } + } + out.finish() +} + +pub(crate) fn build_partial_store_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 18], M5RenderBuildError> { + partial_registers(&render_desc::partial_store_registers(p), 0x15381, controls) +} + +pub(crate) fn build_partial_resume_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 25], M5RenderBuildError> { + partial_registers(&render_desc::partial_resume_registers(p), 0x15381, controls) +} + +pub(crate) fn build_partial_load_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 12], M5RenderBuildError> { + partial_registers(&render_desc::partial_load_registers(p), 0x15309, controls) +} + +fn work_key( + p: &G17pRenderParameters, + metadata: RenderDescriptorMetadata, +) -> Result { + if metadata.context_id == 0 || metadata.context_id > 255 { + return Err(M5RenderBuildError::InvalidContext); + } + if metadata.ta_hardware_buffer_id != p.ta_hardware_buffer_id { + return Err(M5RenderBuildError::OwnershipMismatch); + } + metadata + .submission_ordinal + .checked_add(metadata.submission_ordinal / 2) + .ok_or(M5RenderBuildError::Common( + RenderBuildError::ArithmeticOverflow, + ))?; + Ok(((metadata.context_id as u64) << 8) | (metadata.command_buffer_state_word & 0x3f) as u64) +} + +fn alias_offset(alias: u64, offset: u64) -> Result { + alias.checked_add(offset).ok_or(M5RenderBuildError::Common( + RenderBuildError::ArithmeticOverflow, + )) +} + +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Construct the M5 TA record's common layout and all register programs. +/// Runtime ownership/completion fields must be populated before publication. +#[inline(never)] +pub(crate) fn build_ta_descriptor_base( + p: &G17pRenderParameters, + controls: &M5RenderControls, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + low_alias: u64, + raw: &mut [u8], +) -> Result<(), M5RenderBuildError> { + let mut controls = *controls; + controls.ta.work_key = work_key(p, metadata)?; + let program = alias_offset(low_alias, 0x60)?; + alias_offset(low_alias, render_desc::TA_DESCRIPTOR_SIZE as u64)?; + let registers = build_ta_registers(p, &controls)?; + render_desc::build_ta_descriptor_base(p, objects, metadata, ®isters, raw)?; + // Native boundaries: main work ends before the final a739/deflake/dispatch. + for (offset, count) in [ + (0x38, TA_REGISTER_COUNT - 3), + (0x3a, TA_REGISTER_COUNT), + (0x3c, TA_REGISTER_COUNT), + ] { + put_u16(raw, offset, count as u16); + } + put_u64(raw, 0x760, program); + Ok(()) +} + +/// Construct the M5 fragment record, including all three partial programs. +/// The shared encoder is pure and does not apply Neo diagnostic overrides. +#[inline(never)] +pub(crate) fn build_fragment_descriptor_base( + p: &G17pRenderParameters, + controls: &M5RenderControls, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + low_alias: u64, + raw: &mut [u8], +) -> Result<(), M5RenderBuildError> { + let mut controls = *controls; + controls.fragment.work_key = work_key(p, metadata)?; + alias_offset(low_alias, render_desc::FRAGMENT_DESCRIPTOR_SIZE as u64)?; + let main = build_fragment_registers(p, &controls)?; + let store = build_partial_store_registers(p, &controls)?; + let resume = build_partial_resume_registers(p, &controls)?; + let load = build_partial_load_registers(p, &controls)?; + render_desc::build_fragment_descriptor_base( + p, + objects, + metadata, + render_desc::G17pFragmentRegisters { + main: &main, + store: &store, + resume: &resume, + load: &load, + }, + raw, + )?; + put_u32(raw, 0x50, 1); + put_u32(raw, 0x90, 1); + for (offset, count) in [ + (0x80, FRAGMENT_REGISTER_COUNT - 3), + (0x82, FRAGMENT_REGISTER_COUNT - 1), + (0x84, FRAGMENT_REGISTER_COUNT - 1), + (0x88, FRAGMENT_REGISTER_COUNT), + ] { + put_u16(raw, offset, count as u16); + } + for (offset, target) in [ + (0x7a0, 0xa0), + (0xec0, 0x7c0), + (0x15e0, 0xee0), + (0x1d00, 0x1600), + ] { + put_u64(raw, offset, alias_offset(low_alias, target)?); + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/m5/render_desc.rs b/drivers/gpu/drm/asahi/m5/render_desc.rs new file mode 100644 index 0000000000000..42a556acde208 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/render_desc.rs @@ -0,0 +1,1138 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +const TA_REGISTER_COUNT: usize = 73; + +const FRAGMENT_REGISTER_COUNT: usize = 89; + +/// Both embedded register lists match the kext EXACTLY -- same writes, same +/// unique registers, same order -- verified against 2048-byte captures of the +/// native TA and 3D command records (macos-trace recs2048-gfx.txt): +/// +/// TA 73 writes, 71 unique, list ends at +0x3cc +/// 3D 89 writes, 86 unique, list ends at +0x4cc +/// +/// A 1024-byte capture cuts the 3D list at exactly +0x400 and hides its last +/// 17 writes, which briefly looked like 14 registers "Metal never writes" -- +/// they were entries 72..85. Assert the counts so a future truncated capture +/// cannot produce that conclusion again. +const _: () = assert!(TA_REGISTER_COUNT == 73); + +const _: () = assert!(FRAGMENT_REGISTER_COUNT == 89); + +const PARTIAL_STORE_REGISTER_COUNT: usize = 16; + +const PARTIAL_RESUME_REGISTER_COUNT: usize = 23; + +const PARTIAL_LOAD_REGISTER_COUNT: usize = 10; + +const REGISTER_BYTES: usize = 12; + +pub(crate) const TA_DESCRIPTOR_SIZE: usize = 0x9c0; + +pub(crate) const FRAGMENT_DESCRIPTOR_SIZE: usize = 0x2240; + +/// Native render-setup field copied through command descriptor +0xc00 to the +/// final 3D command tail at +0x2110. This is independent of SGX's GPC +/// performance-state map register. +const G17P_FRAGMENT_TAIL_SETUP_WORD: u32 = u32::MAX; + +/// J700 shared-runtime flag snapshot copied by native 3D submit into type-1 +/// command lanes +0x2124/+0x2128. +/// +/// The kext resolves the accelerator's shared event-stamp/HWDSID internal +/// resource at +0x1a130, obtains its CPU mapping, reads byte bit 0 at mapping +/// +0x2ad/+0x2ae, and widens them before per-draw register generation. The +/// working Neo Metal snapshot contains 1/1. +const G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT: [u32; 2] = [1, 1]; + +pub(crate) const G17P_EOT_RESOURCE_SPEC: u64 = 0x8; + +pub(crate) const G17P_LOAD_RESOURCE_SPEC: u64 = 0x0007_8000_0000_0040; + +pub(crate) const G17P_SOURCE_BG_PIPELINE: u64 = 0x0000_0100_0199_0240; + +pub(crate) const G17P_SOURCE_EOT_PIPELINE: u64 = 0x0000_0100_0199_0640; + +const TILE_WIDTH: u64 = 32; + +const TILE_HEIGHT: u64 = 32; + +const MACRO_TILES_X: u64 = 4; + +const MACRO_TILES_Y: u64 = 4; + +const REGION_ENTRY_SIZE: u64 = 5; + +/// Base added by G17P's compact VDM control-stream register (0x1c880). +/// The register retains the legacy 40-bit {TTBR selector, 39-bit offset} +/// shape, so lower-root user addresses are encoded relative to this base. +const G17P_USER_BASE: u64 = 0x10_0000_0000; + +const G17P_VDM_USER_WINDOW_SIZE: u64 = 1 << 39; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RenderBuildError { + ZeroDimensions, + LayerCountOutOfRange, + UtileDimensions, + AddressBeforeContext, + UserAddressOutOfRange, + VdmAddressOutOfRange, + BufferTooSmall, + ArithmeticOverflow, +} + +impl RenderBuildError { +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWrite { + pub(crate) number: u32, + pub(crate) value: u64, +} + +impl RegisterWrite { + const fn new(number: u32, value: u64) -> Self { + Self { number, value } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17pRenderParameters { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) context_base: u64, + /// Allocation epoch of the trusted execution-context lease, shared by TA/3D. + pub(crate) context_generation: u8, + pub(crate) tilemap: u64, + /// Per-layer tiler metadata, which native places between the tilemap and + /// the heap metadata. It is empty, and so equal to `heapmeta`, for a + /// single-layer target. + pub(crate) layermeta: u64, + pub(crate) heapmeta: u64, + pub(crate) tpc: u64, + /// Geometry size of the owned TPC subregion, before capacity page rounding. + pub(crate) tpc_size: u64, + pub(crate) deflake_1: u64, + pub(crate) deflake_2: u64, + pub(crate) deflake_3: u64, + pub(crate) encoder: u64, + pub(crate) ta_status: u64, + pub(crate) fragment_status: u64, + pub(crate) store_pipeline_bind: u64, + pub(crate) store_pipeline: u64, + pub(crate) load_pipeline_bind: u64, + pub(crate) load_pipeline: u64, + pub(crate) partial_store_pipeline_bind: u64, + pub(crate) partial_store_pipeline: u64, + pub(crate) partial_load_pipeline_bind: u64, + pub(crate) partial_load_pipeline: u64, + pub(crate) scissor_array: u64, + pub(crate) depth_bias_array: u64, + pub(crate) aux_fb: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth_buffer: u64, + pub(crate) stencil_buffer: u64, + pub(crate) depth_aux_buffer: u64, + pub(crate) stencil_aux_buffer: u64, + pub(crate) depth_stride: u64, + pub(crate) stencil_stride: u64, + pub(crate) depth_aux_stride: u64, + pub(crate) stencil_aux_stride: u64, + pub(crate) depth_clear_value_bits: u32, + pub(crate) stencil_clear_value: u32, + pub(crate) depth_flags: u64, + /// ZLS control used by every firmware partial-store/load/resume phase. + /// Unlike the pass-boundary control, an attached depth/stencil surface + /// must be both loaded and stored across an internal split. + pub(crate) partial_depth_flags: u64, + pub(crate) depth_dimensions: u64, + pub(crate) merge_upper_x_bits: u32, + pub(crate) merge_upper_y_bits: u32, + pub(crate) layers: u16, + /// Sample count: 1, 2 or 4. Defaults to 1, so every existing parameter + /// vector keeps its exact register program. + pub(crate) samples: u8, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) utile_config: u64, + pub(crate) multisample_control: u64, + pub(crate) ppp_control: u64, + pub(crate) tib_blocks: u64, + pub(crate) tile_config: u64, + pub(crate) aux_fb_flags: u64, + pub(crate) aux_fb_page_count: u64, + pub(crate) cycle: u64, + pub(crate) record_index: u64, + /// TA event correlation written to 0x1ca10/0x014a1/0x0a349. + /// Native sources this independently from the fragment correlation. + pub(crate) lifecycle: u64, + /// Fragment event correlation written to 0x160e0/0x01499/0x0a341. + pub(crate) fragment_lifecycle: u64, + pub(crate) sampler_array: u64, + pub(crate) sampler_count: u32, + pub(crate) process_empty_tiles: bool, + pub(crate) ta_timestamp_start: u64, + pub(crate) ta_timestamp_end: u64, + pub(crate) fragment_timestamp_start: u64, + pub(crate) fragment_timestamp_end: u64, + /// Write the VDM control-stream register (0x1c880) as a full GPU virtual + /// address instead of the native user-process compact form. + /// + /// SCOPE CORRECTED against the macOS oracle. This flag once covered every + /// register `ta_offset` feeds, justified from the legacy G14X path where + /// all of them are whole pointers. A real Metal render pass on THIS chip + /// says otherwise: the captured TA command record writes tilemap, TPC and + /// heapmeta as small context-relative offsets (see `ta_offset`). The legacy + /// encoding does not transfer to G17P, so those six registers are always + /// relative now and this flag applies to the VDM stream alone. Native first + /// lowers the task VA to a task-relative offset, maps the same backing at + /// 0x1000000000 + offset in the GART window, and writes that offset here. + /// Linux userspace supplies the full GART-alias VA; this kernel subtracts + /// the architectural 0x1000000000 base. Hardware adds it back for compact + /// pointers whose bit 39 is clear. + pub(crate) absolute_pointers: bool, + /// + /// Legacy diagnostic OR-mask for the BG resource-specifier high 17 bits. + /// Native calls this field PassControlMaskType and derives it from shader + /// outputs, including extension of the highest active pass bit; it is not + /// an address class or a universal constant. Main and partial shaders can + /// legitimately differ. Default zero preserves the userspace mask; + /// a nonzero value is an explicit diagnostic override only. + pub(crate) bg_resource_prefix: u64, + /// Historical opt-in diagnostic changing byte 2 of the USC private-memory + /// budget qword at TA +0x93d. It must not override +0x94d: that byte is the + /// allocation generation of the execution context, not a PM mode flag. + /// The captured budget is not yet derived from our pool allocation. + pub(crate) native_pm_bytes: bool, + /// Hardware-buffer ID allocated for this TA command. Native allocates it + /// from the accelerator's 0x7f-entry `AGXHardwareBufferIDManager`, stores + /// it in the host-side `AGXTACommandDescriptor` at +0x1248/+0x928, and the + /// register generator emits that value as register 0x1c830. Those two host + /// offsets are not fields in this final 0x9c0-byte firmware command. + pub(crate) ta_hardware_buffer_id: u32, + /// HardwareBufferID of the retained USC private-memory FList shared by TA + /// and 3D. Native allocates this from the USC manager (accelerator + /// +0x133c8), independently of the TA parameter-buffer manager above, and + /// emits the same ID in TA register 0x14308 and fragment register 0x14048. + pub(crate) usc_flist_hardware_buffer_id: u32, + /// Diagnostic capture-value bitmask retained for fields whose proper + /// inputs are not modeled yet. In particular, the 0xa5a* values are + /// assembled from workload-derived USC pool limits. Their captured values + /// are not architectural literals and must not become defaults. + pub(crate) native_ta_registers: u32, + /// Retained GPU-ID diagnostic snapshot, not the fragment parameter-buffer + /// owner. The old 0x1c838 use confused host descriptor +0x928 with a + /// different object's transient raw-command +0x928 field. + pub(crate) gpc_perf_state_map: u32, + /// Low nibble of SGX 0xe01480, copied by native 3D submit to tail +0x210c. + pub(crate) gpc_perf_state_map_low: u32, + /// Bit 0 of SGX 0xe0141c, copied by native 3D submit to tail +0x2104. + pub(crate) gpc_perf_state_control: u32, + /// GPU address of the selected native 0x80 Scene-state entry, or 0 for + /// none. The command-record binding is carried by `RenderDescriptorObjects` + /// in both TA and 3D; this copy is retained for diagnostics. + pub(crate) parameter_buffer: u64, +} + +impl G17pRenderParameters { + + pub(crate) const fn source_rsrc8() -> Self { + Self { + width: 64, + height: 64, + context_base: 0x0000_0010_0000_0000, + context_generation: 1, + tilemap: 0x0000_0010_001b_0000, + layermeta: 0x0000_0010_001b_5000, + heapmeta: 0x0000_0010_001b_5000, + tpc: 0x0000_0010_0024_0000, + tpc_size: 0x800, + deflake_1: 0x0000_0010_0006_82a0, + deflake_2: 0x0000_0010_0006_8020, + deflake_3: 0x0000_0010_0006_8000, + encoder: 0x0000_0010_0001_8000, + ta_status: 0x0000_0010_0007_8000, + fragment_status: 0x0000_0010_001a_8000, + store_pipeline_bind: G17P_EOT_RESOURCE_SPEC, + store_pipeline: G17P_SOURCE_EOT_PIPELINE, + load_pipeline_bind: G17P_LOAD_RESOURCE_SPEC, + load_pipeline: G17P_SOURCE_BG_PIPELINE, + partial_store_pipeline_bind: 0, + partial_store_pipeline: 0, + partial_load_pipeline_bind: 0, + partial_load_pipeline: 0, + scissor_array: 0x0000_0100_019a_0000, + depth_bias_array: 0x0000_0100_01af_8000, + aux_fb: 0x0000_0100_01aa_8000, + occlusion_query_base: 0, + depth_buffer: 0, + stencil_buffer: 0, + depth_aux_buffer: 0, + stencil_aux_buffer: 0, + depth_stride: 0, + stencil_stride: 0, + depth_aux_stride: 0, + stencil_aux_stride: 0, + depth_clear_value_bits: 0x3f80_0000, + stencil_clear_value: 0, + depth_flags: 0, + partial_depth_flags: 0, + depth_dimensions: 0, + merge_upper_x_bits: 0x3cdd_b3d9, + merge_upper_y_bits: 0x3cdd_b3d9, + layers: 1, + samples: 1, + utile_width: 32, + utile_height: 32, + utile_config: 0xa000, + multisample_control: 0x88, + ppp_control: 0x202, + tib_blocks: 8, + tile_config: 0x10280, + aux_fb_flags: 0xc001, + aux_fb_page_count: 0x10_0000, + cycle: 0x17_8020, + record_index: 0x8_0005, + lifecycle: 0, + fragment_lifecycle: 0, + sampler_array: 0, + sampler_count: 0, + process_empty_tiles: true, + ta_timestamp_start: 0xffff_fc20_0002_4c68, + ta_timestamp_end: 0, + fragment_timestamp_start: 0xffff_fc20_0002_4c68, + fragment_timestamp_end: 0xffff_fc20_0002_4c70, + absolute_pointers: false, + bg_resource_prefix: 0, + native_pm_bytes: false, + ta_hardware_buffer_id: 0, + usc_flist_hardware_buffer_id: 0, + native_ta_registers: 0, + gpc_perf_state_map: 0, + gpc_perf_state_map_low: 0, + gpc_perf_state_control: 0, + parameter_buffer: 0, + } + } +} + +#[derive(Debug, Copy, Clone)] +struct Geometry { + tiles_x: u64, + tiles_y: u64, + size1: u64, + size2: u64, + size3: u64, + x_blocks: u64, + y_blocks: u64, + screen: u64, + pixels: u64, + macro_size: u64, +} + +const fn align(value: u64, alignment: u64) -> u64 { + (value + alignment - 1) & !(alignment - 1) +} + +fn geometry(parameters: &G17pRenderParameters) -> Result { + if parameters.width == 0 || parameters.height == 0 { + return Err(RenderBuildError::ZeroDimensions); + } + if parameters.layers == 0 || parameters.layers > 2048 { + return Err(RenderBuildError::LayerCountOutOfRange); + } + + let width = parameters.width as u64; + let height = parameters.height as u64; + let tiles_x = (width + TILE_WIDTH - 1) / TILE_WIDTH; + let tiles_y = (height + TILE_HEIGHT - 1) / TILE_HEIGHT; + let (utile_width, utile_height) = (parameters.utile_width, parameters.utile_height); + if !matches!((utile_width, utile_height), (32, 32) | (32, 16) | (16, 16)) { + return Err(RenderBuildError::UtileDimensions); + } + let utiles_per_tile_x = TILE_WIDTH / utile_width as u64; + let utiles_per_tile_y = TILE_HEIGHT / utile_height as u64; + // Four samples split the utile in Y. Measured against Apple's own + // four-sample render: size1 0x14 -> 0x28, size3 0x20 -> 0x40 and + // macro_size 0x40004 -> 0x40008, where only the y term moves. All three + // follow from doubling this one term; two samples do not split. + let utiles_per_tile_y = if parameters.samples == 4 { + utiles_per_tile_y * 2 + } else { + utiles_per_tile_y + }; + let utiles_per_tile = utiles_per_tile_x * utiles_per_tile_y; + let mtile_x1 = align((tiles_x + MACRO_TILES_X - 1) / MACRO_TILES_X, 4); + let mtile_y1 = align((tiles_y + MACRO_TILES_Y - 1) / MACRO_TILES_Y, 4); + let mtile_x2 = 2 * mtile_x1; + let mtile_x3 = 3 * mtile_x1; + let mtile_y2 = 2 * mtile_y1; + let mtile_y3 = 3 * mtile_y1; + let mtile_stride = mtile_x1 * mtile_y1; + let size1 = (REGION_ENTRY_SIZE * mtile_stride * utiles_per_tile + 3) / 4; + + Ok(Geometry { + tiles_x, + tiles_y, + size1, + size2: mtile_stride, + size3: 2 * mtile_stride * utiles_per_tile, + x_blocks: mtile_x3 | (mtile_x2 << 9) | (mtile_x1 << 18), + y_blocks: mtile_y3 | (mtile_y2 << 9) | (mtile_y1 << 18), + screen: ((tiles_y - 1) << 12) | (tiles_x - 1), + pixels: (width - 1) | ((height - 1) << 16), + macro_size: mtile_y1 * utiles_per_tile_y | ((mtile_x1 * utiles_per_tile_x) << 16), + }) +} + +/// Context-relative offset. ALWAYS relative -- a real Metal render pass on this +/// chip writes small offsets into these registers, not addresses. +/// +/// Oracle: the TA command record captured from `mtlg.m` (macos-trace +/// `recs-gfx.txt`, dumped by `recs.d` at `requestChannelCommandTA`) carries an +/// embedded register list that decodes field-for-field against this function's +/// output -- same registers, same order, 12-byte entries of u32 number plus u64 +/// value -- with these values: +/// +/// 0x01748 = 1 (matches) +/// 0x10141 = 0x200 (matches) +/// 0x1c039 = 0x0119ba00 tilemap <- OFFSET, not a VA +/// 0x1c9c8 = 0x0119ba00 tilemap +/// 0x1c0a1 = 0x01290000 tpc <- OFFSET +/// 0x1c031 = 0x800000000119ca00 heapmeta | bit63 (matches the form) +/// 0x1c9c0 = 0x800000000119ca00 heapmeta +/// 0x1c051 = 0x003a0012006b0003 (matches the constant exactly) +/// +/// Writing full virtual addresses here -- which is what the legacy G14X path +/// does, and what an earlier version of this driver was changed to do -- points +/// the tiler's tilemap, TPC and heapmeta at values the firmware cannot use. +fn ta_resource_offset( + parameters: &G17pRenderParameters, + address: u64, +) -> Result { + address + .checked_sub(parameters.context_base) + .ok_or(RenderBuildError::AddressBeforeContext) +} + +/// Compact user/GART address consumed by the TA register and descriptor +/// fields. Native maps these objects in the architectural 0x1000000000 window +/// and publishes the offset, not the full address. Descriptor +0x7d6 is only +/// 32 bits wide, so reject aliases that cannot be represented consistently. +fn ta_user_offset(address: u64) -> Result { + let offset = address + .checked_sub(G17P_USER_BASE) + .ok_or(RenderBuildError::UserAddressOutOfRange)?; + + if offset > u32::MAX as u64 { + return Err(RenderBuildError::UserAddressOutOfRange); + } + + Ok(offset) +} + +fn vdm_user_offset(address: u64) -> Result { + let offset = address + .checked_sub(G17P_USER_BASE) + .ok_or(RenderBuildError::VdmAddressOutOfRange)?; + + if offset >= G17P_VDM_USER_WINDOW_SIZE { + return Err(RenderBuildError::VdmAddressOutOfRange); + } + + Ok(offset) +} + +pub(crate) fn build_ta_registers( + p: &G17pRenderParameters, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], RenderBuildError> { + build_ta_registers_for_work_key(p, render_work_key(1, 0)) +} + +fn build_ta_registers_for_work_key( + p: &G17pRenderParameters, + work_key: u64, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], RenderBuildError> { + let g = geometry(p)?; + let tilemap = ta_resource_offset(p, p.tilemap)?; + let layermeta = ta_resource_offset(p, p.layermeta)?; + let heapmeta = ta_resource_offset(p, p.heapmeta)?; + let tpc = ta_resource_offset(p, p.tpc)?; + let deflake_1 = ta_user_offset(p.deflake_1)?; + let deflake_2 = ta_user_offset(p.deflake_2)?; + let deflake_3 = ta_user_offset(p.deflake_3)?; + // The VDM control stream is the one pointer the 192-byte capture window + // does not reach (register 0x1c880 sits past the end of the dump), and it + // is the one that is not relative to the driver-private `context_base`. + // macOS first lowers the task VA to a task-relative offset, then maps the + // backing at 0x1000000000 + offset in the GART window. Linux userspace gives + // us that full alias VA, so this hardware encoder subtracts the GART base. + // Bit 39 selects the upper UAT root; refuse an offset that would silently + // change roots. Keep the full-address escape hatch diagnostic-only. + let encoder = if p.absolute_pointers { + p.encoder + } else { + vdm_user_offset(p.encoder)? + }; + // generateTARegisterListG16Common: the layer count less one with bit 15, + // plus bit 14 and a process-empty-tiles select for a layered target. The + // pipe mirror 0x1c9e8 keeps only the layered bit and the count. + let mut layer_mode = 0x8000 | (p.layers as u64 - 1); + if p.layers > 1 { + layer_mode |= 0x4000 | if p.process_empty_tiles { 0x2000 } else { 0x1000 }; + } + + Ok([ + RegisterWrite::new(0x01748, 1), + // Native N=1 uses 0x200 and N=2 uses 0x0. The one-layer source profile + // selects the N=1 value, matching descriptor byte +0x789 above. + RegisterWrite::new(0x10141, 0x200), + RegisterWrite::new(0x1c039, tilemap), + RegisterWrite::new(0x1c9c8, tilemap), + RegisterWrite::new(0x1c0a1, tpc), + RegisterWrite::new(0x1c031, heapmeta | 0x8000_0000_0000_0000), + RegisterWrite::new(0x1c9c0, heapmeta | 0x8000_0000_0000_0000), + RegisterWrite::new(0x1c051, 0x003a_0012_006b_0003), + RegisterWrite::new(0x1c061, 1), + RegisterWrite::new(0x10149, p.utile_config), + RegisterWrite::new(0x10139, p.multisample_control), + RegisterWrite::new(0x10111, deflake_1), + RegisterWrite::new(0x1c9b0, deflake_1), + RegisterWrite::new(0x10119, deflake_2), + RegisterWrite::new(0x1c9b8, deflake_2), + RegisterWrite::new(0x1c958, 1), + RegisterWrite::new(0x1c950, deflake_3 | 0x0004_0000_0000_0000), + RegisterWrite::new(0x1c930, 0), + RegisterWrite::new(0x1c880, encoder), + RegisterWrite::new(0x1c079, layermeta), + RegisterWrite::new(0x1c9d8, layermeta), + RegisterWrite::new(0x10151, 0), + RegisterWrite::new(0x1c199, 0), + RegisterWrite::new(0x1c1a1, 0), + RegisterWrite::new(0x1c1a9, 0), + RegisterWrite::new(0x1c1b1, 0), + RegisterWrite::new(0x1c1b9, 0), + RegisterWrite::new(0x1c8f8, 0x8860), + RegisterWrite::new(0x1c0b1, g.size1), + RegisterWrite::new(0x1c850, g.size1), + // Silicon holds 0x10131 at 0x88 at ONE, TWO and FOUR samples while + // 0x10139 carries the real sample position table (all five captures in + // logs/m5-msaa-native-hv-20260915-101837). Driving this from + // multisample_control would write the position table here, which + // Apple never does. + RegisterWrite::new(0x10131, 0x88), + RegisterWrite::new(0x10121, p.ppp_control), + RegisterWrite::new(0x10129, g.pixels), + RegisterWrite::new(0x101b9, g.screen), + RegisterWrite::new(0x1c069, g.x_blocks), + RegisterWrite::new(0x1c071, g.y_blocks), + RegisterWrite::new(0x1c081, g.size2), + RegisterWrite::new(0x1c0a9, g.size3), + RegisterWrite::new(0x10171, 0x100), + RegisterWrite::new(0x10169, layer_mode), + RegisterWrite::new(0x0a309, 0), + RegisterWrite::new(0x1c8e0, u64::MAX), + RegisterWrite::new(0x1c8e8, u64::MAX), + RegisterWrite::new(0x1c898, 0), + RegisterWrite::new(0x101e1, 0x1c), + RegisterWrite::new(0x1c9e8, layer_mode & 0x47ff), + RegisterWrite::new(0x1a099, 0), + RegisterWrite::new(0x1a0a1, 0), + RegisterWrite::new(0x1a069, 0), + RegisterWrite::new(0x1a071, 0), + RegisterWrite::new(0x1a0c9, 0), + RegisterWrite::new(0x1a0d1, 0), + RegisterWrite::new(0x101c9, 0), + RegisterWrite::new(0x0d471, 0), + RegisterWrite::new(0x1a0f1, 8), + RegisterWrite::new(0x10799, 0xff_0000), + // Native sources this from AGXHardwareBufferIDManager. It is neither a + // PBDesc ring index nor a capture-stable literal. + RegisterWrite::new(0x1c830, p.ta_hardware_buffer_id as u64), + RegisterWrite::new(0x1ca30, p.cycle), + RegisterWrite::new(0x16c39, p.cycle), + RegisterWrite::new(0x1c910, p.record_index), + // Native derives the high word from the TA AGXUMADataRec PTC/DUPM + // maximum at host descriptor +0x1488, copied through command +0x1158. + // halUpdatePTCMaxCount computes max(0x60, min(pool_input / active_kicks, + // 0x200)); Linux does not yet carry that pool input. Follow the kext's + // exact zero-source fallback 0x60_0040_0020. Bit 1 retains one working + // Metal trace's workload-derived 0xfe value as a diagnostic only. + RegisterWrite::new( + 0x0a5a1, + if p.native_ta_registers & 2 != 0 { + 0x0000_00fe_0040_0020 + } else { + 0x0000_0060_0040_0020 + }, + ), + RegisterWrite::new(0x0d419, 0x0000_0002_0000_0001), + RegisterWrite::new(0x1ca10, p.lifecycle), + RegisterWrite::new(0x014a1, p.lifecycle), + RegisterWrite::new(0x0a349, p.lifecycle), + RegisterWrite::new(0x10209, work_key), + RegisterWrite::new(0x1c9f0, work_key), + RegisterWrite::new(0x14320, work_key), + // AGXTACommandDescriptor::prepare receives the shared USC FList ID + // through descriptor +0x1410. Optional GART/feature payloads are absent + // in Linux, so the retained owner's ID is the complete value. + RegisterWrite::new(0x14308, p.usc_flist_hardware_buffer_id as u64), + RegisterWrite::new(0x14318, p.ta_status | 1), + RegisterWrite::new(0x01740, 1), + RegisterWrite::new(0x1c880, deflake_1), + RegisterWrite::new(0x1c898, 1), + ]) +} + +pub(crate) fn build_fragment_registers( + p: &G17pRenderParameters, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], RenderBuildError> { + build_fragment_registers_for_work_key(p, render_work_key(1, 1)) +} + +fn build_fragment_registers_for_work_key( + p: &G17pRenderParameters, + work_key: u64, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], RenderBuildError> { + let g = geometry(p)?; + let tile_state = 0x3717f + | ((g.tiles_x - 1) << 44) + | ((g.tiles_y - 1) << 53) + | 0x20_0000_0000 + | (if p.layers > 1 { 0x1_0000_0000 } else { 0 }) + | ((p.utile_config & 0xf000) << 28); + + Ok([ + RegisterWrite::new(0x01739, 1), + RegisterWrite::new(0x10009, p.utile_config), + RegisterWrite::new(0x15379, p.store_pipeline_bind), + RegisterWrite::new(0x15381, p.store_pipeline), + RegisterWrite::new(0x15369, p.load_pipeline_bind), + RegisterWrite::new(0x15371, p.load_pipeline), + RegisterWrite::new(0x15131, p.merge_upper_x_bits as u64), + RegisterWrite::new(0x15139, p.merge_upper_y_bits as u64), + RegisterWrite::new(0x100a1, 0), + RegisterWrite::new(0x15069, 0), + RegisterWrite::new(0x15071, 0), + RegisterWrite::new(0x16058, 0), + RegisterWrite::new(0x10019, p.multisample_control), + RegisterWrite::new(0x100b1, g.macro_size), + RegisterWrite::new(0x16030, g.macro_size), + RegisterWrite::new(0x100d9, g.screen), + RegisterWrite::new(0x0a301, 0), + RegisterWrite::new(0x10791, 0xff_0200), + RegisterWrite::new(0x16098, p.heapmeta), + RegisterWrite::new(0x15109, p.scissor_array), + RegisterWrite::new(0x15101, p.depth_bias_array), + RegisterWrite::new(0x15021, p.aux_fb_flags), + RegisterWrite::new(0x15211, ((p.height as u64) << 32) | p.width as u64), + RegisterWrite::new(0x15049, p.aux_fb_page_count), + RegisterWrite::new(0x10051, p.tib_blocks), + RegisterWrite::new(0x15321, p.depth_dimensions), + RegisterWrite::new(0x15301, p.depth_clear_value_bits as u64), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15311, p.occlusion_query_base), + RegisterWrite::new(0x15319, p.depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + RegisterWrite::new(0x15351, 0), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x15231, 0), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15239, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15401, p.depth_stride), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15409, p.stencil_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15411, p.depth_aux_stride), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15419, p.stencil_aux_stride), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x16429, p.tilemap), + RegisterWrite::new(0x16060, p.layermeta), + RegisterWrite::new(0x16431, (4 * g.size1) << 24), + RegisterWrite::new(0x10039, p.tile_config), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16451, 0), + RegisterWrite::new(0x15359, 0), + RegisterWrite::new(0x100b8, 0x8860), + RegisterWrite::new(0x16461, p.aux_fb), + RegisterWrite::new(0x16090, p.aux_fb), + RegisterWrite::new(0x101e9, 0x1c), + RegisterWrite::new(0x160a8, 0), + RegisterWrite::new(0x16068, tile_state), + RegisterWrite::new(0x1a0a9, 0), + RegisterWrite::new(0x1a0b1, 0), + RegisterWrite::new(0x1a079, 0), + RegisterWrite::new(0x1a081, 0), + RegisterWrite::new(0x1a0d9, 0), + RegisterWrite::new(0x1a0e1, 0), + RegisterWrite::new(0x101c1, 0), + RegisterWrite::new(0x0d469, 0), + RegisterWrite::new(0x1a0f9, 8), + // Fragment analog of TA 0x0a5a1. Native derives its high word from the + // 3D AGXUMADataRec PTC/DUPM maximum at host descriptor +0xc00, copied + // through command +0x818. Linux has no owned pool-size input yet, so + // use the exact 0x60 zero-source fallback. Bit 3 retains one captured, + // workload-derived 0xd5 value only as a diagnostic. + RegisterWrite::new( + 0x0a5a9, + if p.native_ta_registers & 8 != 0 { + 0x0000_00d5_0040_0020 + } else { + 0x0000_0060_0040_0020 + }, + ), + RegisterWrite::new(0x0d429, 0x0000_0002_0000_0001), + RegisterWrite::new(0x160e0, p.fragment_lifecycle), + RegisterWrite::new(0x01499, p.fragment_lifecycle), + RegisterWrite::new(0x0a341, p.fragment_lifecycle), + // The fragment must use the same parameter-buffer owner as TA 1c830. + // Native TA::prepare stores its allocated HWPB ID at host +928; the + // same derived host descriptor is submitted through both channels. + // generate3DRegisterList reads that HOST field at 8a5c7cc. Its wrapper + // at 8a515d8 tail-calls with x1=raw command, x2=host descriptor: the + // unused channel argument in the mangled common-function name is NOT + // present in the optimized calling convention. Raw command +928 has + // a different producer and must not be used to infer this register. + RegisterWrite::new(0x1c838, p.ta_hardware_buffer_id as u64), + RegisterWrite::new(0x1ca28, p.cycle), + RegisterWrite::new(0x10211, work_key), + RegisterWrite::new(0x10420, work_key), + // AGX3DCommandDescriptor::prepare receives this ID from the USC pool's + // HardwareBufferIDManager and generate3DRegisterList ORs it with an + // optional GART payload and feature bit. Linux currently has neither + // optional source, so publish the retained USC FList owner's exact ID. + RegisterWrite::new(0x14048, p.usc_flist_hardware_buffer_id as u64), + RegisterWrite::new(0x14080, p.fragment_status | 1), + RegisterWrite::new(0x01731, 1), + RegisterWrite::new(0x16020, 1), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16068, 0x4_0000), + ]) +} + +pub(crate) fn partial_store_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_STORE_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15379, p.partial_store_pipeline_bind), + RegisterWrite::new(0x15381, p.partial_store_pipeline), + RegisterWrite::new(0x10039, p.tile_config), + RegisterWrite::new(0x15359, 0x20), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +pub(crate) fn partial_resume_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_RESUME_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15379, p.partial_store_pipeline_bind), + RegisterWrite::new(0x15381, p.partial_store_pipeline), + RegisterWrite::new(0x15369, p.partial_load_pipeline_bind), + RegisterWrite::new(0x15371, p.partial_load_pipeline), + RegisterWrite::new(0x10039, p.tile_config & 0xffff), + RegisterWrite::new(0x15359, 0x20), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +pub(crate) fn partial_load_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_LOAD_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15369, p.partial_load_pipeline_bind), + RegisterWrite::new(0x15371, p.partial_load_pipeline), + RegisterWrite::new(0x10039, p.tile_config & 0xffff), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +fn put_u8(raw: &mut [u8], offset: usize, value: u8) { + raw[offset] = value; +} + +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn encode_registers(raw: &mut [u8], offset: usize, registers: &[RegisterWrite]) { + for (index, register) in registers.iter().enumerate() { + let entry = offset + index * REGISTER_BYTES; + put_u32(raw, entry, register.number); + put_u64(raw, entry + 4, register.value); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RenderDescriptorKind { + Tiling, + Fragment, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderDescriptorObjects { + pub(crate) record_a: u64, + pub(crate) shared: u64, + pub(crate) record_b: u64, + pub(crate) zero: u64, +} + +impl RenderDescriptorObjects { + +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderDescriptorMetadata { + pub(crate) context_id: u32, + pub(crate) submission_ordinal: u64, + /// Monotonic state tag allocated by the command buffer's 36-entry state + /// ring. This is neither the selected ring index nor any QID/HWID. Native + /// retains the full word in the selected state object and places only its + /// low six bits in the stage work key. + pub(crate) command_buffer_state_word: u32, + /// TA HardwareBufferID copied to firmware descriptor +0x18. + /// + /// Every working native descriptor corpus couples this field to register + /// 0x1c830 exactly (for example 2/2, 4/4, and 5/5). The same allocator + /// result is also PBState +0x0c and the descriptor-ring index consumed by + /// `halLoadPBDesc`. + pub(crate) ta_hardware_buffer_id: u32, + pub(crate) submit_sequence: u64, +} + +impl RenderDescriptorMetadata { +} + +/// Native's TA/3D register-list producer couples the command-buffer state tag +/// to the descriptor's allocated application-GART context. The physical QID, +/// QoS HardwareBufferID, descriptor-array slot and 36-entry state-ring index +/// are independent namespaces and must not be substituted here. +const fn render_work_key(context_id: u32, command_buffer_state_word: u32) -> u64 { + ((context_id as u64) << 8) | ((command_buffer_state_word & 0x3f) as u64) +} + +fn descriptor_work_ordinal(ordinal: u64) -> u32 { + (ordinal as u32).wrapping_add((ordinal / 2) as u32) +} + +fn write_common_descriptor( + kind: RenderDescriptorKind, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + registers: &[RegisterWrite], + raw: &mut [u8], +) { + let (selector, pointers, register_offset, pointer_gap) = match kind { + RenderDescriptorKind::Tiling => (0u32, 0x10usize, 0x60usize, 8usize), + RenderDescriptorKind::Fragment => (1u32, 0x20usize, 0xa0usize, 0usize), + }; + put_u32(raw, 0, selector); + put_u64(raw, 4, metadata.submit_sequence); + put_u32(raw, 0x0c, metadata.context_id); + + let addresses = [ + objects.record_a, + objects.shared, + objects.record_b, + objects.zero, + ]; + let mut offset = pointers; + for (index, address) in addresses.iter().enumerate() { + put_u64(raw, offset, *address); + offset += 8; + if index == 0 { + offset += pointer_gap; + } + } + encode_registers(raw, register_offset, registers); + + let work = descriptor_work_ordinal(metadata.submission_ordinal); + match kind { + RenderDescriptorKind::Tiling => { + put_u32(raw, 0x18, metadata.ta_hardware_buffer_id); + put_u32(raw, 0x48, work); + } + RenderDescriptorKind::Fragment => {} + } +} + +pub(crate) fn build_ta_descriptor_base( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + registers: &[RegisterWrite], + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < TA_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + if registers.is_empty() || registers.len() > (0x660 - 0x60) / REGISTER_BYTES { + return Err(RenderBuildError::BufferTooSmall); + } + raw[..TA_DESCRIPTOR_SIZE].fill(0); + write_common_descriptor( + RenderDescriptorKind::Tiling, + objects, + metadata, + registers, + raw, + ); + + // Like the fragment boundaries, these are generated for every native TA + // command (25G83 88bdfc4/88be090/88be1a4), not only a queue's first kick. + // The final three entries are 01740, 1c880, 1c898; firmware reads +38 and + // +3a/+3c to select the corresponding embedded RCE boundaries. + put_u16(raw, 0x38, (TA_REGISTER_COUNT - 2) as u16); + put_u16(raw, 0x3a, TA_REGISTER_COUNT as u16); + put_u16(raw, 0x3c, TA_REGISTER_COUNT as u16); + + let deflake = ta_user_offset(parameters.deflake_1)?; + let sampler_max = if parameters.sampler_count == 0 { + 0 + } else { + parameters.sampler_count + 1 + }; + put_u32( + raw, + 0x0768, + (((registers.len() * REGISTER_BYTES) as u32) << 16) | registers.len() as u32, + ); + put_u64(raw, 0x0780, parameters.tpc); + // Native copies RTMemory's geometry-derived TPC size beside its pointer + // on every submission (25G83 88fd970..974), independent of queue lifetime. + put_u64(raw, 0x0788, parameters.tpc_size); + put_u32(raw, 0x07d6, deflake as u32); + put_u32(raw, 0x0876, u32::MAX); + put_u64(raw, 0x087a, parameters.sampler_array); + put_u32(raw, 0x0882, parameters.sampler_count); + put_u32(raw, 0x0886, sampler_max); + // Present in 436/438 valid G17P TA records, including both exact 32x32 + // geometry matches. This belongs to the TA output/activation tail; leaving + // it zero is a structural mismatch before the linked 3D job is generated. + put_u32(raw, 0x0892, 1); + put_u8(raw, 0x0932, 0x44); + put_u8(raw, 0x093c, 1); + put_u8(raw, 0x094d, parameters.context_generation); + put_u64(raw, 0x08fe, parameters.ta_timestamp_start); + put_u64(raw, 0x0906, parameters.ta_timestamp_end); + // +0x090e/+0x0916 are caller-visible timestamp destinations. Firmware + // writes them only inside its timestamp aperture, so they stay zero; the + // completion publishes the pass envelope to userspace instead. + // AGXTAChannelSKSM::submitBuffer mirrors the same dependency/current pair + // consumed by the TA RCE program into the firmware command. These fields + // are deliberately unaligned in the native 0x9c0-byte command layout. + put_u32(raw, 0x086e, (parameters.lifecycle >> 32) as u32); + put_u64(raw, 0x08ce, parameters.lifecycle as u32 as u64); + Ok(()) +} + +fn put_embedded_registers( + raw: &mut [u8], + header_offset: usize, + program_offset: usize, + registers: &[RegisterWrite], +) { + let bytes = registers.len() * REGISTER_BYTES; + put_u32( + raw, + header_offset, + ((bytes as u32) << 16) | registers.len() as u32, + ); + encode_registers(raw, program_offset, registers); +} + +pub(crate) struct G17pFragmentRegisters<'a> { + pub(crate) main: &'a [RegisterWrite], + pub(crate) store: &'a [RegisterWrite], + pub(crate) resume: &'a [RegisterWrite], + pub(crate) load: &'a [RegisterWrite], +} + +pub(crate) fn build_fragment_descriptor_base( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + programs: G17pFragmentRegisters<'_>, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < FRAGMENT_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + if programs.main.is_empty() + || programs.main.len() > (0x6a0 - 0xa0) / REGISTER_BYTES + || programs.store.len() > (0xec0 - 0x7c0) / REGISTER_BYTES + || programs.resume.len() > (0x15e0 - 0xee0) / REGISTER_BYTES + || programs.load.len() > (0x1d00 - 0x1600) / REGISTER_BYTES + { + return Err(RenderBuildError::BufferTooSmall); + } + raw[..FRAGMENT_DESCRIPTOR_SIZE].fill(0); + let geometry = geometry(parameters)?; + let register_count = programs.main.len(); + write_common_descriptor( + RenderDescriptorKind::Fragment, + objects, + metadata, + programs.main, + raw, + ); + + put_u64(raw, 0x40, parameters.tilemap); + put_u64(raw, 0x48, parameters.multisample_control); + // Native copies the render target's sample count through host +0x8c8 + // into every fragment descriptor at +0x50 (25G83: 0x8839d80, + // 0x8837af8..0x8837b08). It is not first-work metadata. The UAPI has + // already encoded log2(samples) in UTILE_CONFIG bits [1:0]. + put_u32(raw, 0x50, 1u32 << (parameters.utile_config & 3)); + put_u32(raw, 0x54, geometry.macro_size as u32); + put_u32(raw, 0x68, parameters.merge_upper_x_bits); + put_u32(raw, 0x6c, parameters.merge_upper_y_bits); + put_u64(raw, 0x78, geometry.tiles_x * geometry.tiles_y); + + // RCE boundaries belong to every command, including retained queue kicks. + // Native emits these from the running register count (25G83 88c0740, + // 88c07ac, 88c08c8, 88c0970); firmware consumes +80/+82/+84/+88 when + // selecting an entry in this command's embedded program. The fixed recipe + // ends with 01731=1, 16020=1, 16020=0, 16068=40000. Boundaries are after + // 01731, after the first 16020, that same entry, and the complete program. + // Keeping these only in the cold wrapper left ordinary restages at zero. + put_u16(raw, 0x80, (register_count - 3) as u16); + put_u16(raw, 0x82, (register_count - 2) as u16); + put_u16(raw, 0x84, (register_count - 2) as u16); + put_u16(raw, 0x88, register_count as u16); + + put_u32( + raw, + 0x07a8, + (((register_count * REGISTER_BYTES) as u32) << 16) | register_count as u32, + ); + put_embedded_registers(raw, 0x0ec8, 0x07c0, programs.store); + put_embedded_registers(raw, 0x15e8, 0x0ee0, programs.resume); + put_embedded_registers(raw, 0x1d08, 0x1600, programs.load); + + put_u64(raw, 0x1d20, parameters.depth_bias_array); + put_u64(raw, 0x1d30, parameters.scissor_array); + put_u64(raw, 0x1d40, parameters.occlusion_query_base); + put_u64(raw, 0x1e78, parameters.load_pipeline_bind); + put_u64(raw, 0x1e80, parameters.load_pipeline); + put_u64(raw, 0x1ea8, parameters.partial_load_pipeline_bind); + put_u64(raw, 0x1eb0, parameters.partial_load_pipeline); + put_u32(raw, 0x1ec0, 0x0404_0404); + put_u64(raw, 0x1f38, parameters.tib_blocks); + put_u64(raw, 0x1f40, parameters.aux_fb_flags); + put_u32(raw, 0x1f48, parameters.width); + put_u32(raw, 0x1f4c, parameters.height); + put_u64(raw, 0x1f50, parameters.aux_fb_page_count); + put_u64(raw, 0x1f58, parameters.tile_config); + put_u32(raw, 0x1f78, parameters.store_pipeline_bind as u32); + put_u64(raw, 0x1f7c, parameters.store_pipeline); + put_u32(raw, 0x1f98, parameters.partial_store_pipeline_bind as u32); + put_u64(raw, 0x1f9c, parameters.partial_store_pipeline); + put_u32(raw, 0x1fa8, parameters.depth_clear_value_bits); + put_u64(raw, 0x1fac, (parameters.tib_blocks << 33) | 0x300); + // Native copies isMemorylessRender through the UMD render record +0x240, + // parsed record +0x1b8, and host descriptor +0x8bf into this byte. It is + // not a TIB-size predicate: native's memoryless policy also selects the + // shortened SPM phase programs. The admitted render UAPI represents + // memory-backed resources and has no memoryless-render policy or matching + // SPM-program construction. Keep that policy clear even for >8 TIB + // blocks; supporting memoryless renders requires modeling both together. + // Empty-tile processing remains independent in register 0x10039 bit 16. + put_u8(raw, 0x2100, 0); + // Exact G17P tail copied by AGX3DChannelSKSM::submitBuffer. +0x2104 is the + // accelerator control snapshot and +0x210c its low map nibble. +0x2110 is + // a separate render-setup field: prepare copies descriptor +0xc00 to setup + // +0x818, processRenderSetup carries it to +0x89c, and submitBuffer emits + // it here. Working Metal commands carry the invariant all-ones value. + put_u32(raw, 0x2104, parameters.gpc_perf_state_control); + // AGX3DChannelSKSM::submitBuffer mirrors the same dependency/current pair + // consumed by the fragment RCE program into the firmware command. + put_u32(raw, 0x2108, (parameters.fragment_lifecycle >> 32) as u32); + put_u32(raw, 0x210c, parameters.gpc_perf_state_map_low); + put_u32(raw, 0x2110, G17P_FRAGMENT_TAIL_SETUP_WORD); + // Native submitBuffer zeros +0x2114..+0x2123. Sampler state reaches the + // GPU through its real register/program path, not these command-tail + // lanes. The first two following flag lanes reproduce the working Neo + // shared-runtime snapshot; the final two come from per-command descriptor + // bytes and retain their constructor-zero values until those inputs are + // modeled. + put_u32( + raw, + 0x2124, + G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT[0], + ); + put_u32( + raw, + 0x2128, + G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT[1], + ); + put_u64(raw, 0x2168, parameters.fragment_lifecycle as u32 as u64); + put_u64(raw, 0x2198, parameters.fragment_timestamp_start); + put_u64(raw, 0x21a0, parameters.fragment_timestamp_end); + // +0x21a8/+0x21b0: caller-visible destinations, zero as in the TA record. + put_u8(raw, 0x21cc, 0x53); + put_u8(raw, 0x21d6, 1); + put_u8(raw, 0x21e7, parameters.context_generation); + put_u8(raw, 0x2208, 1); + put_u8(raw, 0x220c, 1); + put_u8(raw, 0x222d, g17p_frg_hwdsid_mode()); + Ok(()) +} + +/// FRG +0x222d mode byte: Linux has always written 1; all 160 native light +/// FRG captures hold 0. +fn g17p_frg_hwdsid_mode() -> u8 { + 1 +} + diff --git a/drivers/gpu/drm/asahi/m5/render_target.rs b/drivers/gpu/drm/asahi/m5/render_target.rs new file mode 100644 index 0000000000000..a277377cb5851 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/render_target.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Render-target allocation geometry and M5 viewport state. + +pub(crate) const PAGE_BYTES: usize = 0x4000; +pub(crate) const AUX_BYTES: usize = 0x8000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct M5RenderTargetLayout { + pub(crate) tilemap_bytes: usize, + pub(crate) heapmeta_offset: usize, + pub(crate) heapmeta_bytes: usize, + pub(crate) target_bytes: usize, + pub(crate) tpc_bytes: usize, +} + +fn align(value: usize, alignment: usize) -> Option { + Some(value.checked_add(alignment - 1)? & !(alignment - 1)) +} + +impl M5RenderTargetLayout { + pub(crate) fn new( + width: u32, height: u32, layers: u16, utile_width: u8, utile_height: u8, + clusters: u32, + ) -> Option { + if !(1..=16384).contains(&width) || !(1..=16384).contains(&height) + || !(1..=2048).contains(&layers) || clusters == 0 + || !matches!((utile_width, utile_height), (32, 32) | (32, 16) | (16, 16)) { + return None; + } + // Same 4x4 macrotile geometry as the register builders. M5 packs its + // boundaries in ten-bit fields; that does not change allocation size. + let tiles_x = (width as usize + 31) / 32; + let tiles_y = (height as usize + 31) / 32; + let per_mtile = align((tiles_x + 3) / 4, 4)? + .checked_mul(align((tiles_y + 3) / 4, 4)?)?; + let utiles = (32 / utile_width as usize) * (32 / utile_height as usize); + let tilemap_bytes = align(5usize.checked_mul(per_mtile)?.checked_mul(utiles)?, 4)? + .checked_mul(16)?.checked_mul(layers as usize)?; + let heapmeta_offset = align(tilemap_bytes, 0x1000)?; + let heapmeta_bytes = if layers > 1 { 0x300 } else { 0x200 }; + let target_bytes = align(heapmeta_offset.checked_add(heapmeta_bytes)?, PAGE_BYTES)?; + let tpc_bytes = align(8usize.checked_mul(per_mtile)?.checked_mul(utiles)? + .checked_mul(16)?.checked_mul(layers as usize)? + .checked_mul(clusters as usize)?, PAGE_BYTES)?; + Some(Self { tilemap_bytes, heapmeta_offset, heapmeta_bytes, target_bytes, tpc_bytes }) + } +} + +fn half_float_bits(value: u32) -> u32 { + // The admitted integer dimensions are exactly representable in binary32. + let highest = 31 - value.leading_zeros(); + ((highest + 126) << 23) | ((value << (23 - highest)) & 0x7fffff) +} + +/// Native M5 deflake page: viewport at +0x9c0, command header 0x3000. +/// G17P uses +0x900/0xc00 and must not supply this page. The eight nonzero +/// dwords match the M5 25G76 resource capture; dimensions are caller-owned. +pub(crate) fn initialize_deflake(page: &mut [u8], width: u32, height: u32) -> Option<()> { + if page.len() != PAGE_BYTES || !(1..=16384).contains(&width) + || !(1..=16384).contains(&height) { return None; } + page.fill(0); + let x = half_float_bits(width); + let y = half_float_bits(height); + for (offset, value) in [(0x9c0, 0x3000), + (0x9c4, 0x80000000 | ((width + 31) / 32 - 1)), + (0x9c8, (height + 31) / 32 - 1), (0x9d0, x), (0x9d4, x), + (0x9d8, y), (0x9dc, y | 0x80000000), (0x9e4, 0x3f800000)] { + page[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + Some(()) +} + diff --git a/drivers/gpu/drm/asahi/m5/resources.rs b/drivers/gpu/drm/asahi/m5/resources.rs new file mode 100644 index 0000000000000..30d94f4d97148 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources.rs @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![allow(dead_code)] + +//! Retained RAM for the J813 G17G/25G76 initialization graph. +//! +//! Construction only: no processor start, mailbox publication, work submission +//! or DRM registration. The eventual boot session must retain this entire +//! owner until BOTH firmware processors are stopped, including on failure. +//! M5 UAT lifecycle admission remains separate from this memory layout. + +use super::mapped::MappedObject; +use crate::{driver::AsahiDrmDevice, m5::initdata as init, mmu}; +use crate::m5::completion::{graphics_pointer_head, MAX_GRAPHICS_GENERATION}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; +use kernel::sync::Arc; + +mod firmware_view; +mod graphics_memory; +mod graphics_usc; +mod parameter_buffer; +mod target_memory; +mod graphics_job; +mod graphics_queues; +mod application_compute; + +const KERNEL_BASE: u64 = 0xffff_fc20_0000_0000; +const PRIVATE_BASE: u64 = KERNEL_BASE + 0x2_0000; +const ROOTS_BASE: u64 = KERNEL_BASE + 0x1a_8000; +const SHARED_BASE: u64 = KERNEL_BASE + 0xc078_8000; +const FWCTL_BASE: u64 = KERNEL_BASE + 0x1c_0000; +const SENTINEL_BASE: u64 = KERNEL_BASE + 0xc07e_0000; +const PBDESC_BASE: u64 = KERNEL_BASE + 0x15e_0000; +const UMA_BASE: u64 = KERNEL_BASE + 0x15e_8000; + +/// Owning mappings are dropped before the UAT. Aliases retain the SAME GEM +/// backing; no firmware pointer refers to a copied snapshot or temporary map. +pub(crate) struct M5FirmwareMemory { + primary_root: MappedObject, + secondary_root: MappedObject, + shared: MappedObject, + private: MappedObject, + region_a: MappedObject, + region_c: MappedObject, + fwctl: MappedObject, + aon: MappedObject, + qos: MappedObject, + pbdesc: MappedObject, + uma: MappedObject, + completions: KVec, + aliases: KVec, + registers: KVec, + graphics: Option<(graphics_job::M5GraphicsJob, graphics_queues::M5GraphicsQueues)>, + application_compute: Option, + application_outer_head: u32, + report_counts: [[u32; 2]; 2], + trace_printed: [u32; 2], + compute_kick_observed: bool, + compute_dispatch_returned: bool, + graphics_stamp_flags: u128, + uat: Arc, +} + +impl M5FirmwareMemory { + /// Internal defaults for the initial M5 render-state envelope. The DRM + /// translator replaces all program/input addresses, and owned target + /// allocation replaces private storage before any queue publication. + pub(crate) fn reference_render_profile() -> + (crate::m5::render_desc::G17pRenderParameters, crate::m5::render::M5RenderControls) { + graphics_job::triangle_parameters() + } + + /// All fallible preparation happens before publication. The caller must + /// pass a prepared M5 UAT while both processors await initdata. The + /// caller retains its own Arc until both CPUs stop, including when this + /// unpublished graph fails construction. There is no Neo tuning fallback. + #[inline(never)] + pub(crate) fn new_with_uat( + dev: &AsahiDrmDevice, + uat: Arc, + boot_timestamp: u64, + ) -> Result> { + let memory = Self::allocate_with_uat(dev, uat, boot_timestamp)?; + fence(Ordering::SeqCst); + Ok(memory) + } + + #[inline(never)] + fn allocate_with_uat( + dev: &AsahiDrmDevice, + uat: Arc, + boot_timestamp: u64, + ) -> Result> { + let mut registers = KVec::new(); + for window in init::M5_25G76_REGISTER_WINDOWS { + let (physical, virtual_address, span) = window.mapped_pages().map_err(|_| EINVAL)?; + let protection = match window.flag { + 0 => mmu::PROT_FW_MMIO_RO, + 2 => mmu::PROT_FW_MMIO_RW, + _ => return Err(EINVAL), + }; + // Page-table writes only. No register contents are accessed. + registers.push(uat.kernel_vm().map_io(dev.as_ref().as_ref(), virtual_address, physical.try_into()?, + span.try_into()?, protection)?, GFP_KERNEL)?; + } + let fixed = |address, size, protection| MappedObject::new_wc_at( + dev, uat.kernel_vm(), address, size, protection); + // Install every fixed address before allocator-selected objects. + let mut private = fixed(PRIVATE_BASE, init::M5_25G76_PRIVATE_CLUSTER_SIZE, mmu::PROT_FW_SHARED_RW)?; + // Separate mappings retain the empty page between the two roots. + let mut primary_root = fixed(ROOTS_BASE, init::ROOT_SIZE_PRIMARY, mmu::PROT_FW_SHARED_RO)?; + let mut secondary_root = fixed(ROOTS_BASE + init::SECONDARY_ROOT_DELTA, + init::ROOT_SIZE_SECONDARY, mmu::PROT_FW_SHARED_RO)?; + let mut shared = fixed(SHARED_BASE, init::NATIVE_SHARED_CLUSTER_SIZE, mmu::PROT_FW_PRIV_RW)?; + let region_a = fixed(KERNEL_BASE + 0x1b_8000, init::REGION_A_SIZE, mmu::PROT_FW_SHARED_RO)?; + let mut region_c = fixed(KERNEL_BASE + 0x1a_0000, init::REGION_C_SIZE, mmu::PROT_FW_SHARED_RO)?; + let fwctl = fixed(FWCTL_BASE, 0x4000, mmu::PROT_FW_SHARED_RW)?; + // These two tables have zero-backed mappings at handoff. Mode6 + // prepares its own UMA slot before any compute publication. + let mut pbdesc = fixed(PBDESC_BASE, 0x4000, mmu::PROT_FW_SHARED_RW)?; + let mut uma = fixed(UMA_BASE, 0x4000, mmu::PROT_FW_SHARED_RW)?; + let mut aliases = KVec::new(); + for (object, address) in [(&mut pbdesc, init::REGION_VIEW_LOW_ADDRS[0]), + (&mut uma, init::REGION_VIEW_LOW_ADDRS[1])] { + aliases.push(object.map_alias_at(uat.kernel_lower_vm(), address, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + if uat.kernel_vm().translate_iova(object.iova())? + != uat.kernel_lower_vm().translate_iova(address)? { + return Err(EFAULT); + } + } + + let mut completions = KVec::new(); + let mut completion_bindings = [init::G17PKsmCompletionAliases { low: 0, high: 0 }; 4]; + for (ordinal, binding) in completion_bindings.iter_mut().enumerate() { + // The fixed alias pairs are measured on M5, including ordinals + // 1 and 3 which are absent from the older Neo initialization. + let low = 0x70_0184_8000 + ordinal as u64 * 0x2_8000; + let high = KERNEL_BASE + 0xc07e_8000 + ordinal as u64 * 0x2_8000; + let mut backing = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, + init::M5_25G76_COMPLETION_BACKING_SIZE, mmu::PROT_FW_PRIV_RW)?; + aliases.push(backing.map_alias_at(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..init::M5_25G76_COMPLETION_BACKING_SIZE).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset as u64)? + != uat.kernel_lower_vm().translate_iova(low + offset as u64)? { + return Err(EFAULT); + } + } + *binding = init::G17PKsmCompletionAliases { low, high }; + completions.push(backing, GFP_KERNEL)?; + } + // Keep allocator-selected objects below the high fixed graph, and + // above the PB/UMA tables. This cannot occupy the deferred blank + // sentinel address or its neighboring guard page. + let dynamic = KERNEL_BASE + 0x168_0000..KERNEL_BASE + 0x1000_0000; + let qos = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), dynamic.clone(), + 0xe40, mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW)?; + let mut aon = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), dynamic, + init::M5_25G76_AON_PTD_SIZE, + mmu::UAT_PGSZ as u64, mmu::PROT_FW_PRIV_RW)?; + aon.with_bytes_mut(|out| init::encode_m5_25g76_aon_ptd(out).map_err(|_| EINVAL))?; + region_c.with_bytes_mut(|out| init::encode_region_c_for_profile( + init::RootProfile::J813G17G25G76, out).map_err(|_| EINVAL))?; + private.with_bytes_mut(|out| init::encode_m5_25g76_private_cluster( + FWCTL_BASE, out).map_err(|_| EINVAL))?; + + let addresses = init::SharedAddresses { + region_a: region_a.iova(), region_c: region_c.iova(), + hw_data_bundle: SHARED_BASE, hw_data_bundle_alloc: init::NATIVE_SHARED_CLUSTER_SIZE, + }; + let primary = init::InstanceAddresses { + root: ROOTS_BASE, main_config: SHARED_BASE + init::NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET as u64, + status_a: PRIVATE_BASE + 0xee40, status_b: PRIVATE_BASE + 0x340, + secondary_extras: [0, 0], + }; + let secondary = init::InstanceAddresses { + root: ROOTS_BASE + init::SECONDARY_ROOT_DELTA, + main_config: SHARED_BASE + init::NATIVE_SECONDARY_MAIN_BUNDLE_OFFSET as u64, + status_a: PRIVATE_BASE + 0xc3100, status_b: 0, + secondary_extras: [primary.status_b + init::PRIMARY_STATUS_B_CONFIG_OFFSET as u64, + PRIVATE_BASE + 0x177400], + }; + let views = init::PrimaryRegionViews { + sentinel_high: SENTINEL_BASE, alias_high: [PBDESC_BASE, UMA_BASE], + }; + let trace_mask = crate::module_parameters::m5_fw_trace.value(); + if trace_mask & !0x203 != 0 { return Err(EINVAL); } + // Exact M5 GFX firmware 0x1fadc retains root+0xa8 (status A), + // then 0x3074..0x3088 copies status-A+0 into its trace flags. + // Bits 0 and 1 gate power/submission records; 0x6260 reads the + // NEXT byte and tests bit 1 (full-word bit 9) for recovery summary + // 0x909. Publish through the host field before either descriptor. + let native_trace = private.read_u32(primary.status_a)?; + private.write_u32(primary.status_a, native_trace | trace_mask)?; + shared.with_bytes_mut(|out| { + init::encode_m5_25g76_hw_data(completion_bindings, qos.iova(), + &mut out[..init::M5_25G76_HW_DATA_SIZE]).map_err(|_| EINVAL)?; + init::encode_m5_25g76_bundle_policy(init::M5PolicyBindings { + state: PRIVATE_BASE + 0x177480, aon_ptd: aon.iova(), boot_timestamp, + }, out).map_err(|_| EINVAL)?; + for (role, instance, grid, views) in [ + (init::InstanceRole::Primary, &primary, PRIVATE_BASE, Some(&views)), + (init::InstanceRole::Secondary, &secondary, PRIVATE_BASE + 0x1770c0, None), + ] { + let channels = init::derive_channel_table(role, grid, instance.status_a, + instance.main_config, SHARED_BASE); + let offset = (instance.main_config - SHARED_BASE) as usize; + init::encode_main_config_for_profile(init::RootProfile::J813G17G25G76, + role, &addresses, instance, &channels, views, + &mut out[offset..offset + init::MAIN_CONFIG_SIZE]).map_err(|_| EINVAL)?; + } + Ok(()) + })?; + primary_root.with_bytes_mut(|out| { + init::encode_root_for_profile(init::RootProfile::J813G17G25G76, + init::InstanceRole::Primary, &addresses, &primary, + out).map_err(|_| EINVAL) + })?; + secondary_root.with_bytes_mut(|out| { + init::encode_root_for_profile(init::RootProfile::J813G17G25G76, + init::InstanceRole::Secondary, &addresses, &secondary, + out).map_err(|_| EINVAL) + })?; + // Unlike PB/UMA, this future blank sentinel is unmapped at native + // handoff. It becomes zero-backed later in macOS initialization. + if uat.kernel_vm().translate_iova(SENTINEL_BASE).is_ok() { + return Err(EINVAL); + } + // Writes to all CPU WC mappings precede any future handoff. Only + // the two measured control openings are present; work rings and + // later compute/render descriptors remain unsubmitted. + fence(Ordering::SeqCst); + // Keep the retained graph off the kernel stack, including + // construction and error-return temporaries in the boot caller. + KBox::init(try_init!(Self { primary_root, secondary_root, shared, private, region_a, region_c, fwctl, aon, qos, + pbdesc, uma, completions, aliases, registers, graphics: None, + report_counts: [[0; 2]; 2], trace_printed: [0; 2], + compute_kick_observed: false, compute_dispatch_returned: false, + graphics_stamp_flags: 0, application_compute: None, + application_outer_head: 0, uat }), GFP_KERNEL) + } + + /// Prepare a first application job while retaining its actual VM. No + /// native program bundle or captured input/output pages are requested. + /// The caller has validated the command and all user mapping coverage. + #[allow(clippy::too_many_arguments)] + pub(crate) fn initialize_application_graphics( + &mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm, + parameters: crate::m5::render_desc::G17pRenderParameters, + controls: crate::m5::render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], + ) -> Result { + if self.graphics.is_some() { return Err(EBUSY); } + let job = graphics_job::M5GraphicsJob::from_vm( + dev, &self.uat, vm, parameters, controls, clusters, usc_required)?; + self.attach_graphics(dev, job) + } + + fn attach_graphics(&mut self, dev: &AsahiDrmDevice, job: graphics_job::M5GraphicsJob) -> Result { + if self.graphics.is_some() { return Err(EBUSY); } + let queues = graphics_queues::M5GraphicsQueues::new(dev, &self.uat, &job)?; + let pb = job.parameter_descriptor()?; + self.pbdesc.with_bytes_mut(|out| { + let base = graphics_job::PB_HARDWARE_ID as usize * 16; + for (index, word) in pb.into_iter().enumerate() { + out[base + index * 4..base + index * 4 + 4].copy_from_slice(&word.to_le_bytes()); + } + Ok(()) + })?; + let usc = job.uma_descriptor(); + self.uma.with_bytes_mut(|out| { + let base = graphics_usc::HARDWARE_ID as usize * 32; + for (index, word) in usc.into_iter().enumerate() { + out[base + index * 8..base + index * 8 + 8].copy_from_slice(&word.to_le_bytes()); + } + Ok(()) + })?; + self.graphics = Some((job, queues)); + Ok(()) + } + + pub(crate) fn quarantine_graphics(&self) { + if let Some((job, _)) = &self.graphics { job.quarantine(); } + } + + /// Drain M5 report channels using the native producer contracts. + /// GFX 0x6ca8..0x6d18 emits 256 x 0x48 event records; 0x17e18 + /// emits 512 x 0x48 trace records. In each cursor object the host owns + /// +0 and firmware owns +0x20. Copy before releasing each slot. + pub(crate) fn drain_reports(&mut self, dev: &AsahiDrmDevice) -> Result { + let mut recovery = false; + for (role, status_offset) in [0xee40u64, 0xc3100].into_iter().enumerate() { + for (channel, (cursor_offset, ring_offset, capacity)) in + [(0x40u64, 0x2c0u64, 256u32), (0x240, 0xa6ac0, 512)] + .into_iter().enumerate() { + let cursor = PRIVATE_BASE + status_offset + cursor_offset; + let ring = (status_offset + ring_offset) as usize; + for _ in 0..128 { + let consumer = self.private.read_u32(cursor)?; + let producer = self.private.read_u32(cursor + 0x20)?; + if consumer >= capacity || producer >= capacity { return Err(EIO); } + if consumer == producer { break; } + fence(Ordering::Acquire); + let offset = ring + consumer as usize * 0x48; + let mut raw = [0u8; 0x48]; + self.private.with_bytes_mut(|bytes| { + raw.copy_from_slice(bytes.get(offset..offset + 0x48).ok_or(EINVAL)?); + Ok(()) + })?; + let word = |off: usize| u32::from_le_bytes(raw[off..off + 4].try_into().unwrap()); + let qword = |off: usize| u64::from_le_bytes(raw[off..off + 8].try_into().unwrap()); + let count = self.report_counts[role][channel]; + if channel == 0 { + // Type 1 (graphics stamps) arrives for every job. + if word(0) == 1 { + dev_dbg!(dev.as_ref(), "M5_EVENT role={} ordinal={} type={} payload={:x?}\n", + role, count, word(0), [qword(4), qword(12), qword(20), qword(28), qword(36)]); + } else { + dev_dbg!(dev.as_ref(), "M5_EVENT role={} ordinal={} type={} payload={:x?}\n", + role, count, word(0), [qword(4), qword(12), qword(20), qword(28), qword(36)]); + } + recovery |= word(0) == 4; + if role == 0 && word(0) == 1 { + self.graphics_stamp_flags |= u128::from(qword(4)) + | (u128::from(qword(12)) << 64); + } + } else { + let args = [qword(12), qword(20), qword(28), qword(36)]; + // This one-shot diagnostic owns qid11/stamp1. The + // exact M5 trace point precedes the hardware kick; + // the admission probe also requires its later callback + // return trace. Firmware-private counters can be cached. + if role == 0 && word(0) == 5 && word(44) == 0x41 + && args == [11, 1, 2, 1] { + self.compute_kick_observed = true; + } + // 0x26aa4 invokes the pending-work callback, which + // dispatches tag14 and writes SGX+0x21018 at0x11f5c. + // 0x26ac4 emits 0x130 only after that callback returns. + if role == 0 && self.compute_kick_observed && word(0) == 5 + && word(44) == 0x130 && args == [4, 0, 0, 0] { + self.compute_dispatch_returned = true; + } + // Exact M5 0x24eb4/0x26754 emit zero-argument + // processing-loop brackets. Count and consume them + // without spending the diagnostic print budget. + let loop_marker = word(0) == 5 && args == [0; 4] + && matches!(word(44) & 0x00ff_ffff, 0x105 | 0x106); + if !loop_marker && self.trace_printed[role] < if role == 0 { 512 } else { 16 } { + dev_dbg!(dev.as_ref(), "M5_TRACE role={} ordinal={} timestamp={} type={} id={:#x} args={:x?}\n", + role, count, qword(4), word(0), word(44), args); + self.trace_printed[role] += 1; + } + } + self.report_counts[role][channel] = count.saturating_add(1); + fence(Ordering::SeqCst); + self.private.write_u32(cursor, (consumer + 1) % capacity)?; + } + } + } + Ok(recovery) + } + + pub(crate) fn graphics_status(&mut self) -> Result<([u32; 3], [[u32; 3]; 2], [u32; 5])> { + let mut outer = [0; 3]; + let base = init::WORK_STATE_GRID_OFFSETS[8]; + for index in 0..3 { + outer[index] = self.private.read_u32(PRIVATE_BASE + + (base + index * init::CHANNEL_STATE_SPACING) as u64)?; + } + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + Ok((outer, queues.counters()?, job.progress()?)) + } + + pub(crate) fn graphics_completion(&mut self) -> Result { + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + let progress = job.progress()?; + let scheduler = job.scheduler_status()?; + Ok(crate::m5::completion::M5GraphicsCompletion { + submitted: progress[0], completed: scheduler[0], linked: scheduler[1], + list: queues.completion_list()?, stamps: [progress[1], progress[3]], + signalled: self.graphics_stamp_flags, + }) + } + + pub(crate) fn graphics_pair_complete( + completion: crate::m5::completion::M5GraphicsCompletion, + generation: u32, + ) -> bool { + (1..=MAX_GRAPHICS_GENERATION).contains(&generation) + && completion.pair_complete_at(graphics_queues::JOB_LIST, generation * 0x100) + } + + pub(crate) fn graphics_installation_status(&mut self) -> Result<([[u64; 5]; 2], [u64; 2], [u64; 2])> { + self.graphics.as_mut().ok_or(EINVAL)?.1.installation_status() + } + + pub(crate) fn graphics_queues_installed(&mut self) -> Result { + self.graphics.as_mut().ok_or(EINVAL)?.1.installed() + } + + pub(crate) fn publish_graphics_install(&mut self) -> Result { + let (outer, inner, progress) = self.graphics_status()?; + if outer != [0; 3] || inner != [[0, 0, 1]; 2] + || progress != [2, 0, 0, 0, 0] { return Err(EBUSY); } + if self.graphics_installation_status()? != ([[0; 5]; 2], [0; 2], [0xffff, 0]) { + return Err(EBUSY); + } + self.publish_graphics_records(0, 1) + } + + pub(crate) fn publish_graphics_job(&mut self) -> Result { + let (outer, inner, progress) = self.graphics_status()?; + if outer != [2; 3] || inner != [[1; 3]; 2] + || progress != [2, 0, 0, 0, 0] { return Err(EBUSY); } + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + if !queues.installed()? { return Err(EBUSY); } + self.graphics_stamp_flags = 0; + job.publish_context()?; + queues.publish_draw()?; + self.publish_graphics_records(2, 4) + } + + /// Reuse only a fully retired pair, replacing its application descriptors + /// while preserving firmware-owned PB/USC state and the installed queues. + #[allow(clippy::too_many_arguments)] + /// A different submitting VM takes over the retired job in place: same + /// context slot, descriptor graph, queues and PB/USC pools. The compute + /// owner shares that context, so its operands follow the job. + fn hand_off_application(&mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let (job, _) = self.graphics.as_mut().ok_or(EINVAL)?; + if job.owns_vm(vm) { return Ok(()); } + job.rebind_vm(dev, vm).inspect_err(|e| pr_err!("M5: rebind_vm failed: {:?}\n", e))?; + job.publish_context().inspect_err(|e| pr_err!("M5: publish_context failed: {:?}\n", e))?; + Ok(()) + } + + pub(crate) fn publish_next_application_job( + &mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm, + parameters: crate::m5::render_desc::G17pRenderParameters, + controls: crate::m5::render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], generation: u32, + ) -> Result { + if !(2..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + let previous = generation - 1; + let (outer, inner, _) = self.graphics_status()?; + if outer != [self.application_outer_head; 3] + || inner != [[graphics_pointer_head(previous); 3]; 2] + || !Self::graphics_pair_complete(self.graphics_completion()?, previous) { + return Err(EBUSY); + } + self.hand_off_application(dev, vm)?; + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + job.prepare_application_frame(dev, vm, parameters, controls, clusters, + usc_required, generation).inspect_err(|e| pr_err!("M5: prepare_application_frame failed: {:?}\n", e))?; + queues.publish_next_draw(job, generation).inspect_err(|e| pr_err!("M5: publish_next_draw failed: {:?}\n", e))?; + self.graphics_stamp_flags = 0; + self.publish_graphics_records(self.application_outer_head as usize, + graphics_pointer_head(generation) as u16) + } + + fn publish_graphics_records(&mut self, outer_index: usize, inner_head: u16) -> Result { + // Only recycle storage after every outer consumer reaches the old + // producer. The application caller also requires paired retirement. + if outer_index >= 256 || self.graphics_status()?.0 != [outer_index as u32; 3] { + return Err(EBUSY); + } + self.shared.with_bytes_mut(|out| { + for stage in 0..2 { + let offset = init::WORK_RING_BUNDLE_OFFSETS[8] + ((outer_index + stage) % 256) * 24; + let record = &mut out[offset..offset + 24]; + record.fill(0); + let address = graphics_queues::M5GraphicsQueues::queue_address(stage)?; + record[8..16].copy_from_slice(&address.to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&inner_head.to_le_bytes()); + record[22] = graphics_job::QUEUE_IDS[stage]; + record[23] = 1; + } + Ok(()) + })?; + fence(Ordering::SeqCst); + // Both stage-tagged SKSM queues travel through the CL_2 command ring. + let producer = init::WORK_STATE_GRID_OFFSETS[8] + 2 * init::CHANNEL_STATE_SPACING; + self.private.write_u32(PRIVATE_BASE + producer as u64, ((outer_index + 2) % 256) as u32)?; + self.application_outer_head = ((outer_index + 2) % 256) as u32; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn application_outer_head(&self) -> u32 { self.application_outer_head } + + /// Initial admission is deliberately one compute job after retired graphics. + /// No private diagnostic context, queue 11, or replacement UMA is installed. + pub(crate) fn initialize_application_compute(&mut self, dev: &AsahiDrmDevice, + vm: &mmu::Vm, command: &crate::m5::command::TranslatedComputeCommand, + graphics_generation: u32) -> Result { + if self.application_compute.is_some() || self.application_outer_head + 2 > 256 { + return Err(ENOTSUPP); + } + let (outer, inner, _) = self.graphics_status()?; + if graphics_generation == 0 || outer != [self.application_outer_head; 3] + || inner != [[graphics_pointer_head(graphics_generation); 3]; 2] + || !Self::graphics_pair_complete(self.graphics_completion()?, graphics_generation) { + return Err(EBUSY); + } + self.hand_off_application(dev, vm)?; + let job = &self.graphics.as_ref().ok_or(EINVAL)?.0; + self.application_compute = Some(application_compute::M5ApplicationCompute::new( + dev, &self.uat, job, command)?); + Ok(()) + } + + pub(crate) fn application_compute_installed(&mut self) -> Result { + let compute = self.application_compute.as_mut().ok_or(EINVAL)?; + Ok(compute.counters()? == [1; 3] && compute.installed()?) + } + + /// Publish one command-ring record for queue 12. `dispatch` false is the + /// installation record; true is dispatch generation `generation`, whose + /// inner head is the producer that generation's three pointers reach. + pub(crate) fn publish_application_compute(&mut self, dispatch: bool, + generation: u32) -> Result { + let outer_index = self.application_outer_head as usize; + if outer_index >= 256 || self.graphics_status()?.0 != [outer_index as u32; 3] { + return Err(EBUSY); + } + let inner_head = u16::try_from(application_compute::compute_pointer_head( + if dispatch { generation } else { 0 })).map_err(|_| EINVAL)?; + let compute = self.application_compute.as_mut().ok_or(EINVAL)?; + if dispatch { + // Preserve the retired graphics notifications for later render reuse. + self.graphics_stamp_flags &= !(1u128 << application_compute::QUEUE_ID); + compute.publish_dispatch(generation)?; + } else if compute.counters()? != [0, 0, 1] { return Err(EBUSY); } + let offset = init::WORK_RING_BUNDLE_OFFSETS[8] + outer_index * 24; + self.shared.with_bytes_mut(|out| { + let record = &mut out[offset..offset + 24]; + // Every outer consumer reached this slot (checked above), so its + // prior user, graphics or compute, has retired. Graphics records + // wrap this ring after 128 renders, so even the first compute + // record can land on a used slot. + record.fill(0); + record[8..16].copy_from_slice( + &application_compute::M5ApplicationCompute::queue_address().to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&inner_head.to_le_bytes()); + record[22] = application_compute::QUEUE_ID; + record[23] = 1; + Ok(()) + })?; + fence(Ordering::SeqCst); + let head = ((outer_index + 1) % 256) as u32; + let producer = init::WORK_STATE_GRID_OFFSETS[8] + 2 * init::CHANNEL_STATE_SPACING; + self.private.write_u32(PRIVATE_BASE + producer as u64, head)?; + self.application_outer_head = head; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Stage a further dispatch on the already installed compute queue. The + /// installed owner keeps its VM, USC and operand storage; only the command + /// stream and the generation's stamps change. + pub(crate) fn prepare_next_application_compute(&mut self, dev: &AsahiDrmDevice, + vm: &mmu::Vm, command: &crate::m5::command::TranslatedComputeCommand, + generation: u32) -> Result { + if !(2..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + self.hand_off_application(dev, vm)?; + let (job, _) = self.graphics.as_ref().ok_or(EINVAL)?; + self.application_compute.as_mut().ok_or(EINVAL)? + .prepare_next_dispatch(job, command, generation) + } + + pub(crate) fn application_compute_complete(&mut self, generation: u32) -> Result { + let (outer, _, _) = self.graphics_status()?; + Ok(outer == [self.application_outer_head; 3] + && self.application_compute.as_mut().ok_or(EINVAL)? + .complete(self.graphics_stamp_flags, generation)?) + } + + pub(crate) fn application_compute_status(&mut self) + -> Result<([u32; 3], crate::m5::completion::M5ComputeCompletion)> { + let compute = self.application_compute.as_mut().ok_or(EINVAL)?; + Ok((compute.counters()?, compute.completion(self.graphics_stamp_flags)?)) + } + + pub(crate) fn publish_idle_wake(&mut self) -> Result { + if self.control_counters()? != [[1; 3]; 2] { return Err(EBUSY); } + self.shared.with_bytes_mut(|out| { + let record = &mut out[0x1eac0..0x1eb00]; + if record.iter().any(|x| *x != 0) { return Err(EBUSY); } + record[..4].copy_from_slice(&0x0au32.to_le_bytes()); + Ok(()) + })?; + fence(Ordering::SeqCst); + let producer = init::WORK_STATE_GRID_OFFSETS[init::CONTROL_CHANNEL_INDEX] + + 2 * init::CHANNEL_STATE_SPACING; + self.private.write_u32(PRIVATE_BASE + producer as u64, 2)?; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn root(&self, secondary: bool) -> u64 { + if secondary { self.secondary_root.iova() } else { self.primary_root.iova() } + } + + /// Read the actual shared control state, not a detached host copy. The + /// offsets and three pointers per role are checked by the M5 capture + /// verifier against both native main objects and the private arena. + pub(crate) fn control_counters(&mut self) -> Result<[[u32; 3]; 2]> { + self.private.with_bytes_mut(|bytes| { + let mut counters = [[0; 3]; 2]; + for (role, grid) in [0usize, 0x1770c0].into_iter().enumerate() { + let base = grid + init::WORK_STATE_GRID_OFFSETS[init::CONTROL_CHANNEL_INDEX]; + for (index, value) in counters[role].iter_mut().enumerate() { + let offset = base + index * init::CHANNEL_STATE_SPACING; + let word = bytes.get(offset..offset + 4).ok_or(EINVAL)?; + // SAFETY: arena and native counter offsets are u32 aligned; + // this bounded WC mapping is retained while firmware runs. + *value = unsafe { core::ptr::read_volatile(word.as_ptr().cast::()) }; + } + } + fence(Ordering::Acquire); + Ok(counters) + }) + } + +} diff --git a/drivers/gpu/drm/asahi/m5/resources/application_compute.rs b/drivers/gpu/drm/asahi/m5/resources/application_compute.rs new file mode 100644 index 0000000000000..4f84edbd0875b --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/application_compute.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! One application compute dispatch sharing the retained graphics VM and USC. +//! +//! Queue 12 is separate from TA/fragment queues 10/11. All publication and +//! completion gates are explicit; constructing this owner does not submit it. + +use super::{graphics_job::M5GraphicsJob, graphics_usc, + firmware_view::FirmwareDataView, MappedObject}; +use crate::{driver::AsahiDrmDevice, m5::submission, m5::command, + m5::channel, m5::completion::{graphics_pointer_head, GRAPHICS_POINTER_COUNT, + MAX_GRAPHICS_GENERATION}, m5::compute, m5::sksm, mmu}; +use kernel::prelude::*; + +pub(super) const QUEUE_ID: u8 = 12; +const GRAPH: u64 = 0xffff_fc20_c008_0000; +const POINTERS: usize = 0x4000; +const ITEMS: usize = 0x8000; +/// Past the 0x500 command pointers the channel state declares, as for the +/// graphics queues, so the producer head wraps like theirs. +const CONFIG: usize = ITEMS + GRAPHICS_POINTER_COUNT as usize * 8; +const CONTROL: usize = 0xc000; +const _: () = assert!(CONFIG + 0x300 <= CONTROL); + +/// Installation publishes one command pointer; every dispatch generation then +/// appends the same three -- descriptor, channel config, kick -- so the +/// producer head runs 1, 4, 7, ... (mod 0x500) exactly as +/// graphics_pointer_head() does for queues 10/11. +pub(super) const fn compute_pointer_head(generation: u32) -> u32 { + graphics_pointer_head(generation) +} + +pub(super) const fn compute_generation_admitted(generation: u32) -> bool { + generation >= 1 && generation <= MAX_GRAPHICS_GENERATION +} + +const LIST: u64 = GRAPH + 0x10000; +const ENTRY: u64 = 0xffff_fc20_003d_0000; +const ENTRY_LOW: u64 = 0x70_0063_0000; +const DESC: u64 = 0xffff_fc20_c040_0000; +const DESC_LOW: u64 = 0x70_0038_0000; +const SCHEDULER: u64 = DESC + 0x4000; +const ZERO: u64 = DESC + 0x8000; +const SIGNALS: u64 = DESC + 0x10000; + +pub(super) struct M5ApplicationCompute { + aliases: KVec, + descriptor: MappedObject, + graph: MappedObject, + entry: MappedObject, + list: MappedObject, + signals: MappedObject, + /// GPU address of the job's compute storage. + storage: u64, + /// The operands mapped into the VM that took the graphics job over. + firmware: FirmwareDataView, +} + +impl M5ApplicationCompute { + pub(super) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, job: &M5GraphicsJob, + command: &command::TranslatedComputeCommand) -> Result { + // The adapter excludes helpers, timestamps, legacy sampler registers + // and the Neo add3 proof ABI before allocating or publishing anything. + if command.sampler_heap != 0 || command.sampler_count != 0 + || command.g17p_add3_buffers.is_some() { return Err(ENOTSUPP); } + // Validate observation support and vacancy before publishing a queue. + let firmware = FirmwareDataView::new(dev)?; + if firmware.queue_registration(QUEUE_ID)? != ([0; 5], 0) { return Err(EBUSY); } + // 128 KiB operands, a preemption page and a zero robustness page. + // Their addresses come from the application's reserved driver arena. + let storage = job.compute_storage(); + let mut descriptor = MappedObject::new_wc_at(dev, uat.kernel_vm(), DESC, + 0x10000, mmu::PROT_FW_PRIV_RW)?; + let mut entry = MappedObject::new_wc_at(dev, uat.kernel_vm(), ENTRY, + 0x20000, mmu::PROT_FW_PRIV_RW)?; + let mut aliases = KVec::new(); + for (object, low, size) in [(&mut descriptor, DESC_LOW, 0x10000), + (&mut entry, ENTRY_LOW, 0x20000)] { + aliases.push(object.map_alias_at(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(object.iova() + offset as u64)? + != uat.kernel_lower_vm().translate_iova(low + offset as u64)? { + return Err(EFAULT); + } + } + } + let mut signals = MappedObject::new_wc_at(dev, uat.kernel_vm(), SIGNALS, + 0x4000, mmu::PROT_FW_SHARED_RW)?; + // The submitted count is the host's and stays put; firmware republishes + // the completed count into the scheduler record each generation. + signals.write_u32(SIGNALS, 1)?; + Self::encode_descriptor(&mut descriptor, job, command, storage, 1)?; + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), GRAPH, + 0x10000, mmu::PROT_FW_PRIV_RW)?; + let mut list = MappedObject::new_wc_at(dev, uat.kernel_vm(), LIST, + 0x4000, mmu::PROT_FW_SHARED_RW)?; + list.with_bytes_mut(|out| { put64(out, 8, LIST); Ok(()) })?; + let prepared = Self::prepare_entry(1)?; + entry.with_bytes_mut(|out| { + let offset = prepared.offset as usize; + out[offset..offset + prepared.bytes.len()].copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let configuration = |install| channel::encode_m5_channel_config( + channel::M5ChannelConfig { + ring_low: ENTRY_LOW, ring_high: ENTRY, queue_id: QUEUE_ID, priority: 2, + install_queue: install, gpu_context: job.context_id(), + event: if install { None } else { Some(channel::M5ChannelEvent { + address: job.usc_control(), timestamp: 1, index: graphics_usc::HARDWARE_ID, + }) }, + channel_control: GRAPH + CONTROL as u64, update_scheduler: install, + scheduler_index: 0, channel_stamp_index: 0, + update_context: !install, owner_context: if install { 0 } else { 1 }, + extended_command: false, + }).map_err(|_| EINVAL); + let install = configuration(true)?; + let config = configuration(false)?; + let kick = Self::prepare_kick(1)?; + graph.with_bytes_mut(|out| { + let reset = submission::encode_m5_25g76_channel_state_reset( + submission::M5ChannelStateResetValues { + uncached_state: GRAPH + POINTERS as u64, + command_pointer_ring: GRAPH + ITEMS as u64, gpu_va_10: LIST, + word_48: 0, gpu_va_9c: GRAPH + CONTROL as u64, + }); + let queue = usize::from(QUEUE_ID) * 0xc0; + out[queue..queue + 0xc0].copy_from_slice(&reset); + for offset in [0x28, 0x2c, 0x40] { put32(out, queue + offset, 2); } + out[queue + 0x36..queue + 0x38].fill(0xff); + put32(out, POINTERS + 0x50, u32::MAX); + put32(out, POINTERS + 0x60, 0x500); + out[CONTROL..CONTROL + 2].fill(0xff); + out[CONTROL + 5] = 1; + out[CONTROL + 0x26] = 2; + out[CONTROL + 0x33] = 0xff; + for (offset, bytes) in [(CONFIG, &install[..]), (CONFIG + 0x100, &config[..]), + (CONFIG + 0x200, &kick[..])] { + out[offset..offset + bytes.len()].copy_from_slice(bytes); + } + for (index, address) in [GRAPH + CONFIG as u64, DESC, + GRAPH + CONFIG as u64 + 0x100, GRAPH + CONFIG as u64 + 0x200] + .into_iter().enumerate() { put64(out, ITEMS + index * 8, address); } + put32(out, POINTERS + 0x40, 1); + Ok(()) + })?; + Ok(Self { aliases, descriptor, graph, entry, list, signals, storage, + firmware }) + } + + pub(super) fn queue_address() -> u64 { GRAPH + QUEUE_ID as u64 * 0xc0 } + + /// Re-encode the whole descriptor for one dispatch generation. The + /// scheduler record is cleared and re-seeded exactly as + /// M5GraphicsJob::rebuild_retired_frame() does for queues 10/11: firmware + /// republishes the completed count into it, so a stale one would satisfy + /// the next generation's completion gate before the dispatch even ran. + fn encode_descriptor(descriptor: &mut MappedObject, job: &M5GraphicsJob, + command: &command::TranslatedComputeCommand, storage: u64, + generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + descriptor.with_bytes_mut(|out| { + out[0x4000..0x4000 + 0xc8].fill(0); + put64(out, 0x4000, SIGNALS); + put32(out, 0x4010, 0x50); + compute::encode_m5_compute_descriptor(compute::M5ComputeDescriptor { + registers: compute::M5ComputeParameters { + preempt_base: storage + 0x20000, + cdm_base: command.control_stream_base, + dispatch_identity: 0x0000_0001_0000_0001, + context_id: job.context_id(), work_ordinal: 0, + robustness: storage + 0x24000, + operand_state_base: storage, + private_pool_hw_id: u64::from(graphics_usc::HARDWARE_ID), + descriptor_control_530: 0x54b, descriptor_control_538: 0xf5, + descriptor_flag_60a: false, descriptor_flag_60b: false, + }, + submit_sequence: 0, scheduler_record: SCHEDULER, + descriptor_low_alias: DESC_LOW, + cdm_terminator: command.control_stream_terminator, + dispatch_stamps: [SIGNALS + 0x40, SIGNALS + 0x80], + event_machine_stamp: generation * 0x100, queue_id: u32::from(QUEUE_ID), + queue_ordinal: 0, has_event: false, submission_index: generation, + status: [SIGNALS + 0xc0, SIGNALS + 0xc8], + shared_control: job.usc_control(), shared_control_enabled: true, + event_metadata: [0; 16], zero_page: ZERO, owner_id: 1, + descriptor_work_ordinal: 0, + }, &mut out[..0x1000]).map_err(|_| EINVAL) + }) + } + + fn prepare_entry(generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + let stamp = u64::from(generation); + sksm::prepare_m5_sksm_entry(sksm::M5SksmQueue { + id: QUEUE_ID, priority: 2, current_stamp: stamp, submitted: 0, + previous_stamp: if generation == 1 { None } else { Some(stamp - 1) }, + }, sksm::M5SksmOperands { + barriers: &[], first_predecessor: 0, implicit_dependency_type: 1, + mcache: None, flag_bit1: false, flag_bit61: false, flag_bit63: false, + payload: [DESC, Self::queue_address()], qos_byte_32: 3, qos_class_40: 8, + qos_word_48: 0xffff, event_mask: [0; 4], rce_kind: 0, + rce: [sksm::M5SksmRce { address: DESC_LOW + 0x40, tag: 37 }, + sksm::M5SksmRce { address: DESC_LOW + 0x760, tag: 4 }, + sksm::M5SksmRce { address: 0, tag: 0 }, + sksm::M5SksmRce { address: 0, tag: 0 }], + auxiliary: 0x3f_ffff_ffff_ffff, + }).map_err(|_| EINVAL) + } + + fn prepare_kick(generation: u32) -> Result<[u8; 21]> { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + sksm::encode_m5_sksm_add_kicks(QUEUE_ID, 2, u64::from(generation), 1) + .map_err(|_| EINVAL) + } + + /// Stage a further dispatch on the installed queue. Nothing here is + /// published: the producer head moves only in publish_dispatch(). + pub(super) fn prepare_next_dispatch(&mut self, job: &M5GraphicsJob, + command: &command::TranslatedComputeCommand, generation: u32) -> Result { + if generation < 2 || !compute_generation_admitted(generation) { return Err(EINVAL); } + if self.counters()? != [compute_pointer_head(generation - 1); 3] + || !self.installed()? { return Err(EBUSY); } + Self::encode_descriptor(&mut self.descriptor, job, command, + self.storage, generation)?; + let prepared = Self::prepare_entry(generation)?; + self.entry.with_bytes_mut(|out| { + let offset = prepared.offset as usize; + let slot = &mut out[offset..offset + prepared.bytes.len()]; + // Every preceding dispatch has retired, including the prior user + // of this 256-entry SKSM slot. Preserve the first-use check. + if generation < 256 && slot.iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + slot.copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let kick = Self::prepare_kick(generation)?; + self.graph.with_bytes_mut(|out| { + out[CONFIG + 0x200..CONFIG + 0x200 + kick.len()].copy_from_slice(&kick); + for (index, address) in [DESC, GRAPH + CONFIG as u64 + 0x100, + GRAPH + CONFIG as u64 + 0x200].into_iter().enumerate() { + let absolute = 1 + (generation as usize - 1) * 3 + index; + let offset = ITEMS + (absolute % GRAPHICS_POINTER_COUNT as usize) * 8; + if absolute < GRAPHICS_POINTER_COUNT as usize + && out[offset..offset + 8].iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + put64(out, offset, address); + } + Ok(()) + }) + } + + pub(super) fn counters(&mut self) -> Result<[u32; 3]> { + let mut out = [0; 3]; + for (index, offset) in [0, 0x30, 0x40].into_iter().enumerate() { + out[index] = self.graph.read_u32(GRAPH + (POINTERS + offset) as u64)?; + } + Ok(out) + } + + pub(super) fn installed(&mut self) -> Result { + let (slot, scheduler) = self.firmware.queue_registration(QUEUE_ID)?; + let header = self.graph.read_u32(GRAPH + CONTROL as u64)?; + let bitmap = self.graph.read_u64(GRAPH + CONTROL as u64 + 0x12)?; + Ok(slot[0] & 1 != 0 && slot[1] == ENTRY && slot[2] == ENTRY_LOW + && slot[3] == 4u64 << 32 && slot[4] as u32 == 2 + && scheduler == GRAPH + CONTROL as u64 + && header & 0xff != 0xff && (header >> 8) & 0xff != 0xff + && (header >> 16) & 0xff == 4 && bitmap == 1u64 << QUEUE_ID) + } + + pub(super) fn publish_dispatch(&mut self, generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + if self.counters()? != [compute_pointer_head(generation - 1); 3] + || !self.installed()? { return Err(EBUSY); } + self.graph.write_u32(GRAPH + POINTERS as u64 + 0x40, + compute_pointer_head(generation)) + } + + pub(super) fn completion(&mut self, signalled: u128) + -> Result { + Ok(crate::m5::completion::M5ComputeCompletion { + submitted: self.signals.read_u32(SIGNALS)?, + completed: self.descriptor.read_u32(SCHEDULER + 0xc)?, + linked: self.descriptor.read_u32(SCHEDULER + 0x94)?, + list: [self.list.read_u64(LIST)?, self.list.read_u64(LIST + 8)?], + stamp: self.signals.read_u32(SIGNALS + 0x40)?, signalled, + }) + } + + pub(super) fn complete(&mut self, signalled: u128, generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + Ok(self.counters()? == [compute_pointer_head(generation); 3] + && self.completion(signalled)? + .complete_at(LIST, QUEUE_ID, generation * 0x100)) + } +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} diff --git a/drivers/gpu/drm/asahi/m5/resources/firmware_view.rs b/drivers/gpu/drm/asahi/m5/resources/firmware_view.rs new file mode 100644 index 0000000000000..54ed6c966ac76 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/firmware_view.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Read-only view of the GFX firmware's DATA segment (the `gfx-data` reserved region). + +use crate::driver::AsahiDrmDevice; +use core::sync::atomic::{fence, Ordering}; +use kernel::{c_str, io, prelude::*}; + +pub(super) struct FirmwareDataView { + map: io::mem::Mem, + cache_line: usize, +} + +// SAFETY: This owns a fixed mapping of reserved RAM, read only through +// bounded volatile accesses. It neither transfers ownership nor writes DATA. +unsafe impl Send for FirmwareDataView {} +// SAFETY: Concurrent read-only volatile observations do not mutate host state. +unsafe impl Sync for FirmwareDataView {} + +impl FirmwareDataView { + pub(super) fn new(dev: &AsahiDrmDevice) -> Result { + let resource = dev.as_ref().as_ref().of_node().ok_or(EINVAL)? + .reserved_mem_region_to_resource_byname(c_str!("gfx-data"))?; + if resource.size() != 0x134000 || resource.start() < 0x100_0000_0000 + || resource.start() & 0x3fff != 0 + || resource.start().checked_add(resource.size()).ok_or(EOVERFLOW)? > 0x104_0000_0000 + || 0x127430 - 0x4c000 + 13 * 0x28 > resource.size() as usize + || 0x1257e0 - 0x4c000 + 13 * 8 > resource.size() as usize + || 0x178458 - 0x4c000 + 8 > resource.size() as usize { + return Err(EINVAL); + } + let ctr: u64; + // SAFETY: CTR_EL0 is an architectural read-only cache geometry register. + unsafe { core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)) }; + let cache_line = 4usize << ((ctr >> 16) & 0xf); + if cache_line > 0x4000 { return Err(EINVAL); } + // SAFETY: WB matches reserved System RAM. The host never writes this + // extent; observations invalidate covering lines before volatile reads. + let map = unsafe { io::mem::Mem::try_new(resource, io::mem::MemFlag::WB.into()) }?; + fence(Ordering::SeqCst); + Ok(Self { map, cache_line }) + } + + fn words(&self, offset: usize) -> [u64; N] { + // All callers use constructor-checked queue, scheduler or power extents. + let line = self.cache_line; + let start = offset & !(line - 1); + let end = (offset + N * 8 + line - 1) & !(line - 1); + for offset in (start..end).step_by(line) { + // SAFETY: Whole covering lines are inside the retained DATA view. + // The host never dirties them; do not clean stale data to firmware. + unsafe { core::arch::asm!("dc ivac, {addr}", + addr = in(reg) self.map.ptr().add(offset), + options(nostack, preserves_flags)) }; + } + // SAFETY: Complete invalidations before volatile observations. + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; + let mut out = [0; N]; + for (index, value) in out.iter_mut().enumerate() { + // SAFETY: The constructor bounds-checks every supported extent. + *value = unsafe { core::ptr::read_volatile( + self.map.ptr().add(offset + index * 8).cast::()) }; + } + fence(Ordering::Acquire); + out + } + + pub(super) fn queue_registration(&self, queue_id: u8) -> Result<([u64; 5], u64)> { + // TA/fragment own 10/11; application compute owns 12. The constructor + // bounds both complete tables through the last admitted queue. + if !(10..=12).contains(&queue_id) { return Err(EINVAL); } + Ok((self.words::<5>(0x127430 - 0x4c000 + queue_id as usize * 0x28), + self.words::<1>(0x1257e0 - 0x4c000 + queue_id as usize * 8)[0])) + } +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_job.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_job.rs new file mode 100644 index 0000000000000..0245811e9b5db --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_job.rs @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned TA/fragment resource graph for the bounded native-program test. +//! Construction leaves roots and queues unpublished. Submission wiring must +//! retain this owner through completion, faults and firmware shutdown. + +use super::{graphics_memory::M5GraphicsMemory, graphics_usc, MappedObject}; +use super::parameter_buffer::{M5ParameterBuffer, BLOCK_BYTES, PAGE_LIST_BYTES, PAGE_COUNT}; +use super::target_memory::M5TargetMemory; +use crate::{driver::AsahiDrmDevice, m5::render_desc, m5::render, mmu}; +use crate::m5::completion::MAX_GRAPHICS_GENERATION; +use kernel::prelude::*; + +const HIGH: u64 = 0xffff_fc20_c100_0000; +const LOW: u64 = 0x70_0030_0000; +const BYTES: usize = 0x40000; +const TA: usize = 0; +const FRAGMENT: usize = 0x4000; +const SCHEDULER: usize = 0x8000; +const SIGNALS_HIGH: u64 = HIGH + 0x80000; +const SUBMITTED: usize = 0; +const PM: usize = 0xc000; +const PM_SHARED: usize = 0x10000; +const PM_COUNTER: usize = 0x14000; +const PM_BLOCKS: usize = 0x18000; +const USC: usize = 0x1c000; +const USC_INNER: usize = 0x20000; +const USC_STATISTICS: usize = 0x24000; +const SCENES: usize = 0x30000; +const SCENE_ZERO: usize = SCENES + 80 * 0x80; +const TIMESTAMPS: usize = 0x38000; +const STAMPS_A: usize = 0x4000; +const STAMPS_B: usize = 0x8000; +const PAGE_LIST_HIGH: u64 = HIGH + 0x40000; +const METRICS_HIGH: u64 = HIGH + 0x60000; +const USER_BASE: u64 = 0x10_0000_0000; +pub(super) const PB_HARDWARE_ID: u32 = 0; +pub(super) const QUEUE_IDS: [u8; 2] = [10, 11]; +// Match Mesa's framebuffer dimensions; target allocations follow each size. +// Parameter-pool growth for more complex scenes remains separate. +use crate::m5::command::M5_MAX_RENDER_DIMENSION as MAX_APPLICATION_DIMENSION; +/// Retired targets kept for reuse. A video player cycles through a handful of +/// pass sizes every frame and a compositor's frames alternate with it, so +/// allocating per size change put several milliseconds on most jobs. +const TARGET_CACHE_ENTRIES: usize = 8; +/// Compute operands (128 KiB), a preemption page and a zero robustness page. +const COMPUTE_STORAGE_BYTES: usize = 0x28000; + +pub(super) struct M5GraphicsJob { + // The first owner's context retracts its roots before page release. + program: M5GraphicsMemory, + parameter_buffer: M5ParameterBuffer, + /// Storage of application compute, reserved with the parameter buffer so that its + /// address is free in every VM the job is handed to. + compute_storage: MappedObject, + target_memory: M5TargetMemory, + aliases: KVec, + // PB/USC pool views in a VM this job was handed to; empty for the first + // owner, whose pools were allocated in its own VM. + vm_aliases: KVec, + graph_alias: mmu::KernelMapping, + graph: MappedObject, + signals: MappedObject, + descriptor_template: KVec, + scene_template: KVec, + generation: u32, + target_shape: [u32; 5], + /// The VM `target_memory` lives in; a handoff leaves it behind. + target_vm: u64, + /// Retired targets by (VM, shape), least recently used first. + target_cache: KVec<(u64, [u32; 5], M5TargetMemory)>, +} + +impl M5GraphicsJob { + + /// Build a job from the submitting VM and translated application state. + /// No captured program is loaded; the caller supplies stage controls and + /// USC requirements derived for its program. Submission remains explicit. + #[allow(clippy::too_many_arguments)] + pub(super) fn from_vm( + dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, + parameters: render_desc::G17pRenderParameters, controls: render::M5RenderControls, + clusters: u32, usc_required: [u64; 2], + ) -> Result { + let program = M5GraphicsMemory::from_vm(dev, uat, vm)?; + Self::from_memory(dev, uat, program, parameters, controls, clusters, usc_required) + } + + #[allow(clippy::too_many_arguments)] + fn from_memory( + dev: &AsahiDrmDevice, uat: &mmu::Uat, program: M5GraphicsMemory, + mut p: render_desc::G17pRenderParameters, controls: render::M5RenderControls, + clusters: u32, usc_required: [u64; 2], + ) -> Result { + if !(1..=MAX_APPLICATION_DIMENSION).contains(&p.width) + || !(1..=MAX_APPLICATION_DIMENSION).contains(&p.height) + || p.layers != 1 || clusters != 1 { + return Err(ENOTSUPP); + } + let mut parameter_buffer = M5ParameterBuffer::new_reference(dev, &program)?; + let compute_storage = program.allocate_compact( + dev, COMPUTE_STORAGE_BYTES, 0x8000, mmu::PROT_GPU_SHARED_RW)?; + let target_memory = M5TargetMemory::new(dev, &program, &p, clusters)?; + target_memory.apply(&mut p); + p.cycle = parameter_buffer.scene_cycle(); + // Keep the PM register and firmware scene descriptor on the same + // owned metrics slot. The captured address is unmapped in a new VM. + p.record_index = render::parameter_buffer_metrics_record( + parameter_buffer.metrics() + 0x40).map_err(|_| EINVAL)?; + p.ta_hardware_buffer_id = PB_HARDWARE_ID; + p.usc_flist_hardware_buffer_id = graphics_usc::HARDWARE_ID; + p.ta_timestamp_start = HIGH + TIMESTAMPS as u64; + p.ta_timestamp_end = p.ta_timestamp_start + 8; + p.fragment_timestamp_start = p.ta_timestamp_start; + p.fragment_timestamp_end = p.ta_timestamp_end; + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), HIGH, + BYTES, mmu::PROT_FW_PRIV_RW)?; + // Native submitted totals and host-visible stamps are uncached in + // firmware. Keep them separate from the cached descriptor graph. + let mut signals = MappedObject::new_wc_at(dev, uat.kernel_vm(), SIGNALS_HIGH, + 0xc000, mmu::PROT_FW_SHARED_RW)?; + signals.write_u32(SIGNALS_HIGH + SUBMITTED as u64, 2)?; + let graph_alias = graph.map_alias_at(uat.kernel_lower_vm(), LOW, + mmu::PROT_GPU_SHARED_RW)?; + for offset in (0..BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(HIGH + offset as u64)? + != uat.kernel_lower_vm().translate_iova(LOW + offset as u64)? { + return Err(EFAULT); + } + } + let aliases = parameter_buffer.firmware_aliases( + uat, program.vm(), PAGE_LIST_HIGH, METRICS_HIGH, + )?; + let block_page_bases = parameter_buffer.block_page_bases()?; + graph.with_bytes_mut(|out| { + put64(out, SCHEDULER, SIGNALS_HIGH + SUBMITTED as u64); + put32(out, SCHEDULER + 0x10, 0x50); + // Two submitted stage records; firmware compares its completed + // count with this total, then unlinks the scheduler. + for (block, base) in block_page_bases.into_iter().enumerate() { + put32(out, PM_BLOCKS + block * 8, base); + } + put32(out, PM_SHARED, 3); + put32(out, PM_SHARED + 4, 3); + put32(out, PM_SHARED + 0x60, 1); + put32(out, PM_COUNTER, 1); + for (offset, value) in [(0xc, PB_HARDWARE_ID), (0x30, PAGE_LIST_BYTES as u32), + (0x34, PAGE_COUNT), (0x38, 1650), (0x3c, 3), (0x54, PAGE_COUNT - 1), + (0x58, BLOCK_BYTES as u32), (0x7c, 1650 * 16), (0x80, 550 * 16)] { + put32(out, PM + offset, value); + } + for (offset, address) in [(0x20, PAGE_LIST_HIGH), (0x28, parameter_buffer.page_list()), + (0x44, HIGH + PM_BLOCKS as u64), (0x4c, HIGH + PM_SHARED as u64), + (0x64, HIGH + PM_COUNTER as u64), (0x84, parameter_buffer.discard_offset())] { + put64(out, PM + offset, address); + } + // This sole job owns scene 1. M5 metrics use 64-byte spacing, + // unlike Neo dwords. Other reserved scenes remain unassigned; + // do not guess a scratch-ring wrap rule from a single draw. + let selected_scene = SCENES + 0x80; + put64(out, selected_scene, parameter_buffer.metrics() + 0x40); + put64(out, selected_scene + 8, METRICS_HIGH + 0x40); + put64(out, selected_scene + 0x28, parameter_buffer.scene_cycle()); + put64(out, selected_scene + 0x40, HIGH + PM_SHARED as u64 + 0x40); + put64(out, SCENES + 0x80 + 0x4c, 1); + put32(out, USC_INNER, 1); + program.usc().encode_initial_owner(HIGH + USC_INNER as u64, + &mut out[USC..USC + 0x70])?; + + Self::encode_descriptors(out, &p, &controls, program.context_id(), usc_required)?; + Ok(()) + })?; + let mut descriptor_template = KVec::new(); + let mut scene_template = KVec::new(); + graph.with_bytes_mut(|out| { + descriptor_template.extend_from_slice(&out[..SCHEDULER], GFP_KERNEL)?; + scene_template.extend_from_slice(&out[SCENES + 0x80..SCENES + 0x100], GFP_KERNEL)?; + Ok(()) + })?; + let target_vm = program.vm().id(); + Ok(Self { program, parameter_buffer, compute_storage, target_memory, aliases, + vm_aliases: KVec::new(), + graph_alias, graph, signals, descriptor_template, scene_template, generation: 1, + target_shape: [p.width, p.height, u32::from(p.layers), + u32::from(p.utile_width), u32::from(p.utile_height)], + target_vm, target_cache: KVec::new() }) + } + + #[allow(clippy::too_many_arguments)] + fn encode_descriptors( + out: &mut [u8], p: &render_desc::G17pRenderParameters, + controls: &render::M5RenderControls, context_id: u8, + usc_required: [u64; 2], + ) -> Result { + let objects = render_desc::RenderDescriptorObjects { + record_a: HIGH + SCHEDULER as u64, + shared: HIGH + PM as u64, + record_b: HIGH + SCENES as u64 + 0x80, + zero: HIGH + SCENE_ZERO as u64, + }; + let metadata = render_desc::RenderDescriptorMetadata { + context_id: context_id as u32, submission_ordinal: 0, command_buffer_state_word: 0, + ta_hardware_buffer_id: PB_HARDWARE_ID, submit_sequence: 1, + }; + render::build_ta_descriptor_base(p, controls, objects, metadata, + LOW + TA as u64, &mut out[TA..TA + render_desc::TA_DESCRIPTOR_SIZE]) + .map_err(|_| EINVAL)?; + render::build_fragment_descriptor_base(p, controls, objects, + render_desc::RenderDescriptorMetadata { submit_sequence: 0, ..metadata }, + LOW + FRAGMENT as u64, + &mut out[FRAGMENT..FRAGMENT + render_desc::FRAGMENT_DESCRIPTOR_SIZE]) + .map_err(|_| EINVAL)?; + let (ta, fragment) = out.split_at_mut(FRAGMENT); + let ta_usc = render::M5RenderUscState { + control: HIGH + USC as u64, required_bytes: usc_required[0], + allocated_bytes: graphics_usc::CAPACITY_BYTES, + statistics: HIGH + USC_STATISTICS as u64, + }; + render::apply_usc_submission_state(ta, fragment, ta_usc, + render::M5RenderUscState { + required_bytes: usc_required[1], statistics: ta_usc.statistics + 0x40, ..ta_usc + }).map_err(|_| EINVAL)?; + // This bounded first submission owns hardware stamp1 in each + // queue and event-machine timestamp0x100. Do not import native + // allocator history (the captured queues had stamps2/3). + put32(ta, 0x79c, QUEUE_IDS[1] as u32); + put64(ta, 0x7a0, 0x100); + put64(ta, 0x7a8, 1); + ta[0x7b0] = 1; + ta[0x7b1] = 1; + put64(ta, 0x8a6, SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[0] as u64 * 4); + put64(ta, 0x8ae, SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[0] as u64 * 4); + put32(ta, 0x8b6, 0x100); + put32(ta, 0x8ba, QUEUE_IDS[0] as u32); + put64(ta, 0x8c6, (1u64 << 40) | 1); + put64(fragment, 0x2140, SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[1] as u64 * 4); + put64(fragment, 0x2148, SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[1] as u64 * 4); + put32(fragment, 0x2150, 0x100); + put32(fragment, 0x2154, QUEUE_IDS[1] as u32); + put32(fragment, 0x215c, 0); + put64(fragment, 0x2160, (1u64 << 40) | 1); + // Fixed 256x256 reference controls independently observed in + // the successful 25G76 draw; these replace inherited Neo values. + ta[0x789] = 8; + ta[0x85f] = 7; + put32(ta, 0x892, 0); + ta[0x932] = 0; + put32(fragment, 0x1fb0, 8); + fragment[0x21cc] = 0; + fragment[0x222d] = 0; + Ok(()) + } + + pub(super) fn parameter_descriptor(&self) -> Result<[u32; 4]> { + render::initial_parameter_buffer_descriptor(self.parameter_buffer.page_list(), PAGE_COUNT) + .map_err(|_| EINVAL) + } + + pub(super) fn uma_descriptor(&self) -> [u64; 4] { self.program.usc().descriptor() } + + pub(super) fn roots(&self) -> [u64; 2] { self.program.roots() } + + pub(super) fn context_id(&self) -> u8 { self.program.context_id() } + + pub(super) fn owns_vm(&self, vm: &mmu::Vm) -> bool { + self.program.vm().id() == vm.id() + } + + /// Hand this fully retired job to another submitting VM. The hardware + /// context slot, descriptor graph, signals, queues and the PB/USC pools + /// all keep their firmware-visible addresses: the context is re-leased + /// for the new VM and the pools are aliased into it. The previous VM's + /// render targets are not used in the new VM: they stay cached under the + /// previous VM for its next turn, and the next application frame takes or + /// allocates targets in the new VM. The caller publishes the new roots + /// before the next draw. + pub(super) fn rebind_vm(&mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + if self.owns_vm(vm) { return Ok(()); } + if self.scheduler_status()? != [2, 0] { return Err(EBUSY); } + let progress = self.progress()?; + if progress[0] != 2 || progress[1] != self.generation * 0x100 + || progress[3] != self.generation * 0x100 { return Err(EBUSY); } + // Release an earlier handoff's views before reserving the same + // addresses in the incoming VM. + self.vm_aliases = KVec::new(); + let mut aliases = self.program.rebind(dev, vm)?; + for alias in self.parameter_buffer.alias_into(vm)? { + aliases.push(alias, GFP_KERNEL)?; + } + let storage = self.compute_storage.iova(); + if let Some(alias) = self.compute_storage.map_alias_at_with_guards( + vm, storage, mmu::PROT_GPU_SHARED_RW, 2)? { + aliases.push(alias, GFP_KERNEL)?; + } + self.vm_aliases = aliases; + dev_dbg!(dev.as_ref(), "M5_RENDER_VM_HANDOFF context={} generation={} vm={}\n", + self.program.context_id(), self.generation, vm.id()); + Ok(()) + } + + /// Compute shares this application's published roots and USC owner. Only + /// its operand/preemption storage is its own: the GPU address of that storage. + pub(super) fn compute_storage(&self) -> u64 { + self.compute_storage.iova() + } + + pub(super) fn descriptor_addresses(&self, stage: usize) -> Result<[u64; 2]> { + let offset = match stage { 0 => TA, 1 => FRAGMENT, _ => return Err(EINVAL) }; + Ok([HIGH + offset as u64, LOW + offset as u64]) + } + + pub(super) fn usc_control(&self) -> u64 { HIGH + USC as u64 } + + pub(super) fn parameter_state(&self) -> u64 { HIGH + PM as u64 } + + pub(super) fn publish_context(&mut self) -> Result { self.program.publish() } + + pub(super) fn quarantine(&self) { self.program.quarantine(); } + + pub(super) fn progress(&mut self) -> Result<[u32; 5]> { + Ok([self.signals.read_u32(SIGNALS_HIGH + SUBMITTED as u64)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[0] as u64 * 4)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[0] as u64 * 4)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[1] as u64 * 4)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[1] as u64 * 4)?]) + } + + pub(super) fn scheduler_status(&mut self) -> Result<[u32; 2]> { + // GFX cleans this 0xc8-byte object after unlinking the finished pair. + Ok([self.graph.read_u32(HIGH + SCHEDULER as u64 + 0xc)?, + self.graph.read_u32(HIGH + SCHEDULER as u64 + 0x94)?]) + } + + fn rebuild_retired_frame(&mut self, generation: u32) -> Result { + if generation != self.generation + 1 || generation > MAX_GRAPHICS_GENERATION + || self.scheduler_status()? != [2, 0] { + return Err(EBUSY); + } + let progress = self.progress()?; + if progress[0] != 2 || progress[1] != self.generation * 0x100 + || progress[3] != self.generation * 0x100 { return Err(EBUSY); } + self.graph.with_bytes_mut(|out| { + out[..SCHEDULER].copy_from_slice(&self.descriptor_template); + out[SCHEDULER..SCHEDULER + 0xc8].fill(0); + put64(out, SCHEDULER, SIGNALS_HIGH + SUBMITTED as u64); + put32(out, SCHEDULER + 0x10, 0x50); + out[SCENES + 0x80..SCENES + 0x100].copy_from_slice(&self.scene_template); + out[TIMESTAMPS..TIMESTAMPS + 16].fill(0); + let stamp = generation * 0x100; + put64(out, TA + 0x7a0, stamp as u64); + put64(out, TA + 0x7a8, generation as u64); + put32(out, TA + 0x8b6, stamp); + put64(out, TA + 0x8c6, (1u64 << 40) | generation as u64); + put32(out, FRAGMENT + 0x2150, stamp); + put64(out, FRAGMENT + 0x2160, (1u64 << 40) | generation as u64); + Ok(()) + })?; + // Leave previous stamps intact: they must be superseded by firmware, + // never host-written to the expected new completion value. + self.generation = generation; + Ok(()) + } + + /// Re-encode a retired application's descriptors without replaying its old + /// command pointers. The current bounded queue retains one VM. Target + /// storage can change only after complete retirement; the job + /// stamp keeps increasing and context replacement remains unsupported. + #[allow(clippy::too_many_arguments)] + pub(super) fn prepare_application_frame( + &mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm, mut p: render_desc::G17pRenderParameters, + controls: render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], generation: u32, + ) -> Result { + if vm.id() != self.program.vm().id() || clusters != 1 + || !(1..=MAX_APPLICATION_DIMENSION).contains(&p.width) + || !(1..=MAX_APPLICATION_DIMENSION).contains(&p.height) || p.layers != 1 + || generation != self.generation + 1 || generation > MAX_GRAPHICS_GENERATION { + return Err(ENOTSUPP); + } + // Validate retirement before allocating or releasing any target pages. + // The caller additionally checks both queues, stamps and event bits. + let progress = self.progress()?; + if self.scheduler_status()? != [2, 0] || progress[0] != 2 + || progress[1] != self.generation * 0x100 + || progress[3] != self.generation * 0x100 { return Err(EBUSY); } + let shape = [p.width, p.height, u32::from(p.layers), + u32::from(p.utile_width), u32::from(p.utile_height)]; + let vm_id = self.program.vm().id(); + let replacement = if (vm_id, shape) != (self.target_vm, self.target_shape) { + let cached = self.target_cache.iter() + .position(|(cached_vm, cached_shape, _)| (*cached_vm, *cached_shape) == (vm_id, shape)); + Some(match cached { + Some(index) => self.target_cache.remove(index).map_err(|_| EINVAL)?.2, + None => M5TargetMemory::new(dev, &self.program, &p, clusters).inspect_err(|e| pr_err!("M5: target memory failed: {:?}\n", e))?, + }) + } else { None }; + replacement.as_ref().unwrap_or(&self.target_memory).apply(&mut p); + p.cycle = self.parameter_buffer.scene_cycle(); + p.record_index = render::parameter_buffer_metrics_record( + self.parameter_buffer.metrics() + 0x40).map_err(|_| EINVAL)?; + p.ta_hardware_buffer_id = PB_HARDWARE_ID; + p.usc_flist_hardware_buffer_id = graphics_usc::HARDWARE_ID; + p.ta_timestamp_start = HIGH + TIMESTAMPS as u64; + p.ta_timestamp_end = p.ta_timestamp_start + 8; + p.fragment_timestamp_start = p.ta_timestamp_start; + p.fragment_timestamp_end = p.ta_timestamp_end; + let mut descriptors = KVec::new(); + descriptors.extend_from_slice(&self.descriptor_template, GFP_KERNEL)?; + Self::encode_descriptors(&mut descriptors, &p, &controls, + self.program.context_id(), usc_required).inspect_err(|e| pr_err!("M5: encode_descriptors failed: {:?}\n", e))?; + self.descriptor_template = descriptors; + // The application owns its output. It must never be erased by the + // native diagnostic's sentinel/reset logic. + self.rebuild_retired_frame(generation).inspect_err(|e| pr_err!("M5: rebuild_retired_frame failed: {:?}\n", e))?; + if let Some(target) = replacement { + dev_dbg!(dev.as_ref(), "M5_RENDER_TARGET_RESIZE generation={} old={:?} new={:?}\n", + generation, self.target_shape, shape); + let retired = core::mem::replace(&mut self.target_memory, target); + if self.target_cache.len() == TARGET_CACHE_ENTRIES { + self.target_cache.remove(0).map_err(|_| EINVAL)?; + } + self.target_cache.push((self.target_vm, self.target_shape, retired), GFP_KERNEL)?; + self.target_vm = vm_id; + self.target_shape = shape; + } + Ok(()) + } + +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +pub(super) fn triangle_parameters() -> (render_desc::G17pRenderParameters, render::M5RenderControls) { + let mut p = render_desc::G17pRenderParameters::source_rsrc8(); + p.aux_fb = 0x10000288000; + p.aux_fb_flags = 0x10c001; + p.aux_fb_page_count = 0x100000; + p.context_base = USER_BASE; + p.cycle = 0x220040; + p.deflake_1 = 0x1000068340; + p.deflake_2 = 0x1000068040; + p.deflake_3 = 0x1000068000; + p.depth_aux_buffer = 0; + p.depth_aux_stride = 0; + p.depth_buffer = 0; + p.depth_clear_value_bits = 0x3f800000; + p.depth_dimensions = 0; + p.depth_flags = 0; + p.depth_stride = 0; + p.encoder = 0x1000018000; + p.fragment_lifecycle = 0x4089f4c0508e74c; + p.fragment_status = 0x1000258000; + p.heapmeta = 0x1000261000; + p.width = 256; + p.height = 256; + p.layers = 1; + p.lifecycle = 0x4089f4c0508e74d; + p.load_pipeline = 0x10000170200; + p.load_pipeline_bind = 0xffff800000000040; + p.merge_upper_x_bits = 0x3bddb3d9; + p.merge_upper_y_bits = 0x3bddb3d9; + p.multisample_control = 0x88; + p.occlusion_query_base = 0; + p.ppp_control = 0x202; + p.record_index = 0x80041; + p.stencil_aux_buffer = 0; + p.stencil_aux_stride = 0; + p.stencil_buffer = 0; + p.stencil_clear_value = 0; + p.stencil_stride = 0; + p.store_pipeline = 0x10000170400; + p.store_pipeline_bind = 0; + p.ta_hardware_buffer_id = PB_HARDWARE_ID; + p.ta_status = 0x1000078000; + p.tib_blocks = 0; + p.tile_config = 0x10280; + p.tilemap = 0x1000260000; + p.tpc = 0x1000288000; + p.usc_flist_hardware_buffer_id = graphics_usc::HARDWARE_ID; + p.utile_config = 0xa000; + p.utile_height = 32; + p.utile_width = 32; + p.absolute_pointers = false; + p.partial_store_pipeline_bind = 0; + p.partial_store_pipeline = 0x10000170400; + p.partial_load_pipeline_bind = 0xffff800000000040; + p.partial_load_pipeline = 0x10000170000; + p.depth_bias_array = 0x100002d8000; + p.scissor_array = 0x10000180000; + p.ta_timestamp_start = HIGH + TIMESTAMPS as u64; + p.ta_timestamp_end = p.ta_timestamp_start + 8; + p.fragment_timestamp_start = p.ta_timestamp_start; + p.fragment_timestamp_end = p.ta_timestamp_end; + let controls = render::M5RenderControls { + ta: render::M5RenderStageControls { + resource_flags: 0x4054b, resource_control: 0x200000001, ptc_max: 0xc6, work_key: 0x300, + }, + fragment: render::M5RenderStageControls { + resource_flags: 0x104054b, resource_control: 0x200000001, ptc_max: 0xce, work_key: 0x300, + }, + ta_tile_control: 0x10c08860, ta_dispatch_control: 0x700, + fragment_raster_control: 0x200000ff0200, fragment_tile_flags: 0x200003717f, + store_pipeline_2_bind: 0, store_pipeline_2: 0x10000170400, + partial_store_pipeline_2_bind: 0, partial_store_pipeline_2: 0x10000170400, + partial_tile_flags: 0x200000, + }; + (p, controls) +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_memory.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_memory.rs new file mode 100644 index 0000000000000..cf9a2f93efb03 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_memory.rs @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained M5 render VM and private storage, with an optional native test program. +//! +//! Allocation is separate from root publication and queue submission. A future +//! job owner keeps this graph alive through queue completion. After a fault, +//! ownership remains retained until both firmware CPUs stop. Captured physical +//! backing is never installed. + +use super::MappedObject; +use super::graphics_usc::M5GraphicsUsc; +use crate::{driver::AsahiDrmDevice, mmu}; + +/// Pages of a client VM aliased into the context, at most this many. +const PAGE_BYTES: usize = 0x4000; +const MAX_PAGES: usize = 384; +use kernel::prelude::*; + +pub(super) struct M5GraphicsMemory { + // Retract any published roots before dropping their mapping owners. + context: mmu::M5ExecutionContext, + // Keep user mappings and GEM backing across handle close/VM teardown. + // This guard's bookkeeping is generation-neutral despite its old name. + user_mappings: Option, + pages: KVec, + usc: M5GraphicsUsc, + output_first: Option, +} + +impl M5GraphicsMemory { + + /// Keep the actual submitting VM and allocate only driver-owned storage. + /// Application shader/input/output mappings remain in that VM; no native + /// program firmware or captured pages are requested by this constructor. + pub(super) fn from_vm(dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm) -> Result { + crate::m5::vm_layout::compact_range(&vm.kernel_range()).ok_or(EINVAL)?; + let user_mappings = vm.retain_job()?; + let context = uat.new_m5_execution_context(vm)?; + let usc = M5GraphicsUsc::new_application(dev, context.vm())?; + Ok(Self { context, user_mappings: Some(user_mappings), pages: KVec::new(), usc, output_first: None }) + } + + /// Hand the retained USC pool and this context slot to another + /// application VM. The previous roots are retracted first (with ASID + /// invalidation), the same hardware context is re-leased for `vm`, and + /// the pool is aliased at unchanged addresses, so every firmware record + /// that names the context or the pool keeps its value. The caller has + /// verified that the previous owner's work is completely retired and + /// publishes the new roots explicitly afterwards. + pub(super) fn rebind(&mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm) + -> Result> { + crate::m5::vm_layout::compact_range(&vm.kernel_range()).ok_or(EINVAL)?; + // Only application owners hand over; the native diagnostic program + // keeps captured pages in its private VM. + if !self.pages.is_empty() || self.output_first.is_some() { return Err(ENOTSUPP); } + let user_mappings = vm.retain_job()?; + self.context.rebind(vm)?; + // Release the previous VM's mapping hold only after its roots are gone. + self.user_mappings = Some(user_mappings); + let aliases = self.usc.alias_into(vm)?; + dev_dbg!(dev.as_ref(), "M5_GRAPHICS_CONTEXT id={} rebound vm={} roots={:x?}; USC pool aliased\n", + self.context.context_id(), vm.id(), self.context.roots()); + Ok(aliases) + } + + /// Return an owning allocation in the compact TA resource arena. The + /// render job retains it until completion or confirmed processor stop. + pub(super) fn allocate_compact( + &self, dev: &AsahiDrmDevice, size: usize, alignment: u64, prot: mmu::Prot, + ) -> Result { + let range = crate::m5::vm_layout::compact_range(&self.vm().kernel_range()).ok_or(EINVAL)?; + MappedObject::new_wc_in_range_with_guards( + dev, self.vm(), range, size, alignment, prot, 2, + ) + } + + /// Job construction can add its own objects before root publication. + pub(super) fn vm(&self) -> &mmu::Vm { + self.context.vm() + } + + pub(super) fn context_id(&self) -> u8 { self.context.context_id() } + + pub(super) fn roots(&self) -> [u64; 2] { + self.context.roots() + } + + pub(super) fn usc(&self) -> &M5GraphicsUsc { + &self.usc + } + + /// A stop request is not proof that hardware has released these mappings. + /// Close VM_BIND admission before any future error fence can wake userspace. + pub(super) fn quarantine(&self) { + if let Some(guard) = &self.user_mappings { + guard.quarantine(); + } + } + + /// Alias existing owned user pages into the firmware VM. PM page lists + /// and scene counters must share backing with their user GPU views. + /// The returned mappings belong to the eventual retained job owner. + pub(super) fn alias_kernel_pages( + &mut self, + uat: &mmu::Uat, + user_address: u64, + kernel_address: u64, + bytes: usize, + ) -> Result> { + if self.context.is_current() { + return Err(EBUSY); + } + if bytes == 0 + || bytes > MAX_PAGES * PAGE_BYTES + || bytes % PAGE_BYTES != 0 + || (user_address | kernel_address) & (PAGE_BYTES as u64 - 1) != 0 + { + return Err(EINVAL); + } + user_address.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + kernel_address.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + let mut indices = KVec::new(); + for offset in (0..bytes).step_by(PAGE_BYTES) { + let address = user_address + offset as u64; + let index = self + .pages + .iter() + .position(|page| page.iova() == address) + .ok_or(EFAULT)?; + indices.push(index, GFP_KERNEL)?; + } + let mut aliases = KVec::new(); + for (ordinal, index) in indices.iter().enumerate() { + let high = kernel_address + (ordinal * PAGE_BYTES) as u64; + let alias = + self.pages[*index].map_alias_at(uat.kernel_vm(), high, mmu::PROT_FW_SHARED_RW)?; + if uat.kernel_vm().translate_iova(high)? + != self + .context + .vm() + .translate_iova(self.pages[*index].iova())? + { + return Err(EFAULT); + } + aliases.push(alias, GFP_KERNEL)?; + } + Ok(aliases) + } + + /// Read a word from a retained reference page while preparing its job. + pub(super) fn read_word(&mut self, address: u64) -> Result { + if address & 3 != 0 { + return Err(EINVAL); + } + let base = address & !(PAGE_BYTES as u64 - 1); + let page = self + .pages + .iter_mut() + .find(|page| page.iova() == base) + .ok_or(EFAULT)?; + page.read_u32(address) + } + + /// Reset owned PM outputs before exposing the context to either firmware. + pub(super) fn clear_pages(&mut self, address: u64, bytes: usize) -> Result { + if self.context.is_current() || bytes == 0 || bytes % PAGE_BYTES != 0 + || address & (PAGE_BYTES as u64 - 1) != 0 { + return Err(EINVAL); + } + let end = address.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + let count = self.pages.iter().filter(|p| p.iova() >= address && p.iova() < end).count(); + if count != bytes / PAGE_BYTES { return Err(EFAULT); } + for page in self.pages.iter_mut().filter(|p| p.iova() >= address && p.iova() < end) { + page.with_bytes_mut(|out| { out.fill(0); Ok(()) })?; + } + Ok(()) + } + + pub(super) fn write_word(&mut self, address: u64, value: u32) -> Result { + if self.context.is_current() || address & 3 != 0 { return Err(EINVAL); } + let base = address & !(PAGE_BYTES as u64 - 1); + let offset = (address - base) as usize; + let page = self.pages.iter_mut().find(|p| p.iova() == base).ok_or(EFAULT)?; + page.with_bytes_mut(|out| { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + Ok(()) + }) + } + + /// Call only after the complete TA/3D resource graph is prepared. + pub(super) fn publish(&mut self) -> Result { + self.context.publish() + } + + pub(super) fn is_current(&self) -> bool { + self.context.is_current() + } + +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_queues.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_queues.rs new file mode 100644 index 0000000000000..02850fbe073df --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_queues.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Paired first-submission queues for the retained M5 graphics experiment. + +use super::{graphics_job::{M5GraphicsJob, PB_HARDWARE_ID, QUEUE_IDS}, graphics_usc, + firmware_view::FirmwareDataView, MappedObject}; +use crate::{driver::AsahiDrmDevice, m5::submission, m5::channel, m5::sksm, mmu}; +use crate::m5::completion::{graphics_pointer_head, GRAPHICS_POINTER_COUNT, + MAX_GRAPHICS_GENERATION}; +use kernel::prelude::*; + +const HIGH: u64 = 0xffff_fc20_c000_0000; +const ENTRY_HIGH: u64 = 0xffff_fc20_0039_0000; +const ENTRY_LOW: u64 = 0x70_005f_0000; +const ENTRY_BYTES: usize = 0x20000; +const CONTROL: usize = 0x18000; +// Native TA and fragment channel+0x10 both point to fffffc2000000048. +// One scheduler belongs to one intrusive list even when two SKSM queues run it. +pub(super) const JOB_LIST: u64 = HIGH + 0x20000; +const POINTERS: [usize; 2] = [0x4000, 0xc000]; +const ITEMS: [usize; 2] = [0x8000, 0x10000]; +const CONFIG: [usize; 2] = [0xa800, 0x12800]; + +pub(super) struct M5GraphicsQueues { + aliases: KVec, + entries: KVec, + graph: MappedObject, + job_list: MappedObject, + firmware: FirmwareDataView, +} + +impl M5GraphicsQueues { + pub(super) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, job: &M5GraphicsJob) -> Result { + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), HIGH, + 0x20000, mmu::PROT_FW_PRIV_RW)?; + let mut job_list = MappedObject::new_wc_at(dev, uat.kernel_vm(), JOB_LIST, + 0x4000, mmu::PROT_FW_SHARED_RW)?; + job_list.with_bytes_mut(|out| { put64(out, 8, JOB_LIST); Ok(()) })?; + let mut entries = KVec::new(); + let mut aliases = KVec::new(); + for stage in 0..2 { + let id = QUEUE_IDS[stage]; + let high = ENTRY_HIGH + (stage * ENTRY_BYTES) as u64; + let low = ENTRY_LOW + (stage * ENTRY_BYTES) as u64; + let mut entry = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, + ENTRY_BYTES, mmu::PROT_FW_PRIV_RW)?; + let alias = entry.map_alias_at(uat.kernel_lower_vm(), low, mmu::PROT_GPU_SHARED_RW)?; + for offset in (0..ENTRY_BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset as u64)? + != uat.kernel_lower_vm().translate_iova(low + offset as u64)? { + return Err(EFAULT); + } + } + let [descriptor, _] = job.descriptor_addresses(stage)?; + let dm = if stage == 0 { sksm::M5DataMaster::Tiling } + else { sksm::M5DataMaster::Fragment }; + let prepared = Self::prepare_entry(job, stage, 1)?; + entry.with_bytes_mut(|out| { + out[prepared.offset as usize..prepared.offset as usize + prepared.bytes.len()] + .copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let install = channel::encode_m5_channel_config_for(channel::M5ChannelConfig { + ring_low: low, ring_high: high, queue_id: id, priority: 2, + install_queue: true, gpu_context: job.context_id(), event: None, + channel_control: HIGH + CONTROL as u64, update_scheduler: true, + scheduler_index: 0, channel_stamp_index: stage as u32, + update_context: false, owner_context: 0, extended_command: false, + }, stage as u32, 2, None, None).map_err(|_| EINVAL)?; + let config = channel::encode_m5_channel_config_for(channel::M5ChannelConfig { + ring_low: low, ring_high: high, queue_id: id, priority: 2, + install_queue: false, gpu_context: job.context_id(), + event: Some(channel::M5ChannelEvent { address: job.usc_control(), + timestamp: 1, index: graphics_usc::HARDWARE_ID }), + channel_control: HIGH + CONTROL as u64, update_scheduler: false, + scheduler_index: 0, channel_stamp_index: stage as u32, + update_context: true, owner_context: 1, extended_command: false, + }, stage as u32, 2, Some(0), if stage == 0 { + Some(channel::M5ParameterBufferEvent { address: job.parameter_state(), + timestamp: 1, hardware_buffer_id: PB_HARDWARE_ID, manager_id: 0 }) + } else { None }).map_err(|_| EINVAL)?; + let kick = sksm::encode_m5_sksm_add_kicks_for(dm, id, 2, 1, 1) + .map_err(|_| EINVAL)?; + graph.with_bytes_mut(|out| { + let pointers = POINTERS[stage]; + let queue = id as usize * 0xc0; + let reset = submission::encode_m5_25g76_channel_state_reset( + submission::M5ChannelStateResetValues { + uncached_state: HIGH + pointers as u64, + command_pointer_ring: HIGH + ITEMS[stage] as u64, + gpu_va_10: JOB_LIST, word_48: 0, + gpu_va_9c: HIGH + CONTROL as u64, + }); + out[queue..queue + 0xc0].copy_from_slice(&reset); + for offset in [0x28, 0x2c, 0x40] { put32(out, queue + offset, 2); } + out[queue + 0x36..queue + 0x38].fill(0xff); + put32(out, pointers + 0x50, u32::MAX); + put32(out, pointers + 0x60, 0x500); + out[CONFIG[stage]..CONFIG[stage] + install.len()].copy_from_slice(&install); + out[CONFIG[stage] + 0x100..CONFIG[stage] + 0x100 + config.len()].copy_from_slice(&config); + out[CONFIG[stage] + 0x200..CONFIG[stage] + 0x200 + kick.len()].copy_from_slice(&kick); + for (index, address) in [HIGH + CONFIG[stage] as u64, descriptor, + HIGH + CONFIG[stage] as u64 + 0x100, + HIGH + CONFIG[stage] as u64 + 0x200].into_iter().enumerate() { + put64(out, ITEMS[stage] + index * 8, address); + } + // Publish only installation. The draw stays beyond the producer + // until both firmware queue slots and scheduler bindings verify. + put32(out, pointers + 0x40, 1); + Ok(()) + })?; + aliases.push(alias, GFP_KERNEL)?; + entries.push(entry, GFP_KERNEL)?; + } + graph.with_bytes_mut(|out| { + out[CONTROL..CONTROL + 2].fill(0xff); + out[CONTROL + 5] = 1; + out[CONTROL + 0x26] = 2; + out[CONTROL + 0x33] = 0xff; + Ok(()) + })?; + Ok(Self { aliases, entries, graph, job_list, firmware: FirmwareDataView::new(dev)? }) + } + + pub(super) fn queue_address(stage: usize) -> Result { + Ok(HIGH + *QUEUE_IDS.get(stage).ok_or(EINVAL)? as u64 * 0xc0) + } + + fn prepare_entry(job: &M5GraphicsJob, stage: usize, generation: u32) + -> Result { + if !(1..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + let id = *QUEUE_IDS.get(stage).ok_or(EINVAL)?; + let [descriptor, descriptor_low] = job.descriptor_addresses(stage)?; + let stamp = generation as u64; + let barrier = [sksm::M5SksmDependency { queue_id: QUEUE_IDS[0], stamp, kind: 1 }]; + let mut rce = [sksm::M5SksmRce { address: 0, tag: 0 }; 4]; + if stage == 0 { + rce[0] = sksm::M5SksmRce { address: descriptor_low + 0x60, tag: 79 }; + } else { + for (index, (offset, count)) in [(0xa0, 87), (0x7c0, 18), + (0xee0, 25), (0x1600, 12)].into_iter().enumerate() { + rce[index] = sksm::M5SksmRce { address: descriptor_low + offset, tag: count }; + } + } + sksm::prepare_m5_sksm_entry(sksm::M5SksmQueue { + id, priority: 2, current_stamp: stamp, submitted: 0, + previous_stamp: if generation == 1 { None } else { Some(stamp - 1) }, + }, sksm::M5SksmOperands { + barriers: if stage == 0 { &[] } else { &barrier }, first_predecessor: 0, + implicit_dependency_type: 1, mcache: None, flag_bit1: false, + flag_bit61: false, flag_bit63: false, + payload: [descriptor, Self::queue_address(stage)?], + qos_byte_32: 3, qos_class_40: if stage == 0 { 12 } else { 24 }, + qos_word_48: 0xffff, event_mask: [0; 4], rce_kind: 0, rce, + auxiliary: 0x3f_ffff_ffff_ffff, + }).map_err(|_| EINVAL) + } + + pub(super) fn publish_next_draw(&mut self, job: &M5GraphicsJob, generation: u32) -> Result { + if !(2..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + let previous_head = graphics_pointer_head(generation - 1); + if self.counters()? != [[previous_head; 3]; 2] || !self.installed()? + || self.completion_list()? != [0, JOB_LIST] { return Err(EBUSY); } + for stage in 0..2 { + let prepared = Self::prepare_entry(job, stage, generation)?; + self.entries[stage].with_bytes_mut(|out| { + let offset = prepared.offset as usize; + let slot = &mut out[offset..offset + prepared.bytes.len()]; + // Every preceding job has retired, including the prior user + // of this 256-entry SKSM slot. Preserve the first-use check. + if generation < 256 && slot.iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + slot.copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let dm = if stage == 0 { sksm::M5DataMaster::Tiling } + else { sksm::M5DataMaster::Fragment }; + let kick = sksm::encode_m5_sksm_add_kicks_for(dm, QUEUE_IDS[stage], + 2, generation as u64, 1).map_err(|_| EINVAL)?; + self.graph.with_bytes_mut(|out| { + out[CONFIG[stage] + 0x200..CONFIG[stage] + 0x200 + kick.len()] + .copy_from_slice(&kick); + let addresses = [job.descriptor_addresses(stage)?[0], + HIGH + CONFIG[stage] as u64 + 0x100, + HIGH + CONFIG[stage] as u64 + 0x200]; + for (index, address) in addresses.into_iter().enumerate() { + let absolute = 1 + (generation as usize - 1) * 3 + index; + let offset = ITEMS[stage] + (absolute % GRAPHICS_POINTER_COUNT as usize) * 8; + if absolute < GRAPHICS_POINTER_COUNT as usize + && out[offset..offset + 8].iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + put64(out, offset, address); + } + Ok(()) + })?; + } + for pointers in POINTERS { + self.graph.write_u32(HIGH + pointers as u64 + 0x40, + graphics_pointer_head(generation))?; + } + Ok(()) + } + + pub(super) fn counters(&mut self) -> Result<[[u32; 3]; 2]> { + let mut result = [[0; 3]; 2]; + for stage in 0..2 { + for (index, offset) in [0, 0x30, 0x40].into_iter().enumerate() { + result[stage][index] = self.graph.read_u32(HIGH + (POINTERS[stage] + offset) as u64)?; + } + } + Ok(result) + } + + pub(super) fn completion_list(&mut self) -> Result<[u64; 2]> { + Ok([self.job_list.read_u64(JOB_LIST)?, self.job_list.read_u64(JOB_LIST + 8)?]) + } + + pub(super) fn installation_status(&mut self) -> Result<([[u64; 5]; 2], [u64; 2], [u64; 2])> { + let mut slots = [[0; 5]; 2]; + let mut pointers = [0; 2]; + for stage in 0..2 { + (slots[stage], pointers[stage]) = self.firmware.queue_registration(QUEUE_IDS[stage])?; + } + Ok((slots, pointers, [u64::from(self.graph.read_u32(HIGH + CONTROL as u64)?), + self.graph.read_u64(HIGH + CONTROL as u64 + 0x12)?])) + } + + pub(super) fn installed(&mut self) -> Result { + let (slots, pointers, control) = self.installation_status()?; + for stage in 0..2 { + let slot = slots[stage]; + let offset = (stage * ENTRY_BYTES) as u64; + // GFX 0x12240/0x1241c copies tag15 +0x22 to slot +0x20: + // this word is the TA/fragment stage, not the outer channel. + if slot[0] & 1 == 0 || slot[1] != ENTRY_HIGH + offset + || slot[2] != ENTRY_LOW + offset || slot[3] != 4u64 << 32 + || slot[4] as u32 != stage as u32 || pointers[stage] != HIGH + CONTROL as u64 { + return Ok(false); + } + } + Ok(control[0] & 0xff != 0xff && (control[0] >> 8) & 0xff != 0xff + && (control[0] >> 16) & 0xff == 4 + && control[1] == ((1u64 << QUEUE_IDS[0]) | (1u64 << QUEUE_IDS[1]))) + } + + pub(super) fn publish_draw(&mut self) -> Result { + if self.counters()? != [[1; 3]; 2] || !self.installed()? { return Err(EBUSY); } + for pointers in POINTERS { + self.graph.write_u32(HIGH + pointers as u64 + 0x40, 4)?; + } + Ok(()) + } +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_usc.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_usc.rs new file mode 100644 index 0000000000000..1d5bd727ee719 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_usc.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained USC backing in an M5 application's reserved driver VA region. +//! +//! Firmware records are encoded from the owned mappings, never fixed test +//! addresses. Guard pages belong to each mapping's allocator reservation. +//! No root or firmware table is published here. + +use super::MappedObject; +use crate::{driver::AsahiDrmDevice, mmu}; +use kernel::prelude::*; + +pub(super) const HARDWARE_ID: u32 = 1; +const DIRECTORY_ENTRIES: usize = 0x80000; +const BLOCK_BYTES: usize = 0x20_0000; +const BLOCK_COUNT: usize = 19; +const UNIT_BYTES: usize = 0x1000; +const UNIT_COUNT: usize = BLOCK_COUNT * BLOCK_BYTES / UNIT_BYTES; +pub(super) const CAPACITY_BYTES: u64 = (BLOCK_COUNT * BLOCK_BYTES) as u64; + +pub(super) struct M5GraphicsUsc { + directory: MappedObject, + freelist: MappedObject, + blocks: KVec, +} + +impl M5GraphicsUsc { + pub(super) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + Self::new_in_range(dev, vm, vm.kernel_range()) + } + + pub(super) fn new_application(dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let range = crate::m5::vm_layout::auxiliary_range(&vm.kernel_range()).ok_or(EINVAL)?; + Self::new_in_range(dev, vm, range) + } + + fn new_in_range( + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: core::ops::Range, + ) -> Result { + let allocate = |size, guard_pages| MappedObject::new_wc_in_range_with_guards( + dev, vm, range.clone(), size, 0x8000, mmu::PROT_GPU_SHARED_RW, guard_pages, + ); + let mut directory = allocate(DIRECTORY_ENTRIES * 8, 2)?; + let mut freelist = allocate(mmu::UAT_PGSZ, 1)?; + let mut blocks = KVec::new(); + for _ in 0..BLOCK_COUNT { + blocks.push(allocate(BLOCK_BYTES, 2)?, GFP_KERNEL)?; + } + directory.with_bytes_mut(|out| { + for unit in 0..UNIT_COUNT { + let address = Self::unit_address(&blocks, unit).ok_or(EINVAL)?; + out[unit * 8..unit * 8 + 8].copy_from_slice(&address.to_le_bytes()); + } + Ok(()) + })?; + freelist.with_bytes_mut(|out| { + // Same initial free-block format as the qualified M5 compute + // pool. Every allocation here is free; no native growth history + // or in-use allocator contents are imported. + for block in 0..BLOCK_COUNT { + let address = blocks[block].iova(); + out[block * 0x40..block * 0x40 + 8] + .copy_from_slice(&(address | (1u64 << 61)).to_le_bytes()); + } + Ok(()) + })?; + let mut result = Self { directory, freelist, blocks }; + result.validate(vm)?; + dev_dbg!(dev.as_ref(), + "M5_GRAPHICS_USC reserved={:#x}..{:#x} directory={:#x} freelist={:#x} blocks={} capacity={:#x}; guard pages reserved\n", + range.start, range.end, result.directory.iova(), result.freelist.iova(), + BLOCK_COUNT, CAPACITY_BYTES); + Ok(result) + } + + fn unit_address(blocks: &[MappedObject], unit: usize) -> Option { + if unit >= UNIT_COUNT { return None; } + let units_per_block = BLOCK_BYTES / UNIT_BYTES; + Some(blocks.get(unit / units_per_block)?.iova() + + (unit % units_per_block * UNIT_BYTES) as u64) + } + + pub(super) fn descriptor(&self) -> [u64; 4] { + // M5 halLoadUMADesc: 4-KiB population, not 16-KiB CPU page count. + [self.directory.iova() >> 7, (UNIT_COUNT as u64) << 32, + (DIRECTORY_ENTRIES as u64) << 32 | UNIT_COUNT as u64, 0] + } + + /// Initialize a fresh owner using the qualified M5 initial-state recipe. + /// Native graphics snapshots include later growth generations; copying + /// their seeds/disabled flag would disagree with these all-free blocks. + pub(super) fn encode_initial_owner(&self, inner: u64, out: &mut [u8]) -> Result { + if out.len() < 0x70 || inner < 0xffff_fc00_0000_0000 || inner & 7 != 0 { + return Err(EINVAL); + } + out[..0x70].fill(0); + for (offset, value) in [(0, 1u32), (8, HARDWARE_ID), (0x10, 2), + (0x1c, DIRECTORY_ENTRIES as u32), (0x24, UNIT_COUNT as u32), + (0x2c, UNIT_COUNT as u32), (0x40, 4), (0x48, (BLOCK_COUNT * 8) as u32), + (0x5c, 1), (0x60, 2)] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, address) in [(0x14, self.directory.iova()), + (0x30, self.freelist.iova()), (0x4c, inner)] { + out[offset..offset + 8].copy_from_slice(&address.to_le_bytes()); + } + Ok(()) + } + + /// Map the retained pool into another application VM at the same + /// addresses, reproducing each allocation's guard reservation, so every + /// firmware record naming the directory, free list or blocks stays valid + /// across a render client handoff. The returned aliases must be kept for + /// as long as that VM may submit. + pub(super) fn alias_into(&mut self, vm: &mmu::Vm) -> Result> { + let range = crate::m5::vm_layout::auxiliary_range(&vm.kernel_range()).ok_or(EINVAL)?; + let mut aliases = KVec::new(); + let alias = |object: &mut MappedObject, size: usize, guards: usize, + aliases: &mut KVec| -> Result { + let end = object.iova().checked_add((size + guards * mmu::UAT_PGSZ) as u64) + .ok_or(EOVERFLOW)?; + if object.iova() < range.start || end > range.end { return Err(ERANGE); } + if let Some(mapping) = object.map_alias_at_with_guards( + vm, object.iova(), mmu::PROT_GPU_SHARED_RW, guards)? { + aliases.push(mapping, GFP_KERNEL)?; + } + Ok(()) + }; + alias(&mut self.directory, DIRECTORY_ENTRIES * 8, 2, &mut aliases)?; + alias(&mut self.freelist, mmu::UAT_PGSZ, 1, &mut aliases)?; + for block in self.blocks.iter_mut() { + alias(block, BLOCK_BYTES, 2, &mut aliases)?; + } + // Only the mappings are checked here: the firmware has been + // advancing the free-ring cursors and returning directory units + // while the pool was in use, so the construction-time contents no + // longer hold and must not be re-asserted on handoff. + self.validate_mappings(vm)?; + Ok(aliases) + } + + /// Verify every CPU page named by the populated 4-KiB directory and the + /// free-block heads. The allocator keeps the underlying mappings alive. + pub(super) fn validate(&mut self, vm: &mmu::Vm) -> Result { + self.validate_mappings(vm)?; + self.validate_contents() + } + + /// Range, alignment, guard-gap and translation checks for every pool + /// object in `vm`; valid before and after firmware use of the pool. + fn validate_mappings(&self, vm: &mmu::Vm) -> Result { + if self.blocks.len() != BLOCK_COUNT { return Err(EFAULT); } + let range = vm.kernel_range(); + let validate_guards = |object: &MappedObject, size: usize, guards: usize| -> Result { + let end = object.iova().checked_add((size + guards * mmu::UAT_PGSZ) as u64) + .ok_or(EOVERFLOW)?; + if object.iova() < range.start || end > range.end + || object.iova() & 0x7fff != 0 { return Err(EFAULT); } + // Check after every allocation has completed: a later neighbor + // must not have consumed either reserved 16-KiB guard page. + for guard in 0..guards { + if vm.translate_iova(object.iova() + (size + guard * mmu::UAT_PGSZ) as u64).is_ok() { + return Err(EFAULT); + } + } + Ok(()) + }; + validate_guards(&self.directory, DIRECTORY_ENTRIES * 8, 2)?; + validate_guards(&self.freelist, mmu::UAT_PGSZ, 1)?; + for block in &self.blocks { validate_guards(block, BLOCK_BYTES, 2)?; } + let translate_all = |object: &MappedObject| -> Result { + for offset in (0..object.logical_size).step_by(mmu::UAT_PGSZ) { + vm.translate_iova(object.iova() + offset as u64)?; + } + Ok(()) + }; + translate_all(&self.directory)?; + translate_all(&self.freelist)?; + for block in &self.blocks { translate_all(block)?; } + Ok(()) + } + + /// Construction-time contents: an all-free block list and a directory + /// that names every unit in order. Firmware use changes both. + fn validate_contents(&mut self) -> Result { + for block in 0..BLOCK_COUNT { + let base = self.blocks[block].iova(); + if self.freelist.read_u64(self.freelist.iova() + (block * 0x40) as u64)? + != (base | (1u64 << 61)) { + return Err(EFAULT); + } + } + for unit in 0..UNIT_COUNT { + if self.directory.read_u64(self.directory.iova() + (unit * 8) as u64)? + != Self::unit_address(&self.blocks, unit).ok_or(EFAULT)? { + return Err(EFAULT); + } + } + if self.directory.read_u64(self.directory.iova() + (UNIT_COUNT * 8) as u64)? != 0 { + return Err(EFAULT); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m5/resources/parameter_buffer.rs b/drivers/gpu/drm/asahi/m5/resources/parameter_buffer.rs new file mode 100644 index 0000000000000..3c6b955b6c799 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/parameter_buffer.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! VM-owned parameter buffer for the qualified 256x256 M5 render profile. +//! +//! The allocation addresses are dynamic. Capacity and scene layout remain +//! the witnessed native profile; larger renders and pool growth are separate. + +use super::{graphics_memory::M5GraphicsMemory, MappedObject}; +use crate::{driver::AsahiDrmDevice, mmu, m5::vm_layout}; +use kernel::prelude::*; + +pub(super) const BLOCK_BYTES: usize = 0x80000; +pub(super) const PAGE_LIST_BYTES: usize = 0x1c000; +pub(super) const PAGE_COUNT: u32 = 48; + +pub(super) struct M5ParameterBuffer { + blocks: [MappedObject; 3], + page_list: MappedObject, + metrics: MappedObject, + scene_scratch: MappedObject, + discard: MappedObject, +} + +impl M5ParameterBuffer { + pub(super) fn new_reference(dev: &AsahiDrmDevice, memory: &M5GraphicsMemory) -> Result { + let allocate = |size| memory.allocate_compact( + dev, size, 0x8000, mmu::PROT_GPU_SHARED_RW, + ); + let mut owner = Self { + blocks: [allocate(BLOCK_BYTES)?, allocate(BLOCK_BYTES)?, allocate(BLOCK_BYTES)?], + page_list: allocate(PAGE_LIST_BYTES)?, + metrics: allocate(0x4000)?, + scene_scratch: allocate(0x4000)?, + discard: allocate(0x8000)?, + }; + let block_bases = owner.block_page_bases()?; + owner.page_list.with_bytes_mut(|out| { + for (block, first) in block_bases.into_iter().enumerate() { + for page in 0..16 { + let offset = (block * 16 + page) * 4; + out[offset..offset + 4].copy_from_slice(&(first + page as u32).to_le_bytes()); + } + } + Ok(()) + })?; + owner.validate(memory.vm())?; + dev_dbg!(dev.as_ref(), + "M5_PARAMETER_BUFFER blocks={:x?} page_list={:#x} metrics={:#x} scene={:#x} discard={:#x}; compact mappings owned\n", + owner.blocks.each_ref().map(|b| b.iova()), owner.page_list(), owner.metrics(), + owner.scene_scratch.iova(), owner.discard.iova()); + Ok(owner) + } + + pub(super) fn block_page_bases(&self) -> Result<[u32; 3]> { + let mut bases = [0; 3]; + for (base, block) in bases.iter_mut().zip(self.blocks.iter()) { + let offset = block.iova().checked_sub(vm_layout::DRIVER_BASE).ok_or(ERANGE)?; + if offset & 0x7fff != 0 { return Err(EINVAL); } + *base = u32::try_from(offset >> 15).map_err(|_| ERANGE)?; + } + Ok(bases) + } + + pub(super) fn page_list(&self) -> u64 { self.page_list.iova() } + pub(super) fn metrics(&self) -> u64 { self.metrics.iova() } + pub(super) fn scene_cycle(&self) -> u64 { + self.scene_scratch.iova() - vm_layout::DRIVER_BASE + 0x40 + } + pub(super) fn discard_offset(&self) -> u64 { + self.discard.iova() - vm_layout::DRIVER_BASE + } + + /// Firmware and application views must reference the same backing. + pub(super) fn firmware_aliases( + &mut self, uat: &mmu::Uat, vm: &mmu::Vm, page_list_high: u64, metrics_high: u64, + ) -> Result> { + let mut aliases = KVec::new(); + for (object, high) in [(&mut self.page_list, page_list_high), (&mut self.metrics, metrics_high)] { + let alias = object.map_alias_at(uat.kernel_vm(), high, mmu::PROT_FW_SHARED_RW)?; + for offset in (0..object.logical_size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset as u64)? + != vm.translate_iova(object.iova() + offset as u64)? { + return Err(EFAULT); + } + } + aliases.push(alias, GFP_KERNEL)?; + } + Ok(aliases) + } + + /// Map the retained pool into another application VM at the same compact + /// addresses with the original two-page guard reservations, so the + /// firmware's parameter-manager records keep naming valid pages after a + /// render client handoff. + pub(super) fn alias_into(&mut self, vm: &mmu::Vm) -> Result> { + let range = vm_layout::compact_range(&vm.kernel_range()).ok_or(EINVAL)?; + let mut aliases = KVec::new(); + let alias = |object: &mut MappedObject, aliases: &mut KVec| -> Result { + let end = object.iova() + .checked_add((object.logical_size + 2 * mmu::UAT_PGSZ) as u64) + .ok_or(EOVERFLOW)?; + if object.iova() < range.start || end > range.end { return Err(ERANGE); } + if let Some(mapping) = object.map_alias_at_with_guards( + vm, object.iova(), mmu::PROT_GPU_SHARED_RW, 2)? { + aliases.push(mapping, GFP_KERNEL)?; + } + Ok(()) + }; + for block in self.blocks.iter_mut() { alias(block, &mut aliases)?; } + alias(&mut self.page_list, &mut aliases)?; + alias(&mut self.metrics, &mut aliases)?; + alias(&mut self.scene_scratch, &mut aliases)?; + alias(&mut self.discard, &mut aliases)?; + self.validate(vm)?; + Ok(aliases) + } + + fn validate(&self, vm: &mmu::Vm) -> Result { + let check = |object: &MappedObject| -> Result { + let start = object.iova(); + let end = start.checked_add(object.logical_size as u64).ok_or(EOVERFLOW)?; + if start < vm_layout::DRIVER_BASE || end > vm_layout::COMPACT_END + || start & 0x7fff != 0 { return Err(ERANGE); } + for offset in (0..object.logical_size).step_by(mmu::UAT_PGSZ) { + vm.translate_iova(start + offset as u64)?; + } + for page in 0..2 { + if vm.translate_iova(end + page * mmu::UAT_PGSZ as u64).is_ok() { + return Err(EFAULT); + } + } + Ok(()) + }; + for block in &self.blocks { check(block)?; } + for object in [&self.page_list, &self.metrics, &self.scene_scratch, &self.discard] { + check(object)?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m5/resources/target_memory.rs b/drivers/gpu/drm/asahi/m5/resources/target_memory.rs new file mode 100644 index 0000000000000..f0f2c5176018d --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/target_memory.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Driver-owned render targets in the actual submitting VM. + +use super::{graphics_memory::M5GraphicsMemory, MappedObject}; +use crate::{driver::AsahiDrmDevice, m5::render_desc::G17pRenderParameters, mmu, m5::vm_layout, + m5::render_target::{self, M5RenderTargetLayout, PAGE_BYTES, AUX_BYTES}}; +use kernel::prelude::*; + +pub(super) struct M5TargetMemory { + target: MappedObject, + tpc: MappedObject, + deflake: MappedObject, + ta_status: MappedObject, + fragment_status: MappedObject, + auxiliary: MappedObject, + layout: M5RenderTargetLayout, +} + +impl M5TargetMemory { + pub(super) fn new( + dev: &AsahiDrmDevice, memory: &M5GraphicsMemory, p: &G17pRenderParameters, + clusters: u32, + ) -> Result { + let layout = M5RenderTargetLayout::new(p.width, p.height, p.layers, + p.utile_width, p.utile_height, clusters).ok_or(EINVAL)?; + let allocate = |size| memory.allocate_compact( + dev, size, 0x8000, mmu::PROT_GPU_SHARED_RW, + ); + let mut owner = Self { + target: allocate(layout.target_bytes)?, tpc: allocate(layout.tpc_bytes)?, + deflake: allocate(PAGE_BYTES)?, ta_status: allocate(PAGE_BYTES)?, + fragment_status: allocate(PAGE_BYTES)?, auxiliary: allocate(AUX_BYTES)?, layout, + }; + owner.deflake.with_bytes_mut(|bytes| { + render_target::initialize_deflake(bytes, p.width, p.height).ok_or(EINVAL) + })?; + owner.validate(memory.vm())?; + dev_dbg!(dev.as_ref(), + "M5_RENDER_TARGET target={:#x}/{:#x} heapmeta={:#x} tpc={:#x}/{:#x} deflake={:#x} status={:x?} auxiliary={:#x}; VM-owned\n", + owner.target.iova(), layout.target_bytes, owner.target.iova() + layout.heapmeta_offset as u64, + owner.tpc.iova(), layout.tpc_bytes, owner.deflake.iova(), + [owner.ta_status.iova(), owner.fragment_status.iova()], owner.auxiliary.iova()); + Ok(owner) + } + + pub(super) fn apply(&self, p: &mut G17pRenderParameters) { + p.context_base = vm_layout::DRIVER_BASE; + p.tilemap = self.target.iova(); + p.heapmeta = self.target.iova() + self.layout.heapmeta_offset as u64; + p.tpc = self.tpc.iova(); + p.deflake_3 = self.deflake.iova(); + p.deflake_2 = self.deflake.iova() + 0x40; + p.deflake_1 = self.deflake.iova() + 0x340; + p.ta_status = self.ta_status.iova(); + p.fragment_status = self.fragment_status.iova(); + p.aux_fb = self.auxiliary.iova(); + } + + fn validate(&self, vm: &mmu::Vm) -> Result { + for object in [&self.target, &self.tpc, &self.deflake, &self.ta_status, + &self.fragment_status, &self.auxiliary] { + let start = object.iova(); + let end = start.checked_add(object.logical_size as u64).ok_or(EOVERFLOW)?; + if start < vm_layout::DRIVER_BASE || end > vm_layout::COMPACT_END + || start & 0x7fff != 0 { return Err(ERANGE); } + for offset in (0..object.logical_size).step_by(PAGE_BYTES) { + vm.translate_iova(start + offset as u64)?; + } + for page in 0..2 { + if vm.translate_iova(end + page * PAGE_BYTES as u64).is_ok() { return Err(EFAULT); } + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m5/sksm.rs b/drivers/gpu/drm/asahi/m5/sksm.rs new file mode 100644 index 0000000000000..826f1def2c9b4 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/sksm.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8142/25G76 HAL300 SKSM entry and tag-14 serialization. +//! +//! The M5 host clears 0x200 bytes and writes two-bit dependency types at +//! +0x180. Neo's 0x180-byte entry cannot represent this data. These pure +//! builders do not register queues, publish producers or ring a doorbell. +//! Callers must retain every referenced object and implement the M5 lifetime +//! and completion protocol before any generated bytes reach firmware. + +pub(crate) const M5_SKSM_ENTRY_BYTES: usize = 0x200; +pub(crate) const M5_SKSM_ENTRY_COUNT: u32 = 256; + +/// Logical channel selector measured in native TA, 3D and CL tag-14 records. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum M5DataMaster { + Tiling = 0, + Fragment = 1, + Compute = 2, +} +const STAMP_MASK: u64 = (1 << 40) - 1; +const PACKED_ADDRESS_MASK: u64 = (1 << 38) - 1; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5SksmError { + QueueId, + Priority, + Stamp, + QueueFull, + Dependencies, + DependencyType, + Address, + Mcache, + Rce, + Qos, + AddCount, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmDependency { + pub(crate) queue_id: u8, + /// Already converted to the hardware's 32-bit generation / 8-bit index. + pub(crate) stamp: u64, + pub(crate) kind: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmRce { + pub(crate) address: u64, + pub(crate) tag: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmMcache { + pub(crate) address: u64, + pub(crate) index: u8, + pub(crate) count: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmQueue { + pub(crate) id: u8, + /// Host queue +0x1c, not the outer firmware channel's priority. + pub(crate) priority: u8, + pub(crate) current_stamp: u64, + pub(crate) submitted: u32, + pub(crate) previous_stamp: Option, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmOperands<'a> { + /// Explicit dependencies already resolved by their owning event machine. + pub(crate) barriers: &'a [M5SksmDependency], + /// Native convertEventMachineTSToHWTS(command) - 1, for first publication. + /// This must be grounded in the queue's initialized completion state. + pub(crate) first_predecessor: u64, + pub(crate) implicit_dependency_type: u8, + pub(crate) mcache: Option, + pub(crate) flag_bit1: bool, + pub(crate) flag_bit61: bool, + pub(crate) flag_bit63: bool, + pub(crate) payload: [u64; 2], + pub(crate) qos_byte_32: u8, + pub(crate) qos_class_40: u8, + pub(crate) qos_word_48: u16, + pub(crate) event_mask: [u64; 4], + pub(crate) rce_kind: u8, + pub(crate) rce: [M5SksmRce; 4], + /// Native encoder's scalar x5 argument, copied unmodified to +0x178. + pub(crate) auxiliary: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PreparedM5SksmEntry { + pub(crate) bytes: [u8; M5_SKSM_ENTRY_BYTES], + pub(crate) offset: u32, + /// May only replace host state after entry storage and a dmb ish. + pub(crate) queue_after: M5SksmQueue, +} + +fn check_queue(id: u8, priority: u8, stamp: u64) -> Result<(), M5SksmError> { + if id > 0x7f { + return Err(M5SksmError::QueueId); + } + if priority > 3 { + return Err(M5SksmError::Priority); + } + if stamp & !STAMP_MASK != 0 { + return Err(M5SksmError::Stamp); + } + Ok(()) +} + +fn packed_address(address: u64) -> Result { + if address & 31 != 0 || address >> 5 > PACKED_ADDRESS_MASK { + return Err(M5SksmError::Address); + } + Ok(address >> 5) +} + +fn put(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Recovered from M5 encodeKickEntry at 25G76 0xfffffe00089e750c and its +/// encodeDependencyList at 0xfffffe00089e781c. Geometry is deliberately fixed +/// to the independently captured 256 x 512-byte M5 queues. +pub(crate) fn prepare_m5_sksm_entry( + queue: M5SksmQueue, + operands: M5SksmOperands<'_>, +) -> Result { + check_queue(queue.id, queue.priority, queue.current_stamp)?; + if queue.submitted >= M5_SKSM_ENTRY_COUNT { + return Err(M5SksmError::QueueFull); + } + // Native always appends one implicit predecessor after explicit barriers. + if operands.barriers.len() >= 32 { + return Err(M5SksmError::Dependencies); + } + if operands.qos_class_40 > 31 { + return Err(M5SksmError::Qos); + } + if operands.rce_kind > 3 || operands.rce.iter().any(|r| r.tag > 0x7f) { + return Err(M5SksmError::Rce); + } + let mut bytes = [0; M5_SKSM_ENTRY_BYTES]; + let mut header = (queue.current_stamp << 2) + | ((queue.id as u64) << 42) + | ((queue.priority as u64) << 59) + | ((operands.flag_bit1 as u64) << 1) + | ((operands.flag_bit61 as u64) << 61) + | ((operands.flag_bit63 as u64) << 63); + if let Some(mcache) = operands.mcache { + if mcache.index > 63 || mcache.count == 0 || mcache.count > 64 { + return Err(M5SksmError::Mcache); + } + let packed = packed_address(mcache.address)?; + put( + &mut bytes, + 8, + (mcache.index as u64) | (packed << 6) | ((mcache.count as u64 - 1) << 54), + ); + header |= 1 << 58; + } + put(&mut bytes, 0, header); + put(&mut bytes, 0x10, operands.payload[0]); + put(&mut bytes, 0x18, operands.payload[1]); + let predecessor = M5SksmDependency { + queue_id: queue.id, + stamp: queue.previous_stamp.unwrap_or(operands.first_predecessor), + kind: operands.implicit_dependency_type, + }; + let mut kinds = 0; + for (i, dependency) in operands + .barriers + .iter() + .chain(core::iter::once(&predecessor)) + .enumerate() + { + check_queue(dependency.queue_id, 0, dependency.stamp)?; + if dependency.kind > 3 { + return Err(M5SksmError::DependencyType); + } + put( + &mut bytes, + 0x28 + i * 8, + dependency.stamp | ((dependency.queue_id as u64) << 40), + ); + kinds |= (dependency.kind as u64) << (i * 2); + } + let count = operands.barriers.len() + 1; + let valid = if count == 32 { + u32::MAX + } else { + (1u32 << count) - 1 + }; + put( + &mut bytes, + 0x20, + valid as u64 + | ((operands.qos_byte_32 as u64) << 32) + | ((operands.qos_class_40 as u64) << 40) + | ((operands.qos_word_48 as u64) << 48), + ); + put(&mut bytes, 0x180, kinds); + for (i, mask) in operands.event_mask.into_iter().enumerate() { + put(&mut bytes, 0x128 + i * 8, mask); + } + bytes[0x148] = operands.rce_kind; + for (i, rce) in operands.rce.into_iter().enumerate() { + put( + &mut bytes, + 0x150 + i * 8, + packed_address(rce.address)? | ((rce.tag as u64) << 43), + ); + } + put(&mut bytes, 0x178, operands.auxiliary); + Ok(PreparedM5SksmEntry { + bytes, + offset: (queue.current_stamp as u32 & 255) * M5_SKSM_ENTRY_BYTES as u32, + queue_after: M5SksmQueue { + current_stamp: (queue.current_stamp + 1) & STAMP_MASK, + submitted: queue.submitted + 1, + previous_stamp: Some(queue.current_stamp), + ..queue + }, + }) +} + +/// Native CL add-kicks record at the captured submitBuffer publication site. +/// This is a packed 21-byte command, unrelated to the outer 24-byte entry. +pub(crate) fn encode_m5_sksm_add_kicks( + queue_id: u8, + priority: u8, + first_stamp: u64, + count: u16, +) -> Result<[u8; 21], M5SksmError> { + encode_m5_sksm_add_kicks_for( + M5DataMaster::Compute, + queue_id, + priority, + first_stamp, + count, + ) +} + +/// Encode a retained queue's stage explicitly; graphics never uses CL's DM2. +pub(crate) fn encode_m5_sksm_add_kicks_for( + data_master: M5DataMaster, + queue_id: u8, + priority: u8, + first_stamp: u64, + count: u16, +) -> Result<[u8; 21], M5SksmError> { + check_queue(queue_id, priority, first_stamp)?; + if count == 0 || count > 256 { + return Err(M5SksmError::AddCount); + } + let mut bytes = [0; 21]; + bytes[..4].copy_from_slice(&14u32.to_le_bytes()); + bytes[4..6].copy_from_slice(&(queue_id as u16).to_le_bytes()); + bytes[6..8].copy_from_slice(&count.to_le_bytes()); + bytes[8] = priority; + bytes[9..17].copy_from_slice(&first_stamp.to_le_bytes()); + bytes[17..21].copy_from_slice(&(data_master as u32).to_le_bytes()); + Ok(bytes) +} + diff --git a/drivers/gpu/drm/asahi/m5/submission.rs b/drivers/gpu/drm/asahi/m5/submission.rs new file mode 100644 index 0000000000000..c5414cea1ae2d --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/submission.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +fn put_u32_le(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64_le(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// M5 G17G / macOS 25G76 resets six pairs of 128-bit vectors, covering 0xc0 +/// bytes. Its firmware consumes through +0xa7. The common 0x80-byte reset +/// would omit the moved fields and the firmware-visible tail. +pub(crate) const M5_25G76_CHANNEL_STATE_RESET_SIZE: usize = 0xc0; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ChannelStateResetValues { + pub(crate) uncached_state: u64, + pub(crate) command_pointer_ring: u64, + pub(crate) gpu_va_10: u64, + pub(crate) word_48: u32, + /// Opaque host field +0xf8 copied to the unaligned firmware field +0x9c. + pub(crate) gpu_va_9c: u64, +} + +/// Exact reset stores from M5 AGXChannel::resetChannelState at +/// 0xfffffe000897d36c..0xfffffe000897d410 (AGXG17G 353.14, 25G76). +/// Uncached stores are separately shared with apply_g17_uncached_channel_reset. +/// This codec alone does not admit M5 submission or a Linux GPU runtime. +pub(crate) fn encode_m5_25g76_channel_state_reset( + values: M5ChannelStateResetValues, +) -> [u8; M5_25G76_CHANNEL_STATE_RESET_SIZE] { + let mut raw = [0u8; M5_25G76_CHANNEL_STATE_RESET_SIZE]; + put_u64_le(&mut raw, 0x00, values.uncached_state); + put_u64_le(&mut raw, 0x08, values.command_pointer_ring); + put_u64_le(&mut raw, 0x10, values.gpu_va_10); + put_u32_le(&mut raw, 0x24, u32::MAX); + put_u32_le(&mut raw, 0x28, 4); + put_u32_le(&mut raw, 0x44, u32::MAX); + put_u32_le(&mut raw, 0x48, values.word_48); + put_u64_le(&mut raw, 0x9c, values.gpu_va_9c); + raw +} + +/// Logical render prefix, resolved at ioctl admission before asynchronous work. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] +pub(crate) struct G17PRenderOrder { + pub(crate) sequence: u64, + pub(crate) wait_through: Option, +} + diff --git a/drivers/gpu/drm/asahi/m5/vm_layout.rs b/drivers/gpu/drm/asahi/m5/vm_layout.rs new file mode 100644 index 0000000000000..19a6593736112 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/vm_layout.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Application VA contract shared with Mesa's agx_m5_va.h. + +use core::ops::Range; + +pub(crate) const DRIVER_BASE: u64 = 0x10_0000_0000; +pub(crate) const COMPACT_END: u64 = DRIVER_BASE + (1u64 << 32); +pub(crate) const DATA_START: u64 = 0x40_0000_0000; + +/// TA deflake fields encode an unsigned 32-bit offset from DRIVER_BASE. +/// Keep the private render arena inside that aperture and the USC backing +/// after it, within the same VM_CREATE reservation. A client may leave a +/// prefix of the compact aperture for its own command data. Mesa's default +/// reservation still begins at DRIVER_BASE and data begins at DATA_START. +pub(crate) fn compact_range(reserved: &Range) -> Option> { + if reserved.start < DRIVER_BASE || reserved.start >= COMPACT_END + || reserved.start & 0x3fff != 0 || reserved.end <= COMPACT_END + || reserved.end > DATA_START || reserved.end & 0x3fff != 0 { + return None; + } + Some(reserved.start..COMPACT_END) +} + +pub(crate) fn auxiliary_range(reserved: &Range) -> Option> { + compact_range(reserved)?; + Some(COMPACT_END..reserved.end) +} + +/// Command pointers must not reach driver-owned mappings, including when +/// the client leaves application data below its private arena reservation. +pub(crate) fn user_range_contains( + reserved: &Range, address: u64, size: u64, user_end: u64, +) -> bool { + let Some(end) = address.checked_add(size) else { return false; }; + size != 0 && address >= DRIVER_BASE && end <= user_end + && (end <= reserved.start || address >= reserved.end) +} + diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 5641384b51bfe..75c4d684b219f 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -209,7 +209,8 @@ struct drm_asahi_params_global { * * G13-G15 on 14.x report DRM_ASAHI_GPU_HAL_LEGACY. * The exact J613 25G83 runtime and the G17P runtime report - * DRM_ASAHI_GPU_HAL_200. + * DRM_ASAHI_GPU_HAL_200, the M5 (T8142) runtime + * DRM_ASAHI_GPU_HAL_300. */ __u32 gpu_hal_generation; @@ -241,7 +242,7 @@ enum drm_asahi_gpu_hal_generation { */ DRM_ASAHI_GPU_HAL_200 = 200, - /** @DRM_ASAHI_GPU_HAL_300: HAL300 command stream. Never reported by this kernel. */ + /** @DRM_ASAHI_GPU_HAL_300: HAL300 command stream, reported by the M5 (T8142) runtime. */ DRM_ASAHI_GPU_HAL_300 = 300, }; From 593c95979e674c61f877a6d9278e0c728a175730 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Thu, 5 Nov 2026 20:12:36 +0100 Subject: [PATCH 065/117] drm/asahi: m5: Take the device binder and the queue's usage client integration/aurora added a bound-device owner to the platform driver data and a usage client to Gpu::new_queue. Create the owner in the M5 probe path like the other backends and accept the usage client on the M5 queue. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/driver.rs | 4 ++++ drivers/gpu/drm/asahi/m5/drm.rs | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 554a82285c865..003ec58b2f5da 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -464,6 +464,9 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::m5::probe(pdev, &drm, res)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -473,6 +476,7 @@ impl AsahiPlatformDriver { // The M5 firmware session retains its firmware-visible state until reboot. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, diff --git a/drivers/gpu/drm/asahi/m5/drm.rs b/drivers/gpu/drm/asahi/m5/drm.rs index a2a7b9d2679ac..981a8531b71a5 100644 --- a/drivers/gpu/drm/asahi/m5/drm.rs +++ b/drivers/gpu/drm/asahi/m5/drm.rs @@ -167,7 +167,8 @@ impl gpu::Gpu for M5DrmGpu { fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, _ualloc_priv: Arc>, - priority: u32, usc_exec_base: u64) -> Result> { + priority: u32, usc_exec_base: u64, + _usage: Arc) -> Result> { if priority > 3 { return Err(EINVAL); } let window = command::QueueUscWindow { base: usc_exec_base, user_start: vm_layout::DRIVER_BASE, user_end: USER_END, From dd85007cff1b6e5d27f83f0311e735c8c952ae42 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 22:16:41 -0500 Subject: [PATCH 066/117] dt-bindings: apple: describe the T8152 SMC and its mailbox --- .../bindings/mailbox/apple,mailbox.yaml | 21 +++++++++++++++++++ .../devicetree/bindings/mfd/apple,smc.yaml | 2 ++ 2 files changed, 23 insertions(+) diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 071ba0f3a6fed..2f281baae5dca 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -22,6 +22,11 @@ description: properties: compatible: oneOf: + - description: + T8152 SMC post-office mailbox with counted FIFOs and interrupt + masking provided by the interrupt controller. + const: apple,t8152-smc-mailbox + - description: ASC mailboxes are the most common variant found on the M1 used for example for the display controller, the system management @@ -126,6 +131,21 @@ properties: maxItems: 2 allOf: + - if: + properties: + compatible: + contains: + const: apple,t8152-smc-mailbox + then: + properties: + interrupts: + minItems: 2 + maxItems: 2 + interrupt-names: + items: + - const: send-empty + - const: recv-not-empty + - if: properties: compatible: @@ -174,6 +194,7 @@ allOf: - apple,t6021-ane-mailbox - apple,t8112-ane-mailbox - apple,t8140-aop-setup-mailbox + - apple,t8152-smc-mailbox then: properties: interrupts: diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index c90df16e9b974..1fd6820d86748 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -16,6 +16,8 @@ description: properties: compatible: oneOf: + - description: T8152 SMC with read-only hardware monitoring. + const: apple,t8152-smc - items: - enum: - apple,t6020-smc From b66000b9aa5aa99de03208ad56671f3901e18f91 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 22:16:41 -0500 Subject: [PATCH 067/117] mfd: macsmc: support T8152 read-only hardware monitoring --- drivers/mfd/macsmc.c | 13 +++++++++++++ drivers/soc/apple/mailbox.c | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index e88af94489c05..546e5cdbb72ad 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -63,6 +63,11 @@ static const struct mfd_cell apple_smc_devs[] = { static const struct mfd_cell apple_smc_low_gpio_dev = MFD_CELL_OF("macsmc-low-gpio", NULL, NULL, 0, 0, "apple,smc-low-gpio"); +/* T8152 currently exposes read-only telemetry; firmware owns fan control. */ +static const struct mfd_cell apple_smc_telemetry_devs[] = { + MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), +}; + static void apple_smc_read_sram(struct apple_smc *smc, void *buf, size_t size) { if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) @@ -541,6 +546,13 @@ static int apple_smc_probe(struct platform_device *pdev) return dev_err_probe(smc->dev, ret, "Failed to get key count"); smc->key_count = be32_to_cpu(count); + /* Notifications, GPIO and power management are not qualified on T8152. */ + if (of_device_is_compatible(dev->of_node, "apple,t8152-smc")) + return devm_mfd_add_devices(smc->dev, PLATFORM_DEVID_NONE, + apple_smc_telemetry_devs, + ARRAY_SIZE(apple_smc_telemetry_devs), + NULL, 0, NULL); + /* Enable notifications */ apple_smc_write_flag(smc, SMC_KEY(NTAP), true); ret = devm_add_action_or_reset(dev, apple_smc_disable_notifications, smc); @@ -572,6 +584,7 @@ static int apple_smc_probe(struct platform_device *pdev) } static const struct of_device_id apple_smc_of_match[] = { + { .compatible = "apple,t8152-smc" }, { .compatible = "apple,t8103-smc" }, { .compatible = "apple,smc" }, {}, diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 8949e45c8d7b1..c13c03f9d92ab 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -1689,7 +1689,25 @@ static const struct apple_mbox_hw apple_mbox_t6050_apif_ascwrap_v6_hw = { .has_irq_controls = false, }; +/* + * T8152 SMC post-office mailbox: counted FIFOs share the status register. + * Interrupt masking is provided by the interrupt controller, not the mailbox. + */ +static const struct apple_mbox_hw apple_mbox_t8152_smc_hw = { + .counted_fifo = true, + .control_full = GENMASK(11, 8), /* free TX slots */ + .control_empty = GENMASK(3, 0), /* RX occupancy */ + .a2i_control = 0x20, + .a2i_send0 = 0x10, + .a2i_send1 = 0x18, + .i2a_control = 0x20, + .i2a_recv0 = 0x00, + .i2a_recv1 = 0x08, + .has_irq_controls = false, +}; + static const struct of_device_id apple_mbox_of_match[] = { + { .compatible = "apple,t8152-smc-mailbox", .data = &apple_mbox_t8152_smc_hw }, { .compatible = "apple,t6031-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8122-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8152-gfx-ascwrap-v8", .data = &apple_mbox_t8152_gfx_ascwrap_v8_hw }, From d04cd737ed6fea480bdcd2fc391a690219eeadab Mon Sep 17 00:00:00 2001 From: Acelogic Date: Thu, 5 Nov 2026 22:16:41 -0500 Subject: [PATCH 068/117] arm64: dts: apple: enable J873 SMC temperature and fan reporting --- arch/arm64/boot/dts/apple/t8152-j873g.dts | 34 ++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts index 6c2e25a991e61..4bb70a95ec1db 100644 --- a/arch/arm64/boot/dts/apple/t8152-j873g.dts +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -45,7 +45,10 @@ #address-cells = <2>; #size-cells = <2>; ranges; - /* Filled by the loader. */ + smc-firmware@30de00000 { + reg = <3 0x0de00000 0 0x120000>; + no-map; + }; }; memory@10000000000 { @@ -238,6 +241,35 @@ #performance-domain-cells = <0>; }; + smc_mbox: mailbox@30c710080 { + compatible = "apple,t8152-smc-mailbox"; + reg = <3 0x0c710080 0 0x80>; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@30c600000 { + compatible = "apple,t8152-smc"; + reg = <3 0x0c600000 0 0x74000>, + <3 0x0de00000 0 0x120000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + hwmon { + compatible = "apple,smc-hwmon"; + apple,discover-temperatures; + + fan-F0Ac { + apple,key-id = "F0Ac"; + apple,fan-minimum = "F0Mn"; + apple,fan-maximum = "F0Mx"; + label = "Fan"; + }; + }; + }; + dockchannel: serial@30812c000 { compatible = "apple,dockchannel-uart"; reg = <3 0x0812c000 0 0x4000>, From a59a888e5ea494931bb6674d8c4e37119836e2c3 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 6 Nov 2026 01:21:18 -0500 Subject: [PATCH 069/117] dt-bindings: apple: describe T8152 PCIe, DART and GPIO Add the T8152 compatibles used by the J873 N1 path. Describe the interleaved port and PHY register resources of its PCIe host. Signed-off-by: Acelogic --- .../devicetree/bindings/iommu/apple,dart.yaml | 1 + .../devicetree/bindings/pci/apple,pcie.yaml | 47 +++++++++++++------ .../bindings/pinctrl/apple,pinctrl.yaml | 1 + 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 350e077e779b4..401b41f8ff18b 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -45,6 +45,7 @@ properties: - apple,t8132-dart - apple,t8140-dart - apple,t8142-dart + - apple,t8152-dart - const: apple,t8110-dart apple,managed-sids: diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 3a499f802b8f6..d5510bafbef11 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -62,6 +62,7 @@ properties: - const: apple,t6020-pcie - const: apple,t6020-pcie - const: apple,t8140-pcie + - const: apple,t8152-pcie - const: apple,t8132-pcie - const: apple,t8103-pciec - items: @@ -319,20 +320,38 @@ allOf: interrupts: maxItems: 3 else: - properties: - reg-names: - minItems: 3 - items: - - const: config - - const: rc - - const: port0 - - const: port1 - - const: port2 - - const: port3 - - const: phy0 - - const: phy1 - - const: phy2 - - const: phy3 + if: + properties: + compatible: + contains: + const: apple,t8152-pcie + then: + properties: + reg: + minItems: 4 + maxItems: 8 + reg-names: + minItems: 4 + maxItems: 8 + items: + enum: [config, rc, port0, phy0, port1, phy1, port2, phy2] + interrupts: + maxItems: 3 + else: + properties: + reg-names: + minItems: 3 + items: + - const: config + - const: rc + - const: port0 + - const: port1 + - const: port2 + - const: port3 + - const: phy0 + - const: phy1 + - const: phy2 + - const: phy3 patternProperties: "^pci@": properties: diff --git a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml index edae16e33adbd..1c74f8ccc37a2 100644 --- a/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/apple,pinctrl.yaml @@ -26,6 +26,7 @@ properties: - apple,t8122-pinctrl - apple,t8132-pinctrl - apple,t8140-pinctrl + - apple,t8152-pinctrl - const: apple,t8103-pinctrl - items: # Do not add additional SoC to this list. From 8bbc3ed23fb48f3691737c3ba210bd17b55f6aa1 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Fri, 6 Nov 2026 00:52:31 -0500 Subject: [PATCH 070/117] PCI: apple: add the T8152 port initialization sequence Power the reference clock buffer and release the lane PIPE reset after PERST#. Adapt the public J873 host-controller change to the current aurora-wip hardware descriptors without changing other SoC sequences. Adapted from aurora-silicon/linux-aurora commit d85dcbe23a28aa6db3629140510a64b2b046c2df. Signed-off-by: Thomas Glanzmann Signed-off-by: Acelogic --- drivers/pci/controller/pcie-apple.c | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 8664a9fd3b7c4..48453fd72e46b 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -88,6 +88,7 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PHY_LANE_CFG_REFCLK1REQ BIT(1) #define PHY_LANE_CFG_REFCLK0ACK BIT(2) #define PHY_LANE_CFG_REFCLK1ACK BIT(3) +#define PHY_LANE_CFG_REFBUF_OFF BIT(4) #define PHY_LANE_CFG_REFCLKEN (BIT(9) | BIT(10)) #define PHY_LANE_CFG_REFCLKCGEN (BIT(30) | BIT(31)) #define PHY_LANE_CTL 0x00004 @@ -172,6 +173,10 @@ MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); #define PORT_T602X_MSIADDR 0x016c #define PORT_T602X_MSIADDR_HI 0x0170 #define PORT_T602X_PERST 0x082c +/* T8152: lane PIPE reset is released by clearing this after PERST#. */ +#define PORT_T8152_PIPE_RST BIT(16) +#define PORT_T8152_PIPE_RST_STS 0x00a8 +#define PORT_T8152_PIPE_RST_STS_LANES GENMASK(3, 0) #define PORT_T602X_RID2SID 0x3000 #define PORT_T602X_MSIMAP 0x3800 @@ -209,6 +214,13 @@ struct hw_info { * keeps its link while the domain stays on, see apple_pcie_hold_domain(). */ bool s2idle_hold_domain; + /* AppleT8152PCIePort::_initializeRefclkBuffer clears REFBUF_OFF. */ + bool refbuf_off; + /* + * APCIECoreRCGen5Port::enablePortHardware: after PERST#, wait for the + * lane PIPE reset status, then release PORT_T8152_PIPE_RST. + */ + bool pipe_rst; }; static const struct hw_info t8103_hw = { @@ -252,6 +264,20 @@ static const struct hw_info t8140_hw = { .root_bus_only = true, }; +/* T8152 (M6) apcie: T602x port layout, refclk buffer powered down by iBoot. */ +static const struct hw_info t8152_hw = { + .phy_lane_ctl = 0, + .port_msiaddr = PORT_T602X_MSIADDR, + .port_msiaddr_hi = PORT_T602X_MSIADDR_HI, + .port_refclk = 0, + .port_perst = PORT_T602X_PERST, + .port_rid2sid = PORT_T602X_RID2SID, + .port_msimap = PORT_T602X_MSIMAP, + .max_rid2sid = 512, + .refbuf_off = true, + .pipe_rst = true, +}; + static const struct hw_info t8103_pciec_hw = { .port_msiaddr = PORT_MSIADDR, .port_perst = PORT_PERST, @@ -1521,6 +1547,11 @@ static int apple_pcie_setup_refclk(struct apple_pcie *pcie, if (pcie->hw->phy_lane_ctl) rmw_clear(PHY_LANE_CTL_CFGACC, port->phy + pcie->hw->phy_lane_ctl); + if (pcie->hw->refbuf_off) { + rmw_clear(PHY_LANE_CFG_REFBUF_OFF, port->phy + PHY_LANE_CFG); + udelay(1); + } + rmw_set(PHY_LANE_CFG_REFCLKEN, port->phy + PHY_LANE_CFG); if (pcie->hw->port_refclk) @@ -1635,6 +1666,17 @@ static int apple_pcie_setup_link(struct apple_pcie *pcie, for (u32 idx = 0; idx < num_aux_resets; idx++) gpiod_set_value_cansleep(aux_reset[idx], 0); + if (pcie->hw->pipe_rst) { + ret = readl_relaxed_poll_timeout(port->base + PORT_T8152_PIPE_RST_STS, + stat, stat & PORT_T8152_PIPE_RST_STS_LANES, + 1, 20000); + if (ret < 0) { + dev_err(pcie->dev, "port %pOF lanes stuck in PIPE reset\n", np); + return ret; + } + rmw_clear(PORT_T8152_PIPE_RST, port->base + pcie->hw->port_perst); + } + /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ msleep(100); @@ -3941,6 +3983,7 @@ static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t6000-pciec", .data = &t8112_pciec_hw }, /* T8132 currently inherits controller and PHY setup from m1n1. */ { .compatible = "apple,t8132-pcie", .data = &t602x_hw }, + { .compatible = "apple,t8152-pcie", .data = &t8152_hw }, { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, { .compatible = "apple,t8140-pcie", .data = &t8140_hw }, { .compatible = "apple,pcie", .data = &t8103_hw }, From 3d866b1577c62a111c307ac3e1ffdeeddc12cd34 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 6 Nov 2026 01:21:23 -0500 Subject: [PATCH 071/117] arm64: dts: apple: connect J873 N1 to the public RAM boot tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe APCIE port 0, its GPIO reset, MSI window and separate DART streams for the N1 control, Wi-Fi and Bluetooth functions. Retain the Aura RAM handoff and secondary CPU park contract. Resources follow the captured J873g ADT and Thomas Glanzmann’s public t8152-j873-pcie.dts. Ethernet remains outside this change. Signed-off-by: Acelogic --- arch/arm64/boot/dts/apple/t8152-j873-n1.dtsi | 78 ++++++++++++++++++++ arch/arm64/boot/dts/apple/t8152-j873g.dts | 3 +- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/boot/dts/apple/t8152-j873-n1.dtsi diff --git a/arch/arm64/boot/dts/apple/t8152-j873-n1.dtsi b/arch/arm64/boot/dts/apple/t8152-j873-n1.dtsi new file mode 100644 index 0000000000000..ffa33863f782b --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8152-j873-n1.dtsi @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J873g N1 on APCIE port 0. Aura releases the SMC-controlled N1 reset, + * initializes the common PHY and applies the inactive DART's tunables. + * Linux owns port training and the three function-specific DMA streams. + * Resources follow the J873g ADT and the public J873 PCIe bring-up tree. + */ +#include + +/ { + soc { + pinctrl_ap: pinctrl@31a000000 { + compatible = "apple,t8152-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x1a000000 0x0 0x100000>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 224>; + apple,npins = <224>; + interrupt-controller; + #interrupt-cells = <2>; + interrupts = , + , + , + , + , + , + ; + }; + + pcie0_dart: iommu@388000000 { + compatible = "apple,t8152-dart", "apple,t8110-dart"; + reg = <0x3 0x88000000 0x0 0xc000>; + #iommu-cells = <1>; + interrupts = ; + }; + + pcie: pcie@1cb0000000 { + compatible = "apple,t8152-pcie"; + device_type = "pci"; + reg = <0x1c 0xb0000000 0x0 0x1000000>, + <0x3 0x8c000000 0x0 0x4000>, + <0x3 0x88028000 0x0 0x4000>, + <0x3 0x8f010000 0x0 0x4000>; + reg-names = "config", "rc", "port0", "phy0"; + interrupts = ; + msi-controller; + msi-parent = <&pcie>; + msi-ranges = <&aic AIC_IRQ 1879 IRQ_TYPE_EDGE_RISING 32>; + iommu-map = <0x100 &pcie0_dart 1 1>, + <0x101 &pcie0_dart 2 1>, + <0x102 &pcie0_dart 3 1>; + iommu-map-mask = <0xffff>; + bus-range = <0 1>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0x0 0x20000000>, + <0x02000000 0x0 0x80000000 0xb 0x80000000 0x0 0x40000000>; + dma-coherent; + + port00: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 3 GPIO_ACTIVE_LOW>; + bus-range = <1 1>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts index 4bb70a95ec1db..f4ccb05ba0aab 100644 --- a/arch/arm64/boot/dts/apple/t8152-j873g.dts +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -20,7 +20,7 @@ #size-cells = <2>; interrupt-parent = <&aic>; - /* Request the m1n1 RAM-only handoff, without PCIe/DAPF setup. */ + /* Retain the RAM handoff and secondary CPU park contract. */ aurora,minimal-boot; aliases { @@ -282,3 +282,4 @@ }; #include "t8152-j873-gpu.dtsi" +#include "t8152-j873-n1.dtsi" From 0bb074066d0bd6b0ffe05e9b4535f026143635ed Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 6 Nov 2026 01:29:48 -0500 Subject: [PATCH 072/117] selftests: asahi-m4: keep retention harness compatible with no-peer guard The shared bind path now calls UatFirmware::is_no_peer and returns ENOTSUPP. Extract the production predicate and provide the missing error constant so the existing retention and fault-injection assertions compile and execute again. No production code or assertions change. Signed-off-by: Acelogic --- .../selftests/drivers/gpu/asahi-m4/test-mmu-retention.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py index 124314ac83a5d..8c194aa176726 100755 --- a/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py +++ b/tools/testing/selftests/drivers/gpu/asahi-m4/test-mmu-retention.py @@ -20,7 +20,7 @@ def body(source,marker): pre=r''' #![allow(dead_code,unused_variables)] use std::{cell::RefCell,rc::Rc,mem::{ManuallyDrop,size_of},ops::{Deref,DerefMut,Range},num::NonZeroUsize,sync::{Mutex as StdMutex,atomic::{AtomicBool as StdAtomicBool,AtomicU8,AtomicU32,AtomicU64,AtomicUsize,Ordering,fence}}}; -type Result = std::result::Result;const EIO:i32=5;const ETIMEDOUT:i32=110;const EOVERFLOW:i32=75; +type Result = std::result::Result;const EIO:i32=5;const ETIMEDOUT:i32=110;const EOVERFLOW:i32=75;const ENOTSUPP:i32=524; static TRIP_LOAD:AtomicUsize=AtomicUsize::new(0); struct AtomicBool(StdAtomicBool);impl AtomicBool{fn new(v:bool)->Self{Self(StdAtomicBool::new(v))}fn store(&self,v:bool,o:Ordering){self.0.store(v,o);}fn load(&self,o:Ordering)->bool{let n=TRIP_LOAD.load(Ordering::Relaxed);if n!=0&&TRIP_LOAD.fetch_sub(1,Ordering::Relaxed)==1{self.0.store(true,Ordering::Release);}self.0.load(o)}} const UAT_NUM_CTX:usize=64;const UAT_USER_CTX_START:usize=1;const TTBR_VALID:u64=1;const TTBR_ASID_SHIFT:usize=48;const UAT_PGSZ:usize=0x4000; @@ -34,6 +34,8 @@ def body(source,marker): #[derive(Clone,Copy,PartialEq,Eq)]enum UatFirmware{Handoff,PublishedRoots} struct Handoff{lock_ap:AtomicU8,lock_fw:AtomicU8,turn:AtomicU32,cur_slot:AtomicU32}impl Handoff{fn new()->Self{Self{lock_ap:AtomicU8::new(0),lock_fw:AtomicU8::new(0),turn:AtomicU32::new(0),cur_slot:AtomicU32::new(0xffff)}} ''' +# Keep the production firmware predicate in the legacy-firmware harness. +pre=pre.replace('struct Handoff{', 'impl UatFirmware{'+body(mm,'const fn is_no_peer(')+'}\nstruct Handoff{', 1) handoff='\n'.join(body(mm,m) for m in ['fn lock(&self) {\n self.lock_ap', 'fn unlock(&self) {\n self.turn', 'fn current_slot(']) mid=r''' } From bd08b7125e1d7f2a421cb7f83c65d2f62c3af65b Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 6 Nov 2026 01:35:25 -0500 Subject: [PATCH 073/117] selftests: apple-dcp: update existing models for H17G helpers Keep the powerlog, service-rebind and external poweroff harnesses compiling with the H17G display changes. Add the referenced state and identifier, extract the actual panel predicate when available, and reject unexpected internal-panel blanking in the external-link test. Preserve all existing assertions. Signed-off-by: Acelogic --- .../selftests/drivers/gpu/apple-dcp/desktop.py | 4 +++- tools/testing/selftests/drivers/gpu/apple-dcp/edid.py | 1 + .../selftests/drivers/gpu/apple-dcp/lifecycle.py | 11 ++++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py index ca4042931e0cd..f63ff4bcf1f08 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/desktop.py @@ -600,7 +600,9 @@ def powerlog_framing(root): #define __le64 u64 #define EPIC_SUBTYPE_STD_SERVICE 0xc0 enum epic_subtype { REPORT = 0xc0 }; -struct apple_dcp {void *dev;int fw_compat;struct {int iomfb_method_profile;} hw;struct {u32 nits;} brightness;}; +struct work_struct { unsigned scheduled; }; +static void __attribute__((unused)) schedule_work(struct work_struct *w) { w->scheduled++; } +struct apple_dcp {void *dev;int fw_compat;struct {int iomfb_method_profile;} hw;struct {u32 nits,maximum;void *bl_dev;} brightness;struct work_struct bl_register_wq;}; struct apple_epic_service; struct apple_dcp_afkep {struct apple_dcp *dcp;unsigned endpoint;struct apple_epic_service *service;}; struct apple_epic_service_ops {int (*report)(struct apple_epic_service *,enum epic_subtype,const void *,size_t);}; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index 9b949a1a6ff1d..5caff361a38ca 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -62,6 +62,7 @@ def function(file,signature): #define DRM_CONNECTOR_REGISTERED 1 #define DPAVSERV_ENDPOINT_V2 0x2a #define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define DCP_IOMFB_METHODS_H17G 7 #define EPIC_SUBTYPE_STD_SERVICE 0xc0 #define EPIC_SUBTYPE_ANNOUNCE_H17P 0x11 #define to_platform_device(x) (x) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py index 2d7667dfc61e0..f42f8251fee56 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py @@ -741,6 +741,7 @@ def poweroff_replug(root): """An old disconnect's CRTC-off must not release a replacement link.""" poweroff = apple_function(root, 'dcp_poweroff') assert poweroff, 'dcp_poweroff' + panel_predicate = apple_function(root, 'dcp_h17g_panel') or '' iomfb = (root / 'drivers/gpu/drm/apple/iomfb.c').read_text() modeset = block(iomfb, iomfb.index('int dcp_crtc_atomic_modeset(')) assert 'dcp->crtc_unplugs = dcp->typec_unplugs;' in modeset @@ -764,7 +765,12 @@ def poweroff_replug(root): struct completion { int done; }; struct apple_connector { bool connected; }; struct dptx_port { bool enabled, connected, available; void *service; }; +#define DCP_FIRMWARE_H17P 1 +#define DCP_IOMFB_METHODS_H17G 7 +#define DRM_MODE_CONNECTOR_eDP 14 struct apple_dcp { + int fw_compat, connector_type; + struct { int iomfb_method_profile; } hw; void *dev, *avep, *hdmi_hpd; int external_lock, hpd_mutex; struct delayed_work typec_reconnect_wq, placeholder_edid_wq, swap_watchdog_wq; @@ -786,6 +792,9 @@ def poweroff_replug(root): { (void)d; (void)generation; return 0; } static void dcp_external_crtc_released(struct apple_dcp *d) { (void)d; } static bool dcp_uses_soft_dpms(struct apple_dcp *d) { return d->soft_dpms; } +static bool __attribute__((unused)) dcp_has_panel(struct apple_dcp *d) { return !d->typec_output; } +static void __attribute__((unused)) iomfb_blank_h17p(struct apple_dcp *d) +{ (void)d; assert(!"unexpected internal-panel blank in external-link test"); } static int dcp_backlight_dpms(struct apple_dcp *d, bool on) { (void)d; (void)on; return 0; } static void av_service_disconnect(struct apple_dcp *d) @@ -802,7 +811,7 @@ def poweroff_replug(root): { (void)d; (void)why; (void)error; (void)delay; retries++; } static bool gpiod_get_value_cansleep(void *gpio) { (void)gpio; return false; } static void disconnected_hpd_event(struct apple_connector *connector) { (void)connector; } -''' + poweroff + r''' +''' + panel_predicate + '\n' + poweroff + r''' static struct apple_dcp dcp; static struct apple_connector connector; static struct platform_device pdev = { .data = &dcp }; From 298fee3294c65aba26b96c973a9d7cb487b875fd Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 6 Nov 2026 03:38:11 -0500 Subject: [PATCH 074/117] wifi: apple: centauri: allow the firmware to deauthenticate on disconnect The disconnect request sends TLV 328 with a zero timeout. The firmware uses zero to bypass its deauthentication/disassociation transmit path and immediately tears down the link. The old AP can retain the station while Linux associates with another BSSID, leaving cross-AP traffic without a working return path despite a completed WPA handshake. Use the firmware's 500 ms default disconnect budget so it can transmit the management frame and wait for completion before tearing down the link. --- drivers/net/wireless/apple/centauri/centauri_wifi.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c index f0da841721374..1c854cb459020 100644 --- a/drivers/net/wireless/apple/centauri/centauri_wifi.c +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -1259,6 +1259,8 @@ static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 r put_unaligned_le16(reason, payload); put_unaligned_le16(328, payload + 8); put_unaligned_le16(4, payload + 10); + /* Preserve the firmware's default disconnect budget. */ + put_unaligned_le32(500, payload + 12); ret = command(wifi, 131073, payload, sizeof(payload)); connected = wifi->connected; wifi->connected = false; From ec501ab5a184f2ead26a12324ea8453e019c69d5 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 11:54:45 +0100 Subject: [PATCH 075/117] drm/asahi: g18: start stage 10 without a G18GFX01 capture Stage 10 (the DRM render node) loaded asahi/g18-26a428-color-a-v2.bin, a private capture of a macOS render job, only to build the H1 job it then never runs: every application frame clears the encoded command and rebuilds it from Mesa's parameters ("no capture bytes execute"). When the file is absent at stage 10, map the colour-a page layout with zeroed pages (output pages poisoned as before) and skip the H1 encode. The capture is still required by stage 9 and still preferred when present. Tested on the J873g: firmware live, G18_DRM_READY, GLES clear and triangle read back exactly at 64, 256 and 1920 square; sway, alacritty and mpv render on the GPU. Signed-off-by: Thomas Glanzmann (cherry picked from commit 150ba9aa9ce8109a6005a61c3078c3d4fc6e20fe) --- .../gpu/drm/asahi/g18/fw/graphics_payload.rs | 7 ++ drivers/gpu/drm/asahi/g18/initdata/mod.rs | 3 +- .../gpu/drm/asahi/g18/runtime/graphics_job.rs | 1 + .../drm/asahi/g18/runtime/graphics_memory.rs | 74 +++++++++++++------ 4 files changed, 63 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/asahi/g18/fw/graphics_payload.rs b/drivers/gpu/drm/asahi/g18/fw/graphics_payload.rs index 71a82541685e0..5e60e8bf37b84 100644 --- a/drivers/gpu/drm/asahi/g18/fw/graphics_payload.rs +++ b/drivers/gpu/drm/asahi/g18/fw/graphics_payload.rs @@ -78,6 +78,13 @@ const WITNESS_RANGES: [(u64, usize, u32); 25] = [ (0x100_002d_8000, 0xc0000, 4), ]; +/// Every colour-a page (address, captured usage) in file order, for the +/// stage-10 layout that is mapped without a capture file. +pub(crate) fn color_a_pages() -> impl Iterator { + RANGES.iter().flat_map(|&(base, length, usage)| + (0..length).step_by(PAGE_BYTES).map(move |offset| (base + offset as u64, usage))) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Capture { ColorA = 1, ColorB = 2, GearsG1 = 3, FragmentWitness = 4 } diff --git a/drivers/gpu/drm/asahi/g18/initdata/mod.rs b/drivers/gpu/drm/asahi/g18/initdata/mod.rs index 05c0d1c17633e..50b8901e252a4 100644 --- a/drivers/gpu/drm/asahi/g18/initdata/mod.rs +++ b/drivers/gpu/drm/asahi/g18/initdata/mod.rs @@ -424,7 +424,8 @@ impl G18FirmwareMemory { let pool = self.uma_pool.prepare_binding()?; self.uma_pool.load_hardware_descriptor(dev, &mut self.uma, pool, j873_sma_device_words(2, 6)?)?; job.parameter_buffer().load_row(&mut self.pbdesc)?; - job.encode_h1(dev)?; + // Without a capture (stage 10 only) the first application frame encodes. + if !job.layout_only_payload() { job.encode_h1(dev)?; } Ok(pool) } diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs index de7e2cd51f5d2..6080cd3bf4b05 100644 --- a/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs @@ -136,6 +136,7 @@ impl G18GraphicsJob { } pub(crate) fn vm(&self) -> &mmu::Vm { self.program.vm() } + pub(crate) fn layout_only_payload(&self) -> bool { self.program.layout_only_payload() } pub(crate) fn context_id(&self) -> u8 { self.program.context_id() } pub(crate) fn scheduler_record(&self) -> Result { Ok(self.scheduler.binding(0)?.record) } pub(crate) fn queues(&mut self) -> &mut G18GraphicsQueues { &mut self.queues } diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs index 0a9a9504925b2..10c9573166088 100644 --- a/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs @@ -30,49 +30,81 @@ impl G18GraphicsMemory { _ => return Err(EINVAL), }; if !uat.is_t8152_geometry() || mmu::UAT_PGSZ != payload::PAGE_BYTES { return Err(ENODEV); } - let firmware = Firmware::request(name, dev.as_ref().as_ref())?; + let firmware = match Firmware::request(name, dev.as_ref().as_ref()) { + Ok(firmware) => firmware, + // Stage 10 executes no capture bytes: every application command + // is rebuilt from Mesa's parameters. It needs only the colour-a + // page layout, which is mapped here with zeroed contents. + Err(e) if e == ENOENT && profile == 1 + && *crate::module_parameters::g18_initdata_test.value() == 10 => + return Self::layout_only(dev, uat), + Err(e) => return Err(e), + }; let input = payload::GraphicsPayload::parse(firmware.data()).map_err(|_| EINVAL)?; if input.capture() as u32 != profile { return Err(EINVAL); } + let (context, aliases, pages) = Self::map_pages(dev, uat, + (0..input.page_count()).map(|index| input.page(index).map(|page| (page.address, Some(page.bytes)))))?; + let mut command = KVec::new(); + command.extend_from_slice(input.command(), GFP_KERNEL)?; + let owner = Self { context, aliases, vdm_alias: None, ppp_aliased: false, pages, capture: input.capture(), command, + body_digest: input.body_digest(), command_digest: payload::fnv64(input.command()) }; + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), "G18_RENDER_PAYLOAD profile={} pages={} low_aliases={} body_fnv={:#018x} command_fnv={:#018x} context={} roots={:x?}; unpublished\n", + profile, owner.pages.len(), owner.aliases.len(), owner.body_digest, owner.command_digest, + owner.context_id(), owner.roots()); + Ok(owner) + } + + /// The colour-a VM layout without its capture: zeroed pages, no command. + fn layout_only(dev: &AsahiDrmDevice, uat: &mmu::Uat) -> Result { + let (context, aliases, pages) = Self::map_pages(dev, uat, + payload::color_a_pages().map(|(address, _)| Some((address, None))))?; + let owner = Self { context, aliases, vdm_alias: None, ppp_aliased: false, pages, + capture: payload::Capture::ColorA, command: KVec::new(), body_digest: 0, command_digest: 0 }; + fence(Ordering::SeqCst); + dev_info!(dev.as_ref(), "G18_RENDER_PAYLOAD layout-only colour-a pages={} low_aliases={} context={} roots={:x?}; no capture, stage 10 encodes from Mesa only\n", + owner.pages.len(), owner.aliases.len(), owner.context_id(), owner.roots()); + Ok(owner) + } + + /// True when no capture was loaded: there is no H1 command to encode. + pub(crate) fn layout_only_payload(&self) -> bool { self.command.is_empty() } + + fn map_pages<'a>(dev: &AsahiDrmDevice, uat: &mmu::Uat, + sources: impl Iterator)>>) + -> Result<(mmu::G18ExecutionContext, KVec, KVec)> { let vm = uat.new_t8152_compute_fixture_vm()?; let context = uat.prepare_t8152_execution_context(&vm)?; let mut pages = KVec::new(); let mut aliases = KVec::new(); - for index in 0..input.page_count() { - let source = input.page(index).ok_or(EINVAL)?; + for source in sources { + let (address, bytes) = source.ok_or(EINVAL)?; // Captured usage flags are not PTE permissions. These private // fixture mappings have an explicit diagnostic RW policy. - let mut page = MappedObject::new_wc_at(dev, &vm, source.address, + let mut page = MappedObject::new_wc_at(dev, &vm, address, payload::PAGE_BYTES, mmu::PROT_GPU_SHARED_RW)?; page.with_bytes_mut(|out| { - out.copy_from_slice(source.bytes); - if (payload::OUTPUT..payload::OUTPUT + payload::OUTPUT_BYTES).contains(&source.address) { + match bytes { Some(bytes) => out.copy_from_slice(bytes), None => out.fill(0) } + if (payload::OUTPUT..payload::OUTPUT + payload::OUTPUT_BYTES).contains(&address) { out.fill(0xa5); } Ok(()) })?; - if source.address < 0x78000 { - aliases.push(page.map_alias_at(&vm, va::COMPACT_BIAS + source.address, + if address < 0x78000 { + aliases.push(page.map_alias_at(&vm, va::COMPACT_BIAS + address, mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; for delta in [0, va::PAGE - 1] { - if vm.translate_iova(source.address + delta)? - != vm.translate_iova(va::COMPACT_BIAS + source.address + delta)? { + if vm.translate_iova(address + delta)? + != vm.translate_iova(va::COMPACT_BIAS + address + delta)? { return Err(EFAULT); } } } - vm.translate_iova(source.address)?; - vm.translate_iova(source.address + va::PAGE - 1)?; + vm.translate_iova(address)?; + vm.translate_iova(address + va::PAGE - 1)?; pages.push(page, GFP_KERNEL)?; } - let mut command = KVec::new(); - command.extend_from_slice(input.command(), GFP_KERNEL)?; - let owner = Self { context, aliases, vdm_alias: None, ppp_aliased: false, pages, capture: input.capture(), command, - body_digest: input.body_digest(), command_digest: payload::fnv64(input.command()) }; - fence(Ordering::SeqCst); - dev_info!(dev.as_ref(), "G18_RENDER_PAYLOAD profile={} pages={} low_aliases={} body_fnv={:#018x} command_fnv={:#018x} context={} roots={:x?}; unpublished\n", - profile, owner.pages.len(), owner.aliases.len(), owner.body_digest, owner.command_digest, - owner.context_id(), owner.roots()); - Ok(owner) + Ok((context, aliases, pages)) } pub(crate) fn vm(&self) -> &mmu::Vm { self.context.vm() } From e80afbf5e5abc19178a71deb434a044e63f1af98 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 11:54:45 +0100 Subject: [PATCH 076/117] drm/asahi: g18: log application remaps only with g18_kick_log With two clients (a compositor and a video player) every frame of each is a new client to the single application job, so G18_APP_VM and six G18_APP_TARGET lines were printed per frame, about 330 lines a second, synchronously to the boot console. Print them with g18_kick_info! like the other per-kick lines. Signed-off-by: Thomas Glanzmann (cherry picked from commit 79886333e01497b5cd2c5afa20bc1fc58d6e3546) --- drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs index 5a43ad3816e62..3eb58cce499b9 100644 --- a/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_job_application.rs @@ -93,10 +93,10 @@ impl ApplicationJob { self.guard=Some(guard);self.generation=frame.generation;self.retired=false; self.client_frames=if new_client {1} else {self.client_frames.saturating_add(1)}; if remap { - dev_info!(dev.as_ref(),"G18_APP_VM frame={} vm={} context={} target={}x{} aliases={}\n", + crate::g18_kick_info!(dev.as_ref(),"G18_APP_VM frame={} vm={} context={} target={}x{} aliases={}\n", frame.generation,vm.id(),self.job.context_id(),render.target.width,render.target.height,self.user_aliases.len()); for (target,name) in self.job.targets.iter().zip(assembly::TARGET_NAMES) { - dev_info!(dev.as_ref(),"G18_APP_TARGET name={} va={:#x} bytes={:#x} context={}\n", + crate::g18_kick_info!(dev.as_ref(),"G18_APP_TARGET name={} va={:#x} bytes={:#x} context={}\n", name,target.iova(),target.logical_size,self.job.context_id()); } } From cf3e2dc18cae8ab0839aa4b19c06f17cd01935d0 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 12:34:29 +0100 Subject: [PATCH 077/117] dt-bindings: power: apple,pmc: add T8152 The T8152 (M6) PMC has the T8140 voter interface: perf-state floors at agent << 14 and a rails block with the honoured states and per-rail voter enables, with three rails (DCS, SOC, DISP) and higher maxima. Add apple,t8152-pmc and its performance controller, apple,t8152-clpc. Signed-off-by: Thomas Glanzmann (cherry picked from commit afb23328ea94f6ce32b0be7bc4f96f17e47b4097) --- Documentation/devicetree/bindings/power/apple,pmc.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index c94de57c40596..825d68ce51998 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -29,6 +29,7 @@ properties: - apple,t8132-pmc - apple,t8140-pmc - apple,t8142-pmc + - apple,t8152-pmc reg: minItems: 2 @@ -230,6 +231,7 @@ patternProperties: - apple,t8132-clpc - apple,t8140-clpc - apple,t8142-clpc + - apple,t8152-clpc reg: maxItems: 1 From 26f8fc80230b2f0f6dbff067c435526d5d945b16 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 12:34:29 +0100 Subject: [PATCH 078/117] pmdomain/soc: apple: T8152 PMC voters and CLPC iBoot leaves every PMC voter enable at 0 on the J873g, so no floor is honoured and DRAM and fabric stay at their lowest states: one core copies at 6.6 GB/s and the GPU clears 4K in 4.2 ms. The AppleT8152PMGR accessors (26A434) use RegMap 0x1c0 = pmgr reg 46 at 0x302800000: floors at agent << 14, honoured states at 0x1c6000, voter enables at 0x1c8000 + rail * 4 (PMC::getVoterEnablesOffset), rails DCS, SOC and DISP with maxima 7, 4 and 3. That is the T8140 layout with a rails block at 0x1c6000. Add apple,t8152-pmc and apple,t8152-clpc, the latter with per-SoC default state maps (T8140 keeps its maps). With PMP, CLPC and AP enabled: 23.5 GB/s, 4K clear 1.45 ms. Signed-off-by: Thomas Glanzmann (cherry picked from commit e19dcfa40295b9702abe25287b3161e914dd765c) --- drivers/pmdomain/apple/pmc.c | 13 +++++++++++++ drivers/soc/apple/clpc.c | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 881d5ef0b1210..3765e9a775ff7 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -1373,10 +1373,23 @@ static const struct apple_pmc_hw apple_pmc_hw_t6040 = { .ap_agent = 3, }; +/* T8152 uses the T8142 voter interface with its rails at 0x3029c6000. */ +static const struct apple_pmc_hw apple_pmc_hw_t8152 = { + .agent_stride = 0x4000, + .agents = 4, + .rails = 3, + .agent_state = 0x0, + .rail_enable = 0x2000, + .dcs_rail = 0, + .dcs_max = 7, + .ap_agent = 3, +}; + static const struct of_device_id apple_pmc_of_match[] = { { .compatible = "apple,t8132-pmc", .data = &apple_pmc_hw_t8132 }, { .compatible = "apple,t8140-pmc", .data = &apple_pmc_hw_t8140 }, { .compatible = "apple,t8142-pmc", .data = &apple_pmc_hw_t8142 }, + { .compatible = "apple,t8152-pmc", .data = &apple_pmc_hw_t8152 }, { .compatible = "apple,t6041-pmc", .data = &apple_pmc_hw_t6040 }, {} }; diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index b31a46b050358..c7b29f0e6a49b 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -1141,10 +1141,20 @@ static struct kunit_suite apple_clpc_suite = { kunit_test_suite(apple_clpc_suite); #endif +/* T8152: DCS states 0-7 and fabric states 0-4. */ +static const struct apple_clpc_hw apple_clpc_hw_t8152 = { + .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, + .map = { + [APPLE_CLPC_DCS] = { 2, 4, 6, 7 }, + [APPLE_CLPC_SOC] = { 1, 2, 3, 4 }, + }, +}; + static const struct of_device_id apple_clpc_of_match[] = { { .compatible = "apple,t8132-clpc", .data = &apple_clpc_hw_t8132 }, { .compatible = "apple,t8140-clpc", .data = &apple_clpc_hw_t8140 }, { .compatible = "apple,t8142-clpc", .data = &apple_clpc_hw_t8142 }, + { .compatible = "apple,t8152-clpc", .data = &apple_clpc_hw_t8152 }, { .compatible = "apple,t6041-clpc", .data = &apple_clpc_hw_t6040 }, {} }; From ac61317b7116812d3064be2325bf988c2c99dcc7 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 11:34:29 +0000 Subject: [PATCH 079/117] arm64: dts: apple: j873: PMC voters and CLPC Bind the PMC voter interface (pmgr reg 46, rails block at 0x3029c6000) with the PMP, CLPC and AP agents as static voters, and the CLPC. Signed-off-by: Thomas Glanzmann Adapt the PMC node to the public t8152-j873g RAM-boot device tree. (cherry picked from commit 9cac3b9679803baa1b11fcb25607848151c9cc8e) --- arch/arm64/boot/dts/apple/t8152-j873-pmc.dtsi | 27 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8152-j873g.dts | 2 ++ 2 files changed, 29 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8152-j873-pmc.dtsi diff --git a/arch/arm64/boot/dts/apple/t8152-j873-pmc.dtsi b/arch/arm64/boot/dts/apple/t8152-j873-pmc.dtsi new file mode 100644 index 0000000000000..3b809b363b84e --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8152-j873-pmc.dtsi @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT + +/ { + soc { + /* + * Perf-state floors of voter agents 0-3 (PMP, CLPC, GFX, AP) and, + * at 0x1c6000, the honoured states and per-rail voter enables + * (rails DCS, SOC, DISP) in the PMC window, pmgr reg 46. + */ + pmc: power-management@302800000 { + compatible = "apple,t8152-pmc"; + reg = <0x3 0x02800000 0x0 0x10000>, + <0x3 0x029c6000 0x0 0x3000>; + reg-names = "voters", "rails"; + #address-cells = <1>; + #size-cells = <0>; + #cooling-cells = <2>; + /* PMP, CLPC and AP agents */ + apple,static-voters = <0 1 3>; + + clpc: performance-controller@1 { + compatible = "apple,t8152-clpc"; + reg = <1>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts index f4ccb05ba0aab..ec3d591410b3d 100644 --- a/arch/arm64/boot/dts/apple/t8152-j873g.dts +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -283,3 +283,5 @@ #include "t8152-j873-gpu.dtsi" #include "t8152-j873-n1.dtsi" + +#include "t8152-j873-pmc.dtsi" From 6520267240f256535e01f62c35f4fc41913e9125 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 14:40:55 +0100 Subject: [PATCH 080/117] dt-bindings: spi: apple,qspi: add T8152 The T8152 (M6) boot NOR sits behind the same QSPIMC controller as on T8140 (ADT /arm-io/qspi, version 0x20100). Accept apple,t8152-qspi with apple,t8140-qspi as fallback. Signed-off-by: Thomas Glanzmann (cherry picked from commit 9aafd590cd45720a93f8c7a5e19c27adbe30fa30) --- .../devicetree/bindings/spi/apple,qspi.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/spi/apple,qspi.yaml b/Documentation/devicetree/bindings/spi/apple,qspi.yaml index 0b98af1ad68d2..40d08a8273712 100644 --- a/Documentation/devicetree/bindings/spi/apple,qspi.yaml +++ b/Documentation/devicetree/bindings/spi/apple,qspi.yaml @@ -19,11 +19,15 @@ allOf: properties: compatible: - enum: - - apple,t8122-qspi - - apple,t8132-qspi - - apple,t6041-qspi - - apple,t8140-qspi + oneOf: + - enum: + - apple,t8122-qspi + - apple,t8132-qspi + - apple,t6041-qspi + - apple,t8140-qspi + - items: + - const: apple,t8152-qspi + - const: apple,t8140-qspi reg: maxItems: 1 From 96aac9b95abf6770d731c4eb22af0b40796e68e7 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sat, 31 Oct 2026 13:51:19 +0000 Subject: [PATCH 081/117] arm64: dts: apple: t8152-j873: boot NOR with the NVRAM through QSPIMC The J873g's boot NOR (Winbond, JEDEC ef 65 14, 2 MiB) sits behind a QSPIMC controller at 0x331118000 (ADT /arm-io/qspi, version 0x20100), the one spi-apple-qspi drives on T8140. Its PMGR power states (devices SPI_P and QSPI) are off once iBoot is done; touching the controller without them takes an asynchronous SError, so describe them as power domains. The NVRAM v3 banks ("3VVN", 64 KiB each) run from 0x24000 to 0x1f4000, as on T8140 J700; the partition needs 4 KiB erase sectors to be writable (CONFIG_MTD_SPI_NOR_USE_4K_SECTORS). Tested on the J873g: asahi-nvram reads, writes and deletes variables; asahi-bless lists Macintosh HD and LinuxStub, and asahi-bless --next --set-boot "Macintosh HD" boots macOS once, after which the mini boots LinuxStub again. Signed-off-by: Thomas Glanzmann Describe the QSPI controller and its required PMGR domains in the public t8152-j873g device tree, without the unrelated legacy audio nodes. (cherry picked from commit 5164bb8e5e3cb419934fdd91eb9ddd192be3b031) --- .../arm64/boot/dts/apple/t8152-j873-qspi.dtsi | 85 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8152-j873g.dts | 1 + 2 files changed, 86 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8152-j873-qspi.dtsi diff --git a/arch/arm64/boot/dts/apple/t8152-j873-qspi.dtsi b/arch/arm64/boot/dts/apple/t8152-j873-qspi.dtsi new file mode 100644 index 0000000000000..c03fbf04b24e4 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8152-j873-qspi.dtsi @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT + +/* + * The boot NOR behind the QSPIMC controller (ADT /arm-io/qspi/spinor: firmware, + * anvram, syscfg), for asahi-nvram and asahi-bless. The controller's power + * states (PMGR devices SPI_P and QSPI) are off once iBoot is done with it; + * reading it without them takes an SError. + */ +/ { + aliases { + nvram = &nvram; + }; + + /* QSPIMC input clock, as on T8140 (ADT clock-ids 412) */ + clk_200m: clock-200m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_200m"; + }; + + soc { + pmgr: power-management@300700000 { + compatible = "apple,t8152-pmgr", "apple,t8103-pmgr", + "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x3 0x700000 0x0 0x14000>; + }; + + qspi: spi@331118000 { + compatible = "apple,t8152-qspi", "apple,t8140-qspi"; + /* One aperture; the command block starts at offset 0x800. */ + reg = <0x3 0x31118000 0x0 0x4800>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_200m>; + power-domains = <&ps_qspi>; + #address-cells = <1>; + #size-cells = <0>; + + spinor: flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <10000000>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + /* + * 64 KiB NVRAM v3 banks ("3VVN") from 0x24000 + * to 0x1f4000, as on T8140 J700. + */ + nvram: partition@24000 { + label = "nvram"; + reg = <0x24000 0x1d0000>; + }; + }; + }; + }; + }; +}; + +&pmgr { + ps_spi_p: power-controller@d40 { + compatible = "apple,t8152-pmgr-pwrstate", + "apple,t8103-pmgr-pwrstate"; + reg = <0xd40 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + + ps_qspi: power-controller@1140 { + compatible = "apple,t8152-pmgr-pwrstate", + "apple,t8103-pmgr-pwrstate"; + reg = <0x1140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts index ec3d591410b3d..1513a2abbc83f 100644 --- a/arch/arm64/boot/dts/apple/t8152-j873g.dts +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -285,3 +285,4 @@ #include "t8152-j873-n1.dtsi" #include "t8152-j873-pmc.dtsi" +#include "t8152-j873-qspi.dtsi" From 90c347e9255d2f2057b51184ff7767667ce15fae Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 6 Nov 2026 04:24:46 -0500 Subject: [PATCH 082/117] drm/asahi: use current G18 parameter namespace in capture-free path --- drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs | 2 +- drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs index 6080cd3bf4b05..01952223a0593 100644 --- a/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_job.rs @@ -289,7 +289,7 @@ impl G18GraphicsJob { /// Caller has allocated/retained PB and pool owners and filled all native /// fields. In particular a default resource word is not an admitted policy. #[inline(never)] - pub(crate) fn encode(&mut self, dev: &AsahiDrmDevice, inputs: &cmd::G18RenderCommandInputs) -> Result { + pub(crate) fn encode(&mut self, _dev: &AsahiDrmDevice, inputs: &cmd::G18RenderCommandInputs) -> Result { if self.encoded.is_some() || self.controls.ta.resource_word == 0 || self.controls.fragment.resource_word == 0 || inputs.context_id != u32::from(self.context_id()) diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs index 10c9573166088..f07b7204b6ab7 100644 --- a/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs @@ -36,7 +36,7 @@ impl G18GraphicsMemory { // is rebuilt from Mesa's parameters. It needs only the colour-a // page layout, which is mapped here with zeroed contents. Err(e) if e == ENOENT && profile == 1 - && *crate::module_parameters::g18_initdata_test.value() == 10 => + && *crate::g18::params::g18_initdata_test.value() == 10 => return Self::layout_only(dev, uat), Err(e) => return Err(e), }; From 4b3ec237a5379e6b28ce7566ea956e948b34a9eb Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 6 Nov 2026 04:26:19 -0500 Subject: [PATCH 083/117] drm/asahi: read the G18 stage parameter by value --- drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs index f07b7204b6ab7..3175072b950e8 100644 --- a/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs +++ b/drivers/gpu/drm/asahi/g18/runtime/graphics_memory.rs @@ -36,7 +36,7 @@ impl G18GraphicsMemory { // is rebuilt from Mesa's parameters. It needs only the colour-a // page layout, which is mapped here with zeroed contents. Err(e) if e == ENOENT && profile == 1 - && *crate::g18::params::g18_initdata_test.value() == 10 => + && crate::g18::params::g18_initdata_test.value() == 10 => return Self::layout_only(dev, uat), Err(e) => return Err(e), }; From 5a5bcaf81de26aa197e777364811201eb23cd210 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:23:03 +0100 Subject: [PATCH 084/117] soc: apple: mailbox: Add the T8142 (M5) GPU ASCWrap v6 mailboxes The M5's two GPU firmware CPUs (GFX and GFX1) sit at T8140's wrapper and IOP VBAR windows moved up by 0x100000000. Add their role table, the T8142 provider and the compatible; the GPU deferred forever without them. Signed-off-by: Thomas Glanzmann --- .../bindings/mailbox/apple,ascwrap-v6.yaml | 1 + drivers/soc/apple/mailbox.c | 35 ++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml index f060194de4b9d..289c29ba752fb 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -21,6 +21,7 @@ properties: enum: - apple,t6040-ascwrap-v6 - apple,t8140-ascwrap-v6 + - apple,t8142-ascwrap-v6 - apple,t6050-apif-ascwrap-v6 - apple,t6050-ascwrap-v6 - apple,t6050-x1n1-ascwrap-v6 diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 6f0d1f7957ec0..7c7c1e0e84531 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -66,6 +66,8 @@ #define APPLE_T6040_GFX_WRAPPER_BASE 0x28a600000ULL #define APPLE_T6040_GFX_IOP_VBAR_BASE 0x28a050000ULL #define APPLE_T6050_ASCWRAP_BASE_DELTA 0x380000000ULL +/* T8142 (M5) moves T8140's 0x4... GPU ASC bank to 0x5.... */ +#define APPLE_T8142_ASCWRAP_BASE_DELTA 0x100000000ULL static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); @@ -825,7 +827,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->ascwrap_v6.wrapper_size = resource_size(res); mbox->ascwrap_v6.iop_vbar_start = iop_vbar_res->start; mbox->ascwrap_v6.iop_vbar_size = resource_size(iop_vbar_res); - /* T8140 and T6040 implement both the CPU start and the CPU stop. */ + /* T8140, T8142 and T6040 implement both the CPU start and stop. */ mbox->ascwrap_v6.missing = 0; mbox->wrapper_regs = @@ -978,6 +980,15 @@ static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t8140_ascwrap_v6_ APPLE_T8140_GFX1_WRAPPER_BASE, APPLE_T8140_GFX1_IOP_VBAR_BASE }, }; +static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t8142_ascwrap_v6_roles[] = { + { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + APPLE_T8140_GFX_WRAPPER_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA, + APPLE_T8140_GFX_IOP_VBAR_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA }, + { "GFX1", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, + APPLE_T8140_GFX1_WRAPPER_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA, + APPLE_T8140_GFX1_IOP_VBAR_BASE + APPLE_T8142_ASCWRAP_BASE_DELTA }, +}; + /* The T6040 runs a single GPU firmware instance. */ static const struct apple_mbox_ascwrap_v6_role_desc apple_mbox_t6040_ascwrap_v6_roles[] = { { "GFX", APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, @@ -1004,6 +1015,27 @@ static const struct apple_mbox_hw apple_mbox_t8140_ascwrap_v6_hw = { .has_irq_controls = false, }; +/* T8142 (M5): T8140's two GPU firmware CPUs at the 0x5... bank. */ +static const struct apple_mbox_hw apple_mbox_t8142_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, + .ascwrap_v6_roles = apple_mbox_t8142_ascwrap_v6_roles, + .num_ascwrap_v6_roles = ARRAY_SIZE(apple_mbox_t8142_ascwrap_v6_roles), + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .has_irq_controls = false, +}; + static const struct apple_mbox_hw apple_mbox_t6040_ascwrap_v6_hw = { .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, .is_ascwrap_v6 = true, @@ -1117,6 +1149,7 @@ static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t8122-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t6040-ascwrap-v6", .data = &apple_mbox_t6040_ascwrap_v6_hw }, { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_t8140_ascwrap_v6_hw }, + { .compatible = "apple,t8142-ascwrap-v6", .data = &apple_mbox_t8142_ascwrap_v6_hw }, { .compatible = "apple,t6050-apif-ascwrap-v6", .data = &apple_mbox_t6050_apif_ascwrap_v6_hw }, { .compatible = "apple,t6050-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, { .compatible = "apple,t6050-x1n1-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, From 436f553431fa74f1ca24d9b2f4a6ec0078ae2f8a Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:24:16 +0100 Subject: [PATCH 085/117] dt-bindings: power: apple,pmc: Add T8142 PMC and CLPC compatibles Signed-off-by: Thomas Glanzmann --- Documentation/devicetree/bindings/power/apple,pmc.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index a3728ed3effee..c94de57c40596 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -28,6 +28,7 @@ properties: - apple,t6041-pmc - apple,t8132-pmc - apple,t8140-pmc + - apple,t8142-pmc reg: minItems: 2 @@ -228,6 +229,7 @@ patternProperties: - apple,t6041-clpc - apple,t8132-clpc - apple,t8140-clpc + - apple,t8142-clpc reg: maxItems: 1 From 64f488e4246e401db68adf2954d668d27854e6bd Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:24:16 +0100 Subject: [PATCH 086/117] pmdomain: apple: pmc: Add T8142 (M5) The AppleT8142PMC accessors put the floors at agent << 14, the per-agent states at 0x106000 and the static voter enables at 0x108000, as on T8140. It has three rails, DCS, SOC and DISP (ADT voter-rail-data), and clamps a floor to DCS 7 and SOC 4. iBoot leaves every rail enable at 0, so without the PMC no vote is honoured and DRAM and the fabric stay at their lowest states (CPU DRAM latency 171 ns instead of 70 ns). Signed-off-by: Thomas Glanzmann --- drivers/pmdomain/apple/pmc.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index b6b5ccdb3e53e..881d5ef0b1210 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -1337,6 +1337,23 @@ static const struct apple_pmc_hw apple_pmc_hw_t8140 = { .ap_agent = 3, }; +/* + * T8142: the AppleT8142PMC accessors put the floors at agent << 14, the + * per-agent states at 0x106000 and the static voter enables at 0x108000; + * rails DCS, SOC and DISP (ADT voter-rail-data). The PMC clamps a floor to + * DCS 7 and SOC 4. + */ +static const struct apple_pmc_hw apple_pmc_hw_t8142 = { + .agent_stride = 0x4000, + .agents = 4, + .rails = 3, + .agent_state = 0x0, + .rail_enable = 0x2000, + .dcs_rail = 0, + .dcs_max = 7, + .ap_agent = 3, +}; + /* * T6040 (M4 Pro): the T8140 layout, with DCS states 0-5 and fabric states * 0-2. The ADT's voter-agent rail mask is 0x7 (DCS, SOC and DISP_INT). @@ -1359,6 +1376,7 @@ static const struct apple_pmc_hw apple_pmc_hw_t6040 = { static const struct of_device_id apple_pmc_of_match[] = { { .compatible = "apple,t8132-pmc", .data = &apple_pmc_hw_t8132 }, { .compatible = "apple,t8140-pmc", .data = &apple_pmc_hw_t8140 }, + { .compatible = "apple,t8142-pmc", .data = &apple_pmc_hw_t8142 }, { .compatible = "apple,t6041-pmc", .data = &apple_pmc_hw_t6040 }, {} }; From f07087c9d563c471171b7386461a3593ba382c65 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:24:29 +0100 Subject: [PATCH 087/117] soc: apple: clpc: Add T8142 (M5) with the GPU input T8142's CLPC drives DCS states 0-7 on rail 0 and fabric states 0-4 on rail 1; the map is the one measured on J813 (four P cores copy at ~40 GB/s with the fabric at 1 and ~110 GB/s at 4). The M5 GPU driver reports its submissions in flight through apple_clpc_gpu_get(), as on T6040, but the CLPC allocated that input only for the t6041 compatible or a gpu_dense table, so on T8142 the GPU waited for it forever. Make it a per-SoC flag, gpu_input, set for T6040 and T8142, and let apple_clpc_gpu_get() accept the T8142 CLPC. Signed-off-by: Thomas Glanzmann --- drivers/soc/apple/clpc.c | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/clpc.c b/drivers/soc/apple/clpc.c index ae0d6d48bac7f..b31a46b050358 100644 --- a/drivers/soc/apple/clpc.c +++ b/drivers/soc/apple/clpc.c @@ -78,6 +78,9 @@ enum apple_clpc_rail { * @pmp_agent, limits the DCS, see apple_clpc_gpu_limited() * @gpu_dense: submissions per sampling period that make light GPU work * dense, see apple_clpc_gpu_dense(); 0 for none + * @gpu_input: the GPU driver reports its work in flight through + * apple_clpc_gpu_get(); while there is some, both rails go to the top + * of @map */ struct apple_clpc_hw { unsigned int rail[APPLE_CLPC_RAILS]; @@ -89,6 +92,7 @@ struct apple_clpc_hw { bool pmp_backoff; unsigned int pmp_agent; unsigned int gpu_dense; + bool gpu_input; }; struct apple_clpc_cluster { @@ -228,7 +232,8 @@ int apple_clpc_gpu_get(struct device *dev, struct apple_clpc_gpu **out) return of_find_property(dev->of_node, "apple,clpc", NULL) ? -EINVAL : 0; if (!of_device_is_compatible(np, "apple,t6041-clpc") && !of_device_is_compatible(np, "apple,t8132-clpc") && - !of_device_is_compatible(np, "apple,t8140-clpc")) { + !of_device_is_compatible(np, "apple,t8140-clpc") && + !of_device_is_compatible(np, "apple,t8142-clpc")) { of_node_put(np); return -EINVAL; } @@ -744,13 +749,11 @@ static int apple_clpc_probe(struct platform_device *pdev) clpc->pmc = dev->parent; clpc->hw = of_device_get_match_data(dev); /* T6040 firmware keeps GFX powered for its whole lifetime. A separate - * host publication/idle input, not that supplier state, drives demand. - * On T8140 the same input counts submissions for dense light work. The - * T8132 GPU driver reports its work through it as well. + * host publication/idle input, not that supplier state, drives demand; + * the T8132 and M5 (T8142) GPU drivers report through the same input. + * On T8140 the same input counts submissions for dense light work. */ - if (of_device_is_compatible(dev->of_node, "apple,t6041-clpc") || - of_device_is_compatible(dev->of_node, "apple,t8132-clpc") || - clpc->hw->gpu_dense) { + if (clpc->hw->gpu_input || clpc->hw->gpu_dense) { clpc->gpu = kzalloc_obj(*clpc->gpu); if (!clpc->gpu) return -ENOMEM; @@ -882,6 +885,7 @@ static const struct apple_clpc_hw apple_clpc_hw_t8132 = { [APPLE_CLPC_SOC] = { 3, 3, 3, 3 }, }, .gpu_demand = false, + .gpu_input = true, }; /* @@ -948,6 +952,26 @@ static const struct apple_clpc_hw apple_clpc_hw_t6040 = { }, .sustain = 3, .gpu_demand = false, + .gpu_input = true, +}; + +/* + * T8142: DCS states 0-7 on rail 0, fabric states 0-4 on rail 1. The fabric + * state bounds CPU memory bandwidth: four P cores copy at ~40 GB/s with the + * fabric at 1 and at ~110 GB/s at 4 (macOS ~124). + * + * The M5 GPU driver reports its submissions in flight. Once the PMP runs, its + * vote (agent 0) drops from iBoot's DCS 7 / fabric 1 to 0 and nothing else + * raises the floor for the GPU: a 4K RGBA16F fill then takes 5.1 ms instead + * of 0.9 ms with both rails at the top of the map. + */ +static const struct apple_clpc_hw apple_clpc_hw_t8142 = { + .rail = { [APPLE_CLPC_DCS] = 0, [APPLE_CLPC_SOC] = 1 }, + .map = { + [APPLE_CLPC_DCS] = { 2, 4, 6, 7 }, + [APPLE_CLPC_SOC] = { 1, 2, 3, 4 }, + }, + .gpu_input = true, }; #if IS_BUILTIN(CONFIG_KUNIT) && !defined(MODULE) @@ -1120,6 +1144,7 @@ kunit_test_suite(apple_clpc_suite); static const struct of_device_id apple_clpc_of_match[] = { { .compatible = "apple,t8132-clpc", .data = &apple_clpc_hw_t8132 }, { .compatible = "apple,t8140-clpc", .data = &apple_clpc_hw_t8140 }, + { .compatible = "apple,t8142-clpc", .data = &apple_clpc_hw_t8142 }, { .compatible = "apple,t6041-clpc", .data = &apple_clpc_hw_t6040 }, {} }; From b8c71bb7f258ddb69ad9d6ae68a51b5c23ff0116 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:25:03 +0100 Subject: [PATCH 088/117] soc: apple: pmp: Add T8142 (M5) The J813 PMP runs the same registry-v2 firmware as T8140. Its DART window starts at 1 TiB, so it takes T8140's 42-bit DMA mask, and like T6041 it starts endpoint 7 (CoreAnalytics) and takes its 8 KiB buffer before it acknowledges the IOP power state. Signed-off-by: Thomas Glanzmann --- drivers/soc/apple/pmp.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index c239fae558cee..2212509efd82c 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -692,6 +692,15 @@ const HW_CFG_T6041: PmpHwConfig = PmpHwConfig { coreanalytics: Ep7::Always, }; +/// T8142: the DART window starts at 1 TiB as on T8140. Its registry-v2 +/// firmware starts endpoint 7 and takes its 8 KiB buffer before it +/// acknowledges the IOP power state. +const HW_CFG_T8142: PmpHwConfig = PmpHwConfig { + dma_mask: Some(DmaMask::new::<42>()), + check_bootargs_uuid: false, + coreanalytics: Ep7::Always, +}; + const HW_CFG_T8132: PmpHwConfig = PmpHwConfig { dma_mask: None, check_bootargs_uuid: false, @@ -711,6 +720,7 @@ kernel::of_device_table!( [ (of::DeviceId::new(c"apple,t8132-pmp-v2"), &HW_CFG_T8132), (of::DeviceId::new(c"apple,t8140-pmp-v2"), &HW_CFG_T8140), + (of::DeviceId::new(c"apple,t8142-pmp-v2"), &HW_CFG_T8142), (of::DeviceId::new(c"apple,t6041-pmp-v2"), &HW_CFG_T6041), (of::DeviceId::new(c"apple,t6000-pmp-v2"), &HW_CFG_DEFAULT), ] From 17faf5087a8a30fe8fb32bafab05578a59d28105 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:21:37 +0100 Subject: [PATCH 089/117] arm64: dts: apple: t8142-j813: E and P states up to the ADT's top levels The J813 ADT lists E-cluster levels up to 2964 MHz (8) and P-cluster levels 11-20 above the 3516 MHz this tree stopped at. Add them; level 19 (4416 MHz) is below level 18 and left out. The SoC throttles the top levels when hot. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813.dts | 77 ++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 576c4637f37df..61f77d206af7f 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -180,6 +180,36 @@ opp-hz = /bits/ 64 <972000000>; opp-level = <2>; }; + + opp-1152000000 { + opp-hz = /bits/ 64 <1152000000>; + opp-level = <3>; + }; + + opp-1584000000 { + opp-hz = /bits/ 64 <1584000000>; + opp-level = <4>; + }; + + opp-1992000000 { + opp-hz = /bits/ 64 <1992000000>; + opp-level = <5>; + }; + + opp-2352000000 { + opp-hz = /bits/ 64 <2352000000>; + opp-level = <6>; + }; + + opp-2700000000 { + opp-hz = /bits/ 64 <2700000000>; + opp-level = <7>; + }; + + opp-2964000000 { + opp-hz = /bits/ 64 <2964000000>; + opp-level = <8>; + }; }; pcluster_opp: opp-table-1 { @@ -235,6 +265,53 @@ opp-hz = /bits/ 64 <3516000000>; opp-level = <10>; }; + + /* ADT levels 11-20, measured on J813 (level 19, 4416 MHz, is below + * level 18 and left out). The SoC throttles them when hot. */ + opp-3720000000 { + opp-hz = /bits/ 64 <3720000000>; + opp-level = <11>; + }; + + opp-3876000000 { + opp-hz = /bits/ 64 <3876000000>; + opp-level = <12>; + }; + + opp-4020000000 { + opp-hz = /bits/ 64 <4020000000>; + opp-level = <13>; + }; + + opp-4176000000 { + opp-hz = /bits/ 64 <4176000000>; + opp-level = <14>; + }; + + opp-4248000000 { + opp-hz = /bits/ 64 <4248000000>; + opp-level = <15>; + }; + + opp-4272000000 { + opp-hz = /bits/ 64 <4272000000>; + opp-level = <16>; + }; + + opp-4332000000 { + opp-hz = /bits/ 64 <4332000000>; + opp-level = <17>; + }; + + opp-4428000000 { + opp-hz = /bits/ 64 <4428000000>; + opp-level = <18>; + }; + + opp-4464000000 { + opp-hz = /bits/ 64 <4464000000>; + opp-level = <20>; + }; }; clkref: clock-ref { From 28f7d9a9cfd7a6d944d3bd492ebe9a1be883c539 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:21:50 +0100 Subject: [PATCH 090/117] arm64: dts: apple: t8142-j813: PMC voters and the CLPC The T8142 PMC (AppleT8142PMC, ADT pmc-pmgr 31 + 0x700000) takes DRAM and fabric floors per voter agent: PMP 0, CLPC 1, GFX 2, AP 3. iBoot leaves every rail enable at 0, so no vote is honoured and DRAM and the fabric stay at their lowest states (CPU DRAM latency 171 ns instead of 70 ns). Describe the PMC with its static voters and the CLPC agent. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813.dts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 61f77d206af7f..f1378b8651878 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -339,6 +339,30 @@ #performance-domain-cells = <0>; }; + /* + * PMC voter interface (AppleT8142PMC, ADT pmc-pmgr 31 + 0x700000): + * floors at 0x382700000 + agent * 0x4000, per-agent states at + * +0x106000, rail enables at +0x108000. Agents (ADT voter-agent): + * PMP 0, CLPC 1, GFX 2, AP 3. iBoot leaves every enable at 0, so + * no vote is honoured and DRAM and fabric stay at their lowest + * states (CPU DRAM latency 171 ns instead of 70 ns). + */ + pmc: power-management@382700000 { + compatible = "apple,t8142-pmc"; + reg = <0x3 0x82700000 0x0 0x10000>, + <0x3 0x82806000 0x0 0x3000>; + reg-names = "voters", "rails"; + #address-cells = <1>; + #size-cells = <0>; + /* PMP, CLPC and AP agents */ + apple,static-voters = <0 1 3>; + + clpc: performance-controller@1 { + compatible = "apple,t8142-clpc"; + reg = <1>; + }; + }; + aic: interrupt-controller@381000000 { compatible = "apple,t8142-aic3", "apple,t8122-aic3"; interrupt-controller; From be8bd67a644d6f72d0ff303ebcb5fc8320248143 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:21:50 +0100 Subject: [PATCH 091/117] arm64: dts: apple: t8142-j813: Describe the panel The 13.6" 2560x1664 panel (1600 lines below the notch) with a 500 nit peak (ADT backlight known-nits). The panel node makes the connector eDP and gives the DCP a backlight to drive. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-display.dtsi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi index 9bd549a5784cc..d9e890af7fd44 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi @@ -91,5 +91,18 @@ iommus = <&disp0_dart 16>; phandle = <&disp0_piodma>; }; + + /* + * 13.6" 2560x1664 (1600 below the notch); 500 nits peak (ADT + * backlight known-nits). Makes the connector eDP and gives the + * DCP a backlight to drive. + */ + panel: panel { + compatible = "apple,panel-j813", "apple,panel"; + width-mm = <290>; + height-mm = <188>; + adj-height-mm = <181>; + apple,max-brightness = <500>; + }; }; }; From 244310db5766ba3bacad8075e67a01ebf55b1057 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:21:50 +0100 Subject: [PATCH 092/117] arm64: dts: apple: t8142-j813: QSPI NVRAM, the PMP and the trackpad reset - The pmgr block is 0x14000 long (the ISP and keyboard backlight power states sit beyond 0x4000). - The QSPI NOR flash with the NVRAM partitions (asahi-nvram, asahi-bless), its 200 MHz clock and the spi_p/qspi power states. - The PMP (DRAM and fabric DVFS coprocessor, registry-v2 firmware) with its DART, mailbox, power states and the J813 tunables. - The trackpad's AFE reset line (SMC GPIO 28, active low) instead of a firmware-managed reset. Signed-off-by: Thomas Glanzmann --- .../dts/apple/t8142-j813-peripherals.dtsi | 145 +++++++++++++++++- .../dts/apple/t8142-j813-pmp-tunables.dtsi | 70 +++++++++ 2 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-pmp-tunables.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi index a6708b1c972df..5ca6fed62568a 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi @@ -1,10 +1,20 @@ // SPDX-License-Identifier: GPL-2.0+ OR MIT +#include #include / { aliases { mtp = &mtp; keyboard = &keyboard; + nvram = &nvram; + }; + + /* J813 ADT /arm-io/qspi clock-ids [396]; rate as the T8140 QSPI. */ + clk_qspi: clock-qspi { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_qspi"; }; led-controller { @@ -24,7 +34,8 @@ &{/soc} { pmgr: power-management@380700000 { compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; - reg = <3 0x80700000 0 0x4000>; + /* As on T8140: up to the ISP17a slots at +0x4000..+0x4060. */ + reg = <3 0x80700000 0 0x14000>; #address-cells = <1>; #size-cells = <1>; @@ -35,6 +46,122 @@ #reset-cells = <0>; label = "fpwm0"; }; + + /* ADT pmgr devices: SPI_P id 94 (+0x390), QSPI id 120 (+0x460). */ + ps_spi_p: power-controller@390 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + }; + + /* ADT pmgr devices: PMP id 159 (+0x598), PMS_SRAM id 160 (+0x5a0). */ + ps_pmp: power-controller@598 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x598 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmp"; + /* The PMP's SRAM, which the driver reads first. */ + power-domains = <&ps_pms_sram>; + }; + + ps_pms_sram: power-controller@5a0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_sram"; + /* T8140 holds it through its always-on PMP report node. */ + apple,always-on; + }; + + ps_qspi: power-controller@460 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x460 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + }; + }; + + /* + * PMP, laid out as on T8140 (ADT /arm-io/pmp, dart-pmp): it runs the + * DRAM controller (DCS) and fabric DVFS. Without it the memory stays at + * the bootloader's state (memcpy ~20 GB/s against ~120 GB/s on macOS). + */ + pmp_dart: iommu@380300000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x3 0x80300000 0x0 0xc000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = /bits/ 64 <0x10000000000 0xbff00000>; + power-domains = <&ps_pmp>; + }; + + pmp: pmp@380500000 { + compatible = "apple,t8142-pmp-v2", "apple,t8140-pmp-v2"; + reg = <0x3 0x80500000 0x0 0xd0000>, + <0x3 0x80e00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + /* ADT /chosen board-id, dram-vendor-id */ + apple,board-id = <36>; + apple,dram-vendor-id = <1>; + /* ADT /arm-io/pmgr pmc, pmc-pmgr (registry-v2 boot arguments) */ + apple,pmc = <1>; + apple,pmc-pmgr = <31>; + }; + + pmp_mbox: mailbox@380e08000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x80e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>; + }; + + /* ADT /arm-io/qspi: 0x195118000 and its FIFO window at +0x800, IRQ 1245. */ + qspi: spi@3a5118000 { + compatible = "apple,t8142-qspi", "apple,t8140-qspi"; + reg = <0x3 0xa5118000 0x0 0x4800>; + clocks = <&clk_qspi>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_qspi>; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + /* Conservative until the input clock is confirmed. */ + spi-max-frequency = <10000000>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + /* ADT spinor/anvram reg: 0x100000 bytes at 0x700000. */ + nvram: partition@700000 { + label = "nvram"; + reg = <0x700000 0x100000>; + }; + }; + }; }; fpwm0: pwm@3a5040000 { @@ -66,6 +193,12 @@ compatible = "apple,smc-hwmon"; apple,discover-temperatures; }; + + smc_gpio: gpio { + compatible = "apple,smc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; }; mtp: mtp@394600000 { @@ -121,13 +254,20 @@ interrupt-names = "tx", "rx"; apple,fifo-size = <0x800>; apple,helper-cpu = <&mtp>; + /* + * ADT dockchannel-mtp/mtp-transport/multi-touch: + * function-afe-reset = smc-pmu pKW4('gP1c'), SMC GPIO 28. + * The MTP firmware cannot reach the AFE on its own ("No + * HINT_L"): after macOS left it running the trackpad + * worked, after a Linux warm reboot it stayed in reset. + */ + apple,afe-reset-gpios = <&smc_gpio 28 GPIO_ACTIVE_LOW>; stm { }; multi-touch { firmware-name = "apple/tpmtfw-j813.bin"; apple,power-method = <2>; - apple,firmware-managed-reset; }; /* Layout is filled by the loader using this machine's own data. */ @@ -135,3 +275,4 @@ }; }; }; +#include "t8142-j813-pmp-tunables.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8142-j813-pmp-tunables.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-pmp-tunables.dtsi new file mode 100644 index 0000000000000..fa34d4d80fe7d --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-pmp-tunables.dtsi @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J813 PMP firmware properties: the ADT /arm-io/pmp/iop-pmp-nub values the + * firmware queries by name at boot (pmp.rs serves apple,tunable-). + * Generated from the J813 ADT by tools/adtnub.py; raw ADT bytes. + */ + +&pmp { + apple,tunable-agx-dcs-bw-constraint = [00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00]; + apple,tunable-mc_quota_req_ptd_count = [08 00 00 00]; + apple,tunable-mc_cmd_ptd_count = [02 00 00 00]; + apple,tunable-ane-dcs-bw-dvfs-filter = [ff ff 1f 00]; + apple,tunable-sochot-sensors = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 15 00 00 00 16 00 00 00 17 00 00 00 18 00 00 00]; + apple,tunable-soc-ni3-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-agx-af-rd-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 35 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 34 00 00 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 00 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 00 00 00 00 00 00 3a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 39 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 2d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 2c 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 11 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3c 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 33 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 7a 00 00 00 00 00 00 00 32 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 79 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 2c 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 8a 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 89 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3e 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 6f 00 00 00 00 00 00 00 3d 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 6e 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 34 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 33 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2c 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 41 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 40 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3c 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 37 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 36 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00]; + apple,tunable-ane-af-bw-dvfs-filter = [ff ff 1f 00]; + apple,tunable-ans-af-bw-threshold = [70 3d 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 70 3d 0b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 cc 0c 1c 00 00 00 00 00]; + apple,tunable-ptd-range = [01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 50 54 44 2d 4e 55 4c 4c 00 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 01 00 00 00 0b 00 00 00 50 4d 50 2d 53 54 41 54 55 53 00 00 00 00 00 00 03 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 44 56 46 53 2d 53 54 41 54 45 00 00 00 00 00 00 04 00 00 00 06 00 00 00 04 00 00 00 00 00 00 00 50 4d 50 54 4f 4f 4c 00 00 00 00 00 00 00 00 00 05 00 00 00 98 00 00 00 08 00 00 00 00 00 00 00 50 4d 53 2d 50 4d 47 52 2d 50 4b 54 00 00 00 00 06 00 00 00 a0 00 00 00 08 00 00 00 00 00 00 00 50 4d 53 2d 44 50 45 2d 50 4b 54 00 00 00 00 00 07 00 00 00 28 02 00 00 18 00 00 00 00 00 00 00 50 4d 53 2d 44 45 56 43 2d 50 4b 54 00 00 00 00 09 00 00 00 a8 00 00 00 80 01 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 4b 54 00 00 00 00 00 0a 00 00 00 27 00 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 52 45 51 00 00 0b 00 00 00 2f 00 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 41 43 4b 00 00 0c 00 00 00 37 00 00 00 10 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 44 56 46 53 00 00 00 00 0d 00 00 00 47 00 00 00 0a 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 42 57 52 00 00 00 00 00 0e 00 00 00 13 00 00 00 0a 00 00 00 00 00 00 00 4d 43 2d 44 52 56 2d 54 4f 2d 43 54 52 4c 00 00 0f 00 00 00 51 00 00 00 01 00 00 00 00 00 00 00 44 56 46 53 2d 43 45 49 4c 00 00 00 00 00 00 00 14 00 00 00 0a 00 00 00 01 00 00 00 00 00 00 00 41 56 44 2d 4e 52 47 00 00 00 00 00 00 00 00 00 15 00 00 00 0b 00 00 00 01 00 00 00 00 00 00 00 53 4f 43 2d 41 4f 4e 2d 4e 52 47 00 00 00 00 00 16 00 00 00 0c 00 00 00 01 00 00 00 00 00 00 00 53 4f 43 2d 52 45 53 54 2d 4e 52 47 00 00 00 00 1e 00 00 00 1d 00 00 00 09 00 00 00 00 00 00 00 4d 43 2d 43 54 52 4c 2d 54 4f 2d 44 52 56 00 00 1f 00 00 00 0f 00 00 00 02 00 00 00 0c 00 00 00 46 41 53 54 2d 44 49 45 2d 43 45 00 00 00 00 00 20 00 00 00 52 00 00 00 07 00 00 00 0d 00 00 00 53 4d 43 2d 54 4f 2d 50 50 4d 00 00 00 00 00 00 21 00 00 00 0d 00 00 00 01 00 00 00 0a 00 00 00 4c 54 53 2d 56 4f 4c 54 2d 43 41 50 00 00 00 00 40 00 00 00 59 00 00 00 05 00 00 00 0e 00 00 00 50 50 4d 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 41 00 00 00 5e 00 00 00 05 00 00 00 0f 00 00 00 43 4c 50 43 2d 54 4f 2d 53 4d 43 00 00 00 00 00 43 00 00 00 11 00 00 00 02 00 00 00 00 00 00 00 46 41 53 54 2d 44 49 45 2d 54 4f 2d 53 4d 43 00 44 00 00 00 26 00 00 00 01 00 00 00 00 00 00 00 4d 45 4d 43 2d 54 4f 2d 53 4d 43 00 00 00 00 00 52 00 00 00 0e 00 00 00 01 00 00 00 00 00 00 00 4c 54 53 2d 54 4f 2d 47 46 58 00 00 00 00 00 00]; + apple,tunable-sram-index = [01 00 00 00]; + apple,tunable-no-shutdown = [01 00 00 00]; + apple,tunable-dvfs-domain = [01 00 00 00 08 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 53 4f 43 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 06 00 00 00 00 00 00 00 44 49 53 50 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-dvfs-disp-min-state = [01 00 00 00]; + apple,tunable-fast-die-ctrl-ce-map = [01 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 45 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 5f 41 4e 45 00 00 00 00 00 00 00 00]; + apple,tunable-ane-dcs-bw-constraint = [00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00]; + apple,tunable-soc-ni2-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-agx-af-wr-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00]; + apple,tunable-user-power-managed = [01 00 00 00]; + apple,tunable-ane-af-bw-threshold = [00 00 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 1c 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 09 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 f4 01 00 00 00 00 00 00 06 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 16 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 41 00 00 00 00 00 00 00 15 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 40 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3f 00 00 00 00 00 00 00 12 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 3e 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3a 00 00 00 00 00 00 00 0c 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 39 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3a 00 00 00 00 00 00 00 0c 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 39 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 31 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 1e 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 0e 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 08 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 02 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 f4 01 00 00 00 00 00 00 02 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 f3 01 06 00 00 00]; + apple,tunable-lts-voltage = [01 00 00 00 03 00 00 00 03 00 00 00 01 00 00 00 45 41 43 43 00 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 02 00 00 00 50 41 43 43 00 00 00 00 03 00 00 00 01 00 00 00 06 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 04 00 00 00 03 00 00 00 05 00 00 00 04 00 00 00 41 47 58 00 00 00 00 00]; + apple,tunable-ane-dcs-bw-threshold = [a3 f0 13 00 01 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 02 00 00 00 a3 f0 12 00 00 00 00 00 1e 05 3a 00 03 00 00 00 66 e6 26 00 00 00 00 00 0a 57 4a 00 04 00 00 00 1e 05 39 00 00 00 00 00 3d 0a 57 00 05 00 00 00 0a 57 49 00 00 00 00 00 3d 8a 65 00 06 00 00 00 3d 0a 56 00 00 00 00 00 5c 0f 74 00 07 00 00 00 3d 8a 64 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 00 00 00 00 a3 f0 13 00 01 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 02 00 00 00 a3 f0 12 00 00 00 00 00 1e 05 3a 00 03 00 00 00 66 e6 26 00 00 00 00 00 0a 57 4a 00 04 00 00 00 1e 05 39 00 00 00 00 00 3d 0a 57 00 05 00 00 00 0a 57 49 00 00 00 00 00 3d 8a 65 00 06 00 00 00 3d 0a 56 00 00 00 00 00 5c 0f 74 00 07 00 00 00 3d 8a 64 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 00 00 00 00 a3 f0 13 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 a3 f0 12 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 00 00 0e 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 00 00 09 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 00 00 00 00 00 00 09 00 06 00 00 00 1e 05 3a 00 00 00 00 00 66 e6 26 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 1e 05 3a 00 00 00 00 00 00 00 0e 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 1e 05 3a 00 00 00 00 00 00 00 0e 00 06 00 00 00 0a 57 4a 00 00 00 00 00 1e 05 39 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 04 00 06 00 00 00 0a 57 4a 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 57 4a 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 15 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 57 4a 00 00 00 00 00 00 00 14 00 06 00 00 00 3d 0a 57 00 00 00 00 00 0a 57 49 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 00 00 06 00 00 00 3d 0a 57 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2a 00 00 00 00 00 00 00 00 00 06 00 00 00 3d 0a 57 00 00 00 00 00 00 00 29 00 06 00 00 00 3d 8a 65 00 00 00 00 00 3d 0a 56 00 06 00 00 00 5c 0f 74 00 00 00 00 00 3d 8a 64 00 06 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 06 00 00 00]; + apple,tunable-bwr-catch-up-factor = [00 00 01 00]; + apple,tunable-controller = [01 00 00 00 1f 00 00 00 43 00 00 00 00 00 00 00 00 00 00 00 44 49 45 2d 54 45 4d 50 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 46 49 2d 54 4f 50 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 2d 52 44 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 2d 57 52 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 10 00 00 00 0e 00 00 00 1e 00 00 00 00 00 00 00 00 00 00 00 4d 45 4d 2d 43 41 43 48 45 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 11 00 00 00 52 00 00 00 21 00 00 00 00 00 00 00 00 00 00 00 4c 49 46 45 54 49 4d 45 2d 53 45 52 56 4f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-soc-device-ps-group = [00 fc ff 1f 00 00 00 00]; + apple,tunable-agx-dcs-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 27 00 00 00 00 00 00 00 12 00 00 00 00 00 00 00 39 00 00 00 00 00 00 00 26 00 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 38 00 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 00 00 00 00 00 00 11 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 27 00 00 00 00 00 00 00 10 00 06 00 00 00 00 00 39 00 00 00 00 00 00 00 26 00 06 00 00 00 00 00 4a 00 00 00 00 00 00 00 38 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 49 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 27 00 00 00 00 00 00 00 0b 00 06 00 00 00 00 00 39 00 00 00 00 00 00 00 26 00 06 00 00 00 00 00 47 00 00 00 00 00 00 00 38 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 46 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 37 00 00 00 00 00 00 00 21 00 06 00 00 00 00 00 47 00 00 00 00 00 00 00 36 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 46 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 1f 00 00 00 00 00 00 00 14 00 06 00 00 00 00 00 47 00 00 00 00 00 00 00 1e 00 06 00 00 00 00 00 57 00 00 00 00 00 00 00 46 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 56 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 07 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 57 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 65 00 00 00 00 00 00 00 56 00 06 00 00 00 00 00 e8 03 00 00 00 00 00 00 64 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 e7 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 17 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 43 00 00 00 00 00 00 00 16 00 06 00 00 00 00 00 65 00 00 00 00 00 00 00 42 00 06 00 00 00 00 00 74 00 00 00 00 00 00 00 64 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2d 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 50 00 00 00 00 00 00 00 2c 00 06 00 00 00 00 00 74 00 00 00 00 00 00 00 4f 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 10 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 3c 00 00 00 00 00 00 00 0f 00 06 00 00 00 00 00 74 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 73 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 1e 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00]; + apple,tunable-soc-ni1-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-mc_quota_rsp_ptd_count = [08 00 00 00]; + apple,tunable-lts-ctrl-ts = [00 00 05 00]; + apple,tunable-temp-sensor = [01 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 80 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 30 69 00 00 00 00 00 00 00 00 00 00 02 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 33 b3 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 31 69 00 00 00 00 00 00 00 00 00 00 03 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 66 e6 03 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 32 69 00 00 00 00 00 00 00 00 00 00 04 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 80 05 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 35 69 00 00 00 00 00 00 00 00 00 00 05 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 66 e6 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 36 69 00 00 00 00 00 00 00 00 00 00 06 00 00 00 08 00 00 00 06 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 cc 4c 05 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 61 30 30 37 69 00 00 00 00 00 00 00 00 00 00 07 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 30 69 00 00 00 00 00 00 00 00 00 00 08 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 31 69 00 00 00 00 00 00 00 00 00 00 09 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 33 69 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 34 69 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 01 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6c 69 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 00 00 01 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6d 69 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 05 00 00 00 00 00 00 00 33 33 03 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6e 69 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 06 00 00 00 00 00 00 00 66 e6 02 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6f 69 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 07 00 00 00 00 00 00 00 99 99 02 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 70 69 00 00 00 00 00 00 00 00 00 00 10 00 00 00 08 00 00 00 04 00 00 00 01 00 00 00 09 00 00 00 00 00 00 00 33 33 02 00 00 80 06 00 6e 00 00 00 7d 00 00 00 54 70 30 30 72 69 00 00 00 00 00 00 00 00 00 00 11 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 39 69 00 00 00 00 00 00 00 00 00 00 12 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 61 69 00 00 00 00 00 00 00 00 00 00 13 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 07 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 36 69 00 00 00 00 00 00 00 00 00 00 14 00 00 00 08 00 00 00 04 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 40 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 70 30 30 37 69 00 00 00 00 00 00 00 00 00 00 15 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 66 66 08 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 37 69 00 00 00 00 00 00 00 00 00 00 16 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 80 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 31 69 00 00 00 00 00 00 00 00 00 00 17 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 99 99 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 32 69 00 00 00 00 00 00 00 00 00 00 18 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 33 b3 04 00 00 80 04 00 6e 00 00 00 7d 00 00 00 54 65 30 30 33 69 00 00 00 00 00 00 00 00 00 00 19 00 00 00 01 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6d 00 00 00 6d 00 00 00 4d 54 52 5f 54 4f 50 5f 41 47 58 00 00 00 00 00]; + apple,tunable-agx-af-wr-bw-dvfs-filter = [fe 7f 00 00]; + apple,tunable-dcs-bw-threshold = [a3 f0 13 00 01 00 00 00 00 00 00 00 00 00 00 00 66 e6 27 00 02 00 00 00 a3 f0 12 00 00 00 00 00 1e 05 3a 00 03 00 00 00 66 e6 26 00 00 00 00 00 0a 57 4a 00 04 00 00 00 1e 05 39 00 00 00 00 00 3d 0a 57 00 05 00 00 00 0a 57 49 00 00 00 00 00 3d 8a 65 00 06 00 00 00 3d 0a 56 00 00 00 00 00 5c 0f 74 00 07 00 00 00 3d 8a 64 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 0f 73 00 00 00 00 00]; + apple,tunable-mc_cmd_ack_ptd_count = [01 00 00 00]; + apple,tunable-soc-ni0-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-fast-die-ctrl-loop = [01 00 00 00 00 00 00 00 00 00 00 00 66 06 00 00 47 41 00 00 00 00 6e 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 01 00 16 00 00 00 15 00 00 00 16 00 00 00 17 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 45 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 66 06 00 00 47 41 00 00 00 00 6e 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 01 00 16 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 02 00 00 00 66 06 00 00 47 41 00 00 00 00 6e 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 01 00 16 00 00 00 01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-agx-dcs-bw-dvfs-filter = [fe 7f 00 00]; + apple,tunable-agx-af-rd-bw-dvfs-filter = [fe 7f 00 00]; + apple,tunable-firmware-name = [74 38 31 34 32 70 6d 70 00]; + apple,tunable-lts-clpc-voltage = [01 00 00 00 98 03 00 00 00 00 69 00 33 33 69 00 00 00 00 00 50 41 43 43 00 00 00 00]; + apple,tunable-soc-ni5-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-energy-counter = [01 00 00 00 01 00 00 00 05 00 00 00 01 00 00 00 41 47 58 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 05 00 00 00 02 00 00 00 41 47 58 2d 53 52 41 4d 00 00 00 00 00 00 00 00 03 00 00 00 01 00 00 00 06 00 00 00 01 00 00 00 41 4e 45 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 14 00 00 00 00 00 00 00 56 44 45 43 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 53 4f 43 5f 41 4f 4e 00 00 00 00 00 00 00 00 00]; + apple,tunable-lts-config = [01 00 00 00 3b 01 00 00 a6 72 0f 00 00 00 00 00 b5 d5 0f 00 6e 00 00 00 69 00 00 00 00 00 11 00 33 b3 00 00 b0 32 47 01 02 00 00 00 47 03 00 00 3b 61 00 00 00 00 00 00 f7 dd 0f 00 6e 00 00 00 69 00 00 00 00 00 11 00 33 b3 00 00 d7 43 ef 02]; + apple,tunable-agx-af-rd-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00]; + apple,tunable-ane-af-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00]; + apple,tunable-mcache-stream = [0e 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0d 00 00 00 38 00 00 00 38 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 11 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3c 00 00 00 3c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1b 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3d 00 00 00 3d 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 13 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 40 00 00 00 40 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 24 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 41 00 00 00 41 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1c 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 42 00 00 00 42 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1e 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2b 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0e 00 00 00 0e 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 05 00 00 00 01 00 00 00 01 00 00 00 07 00 00 00 07 00 00 00 20 00 00 00 20 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 06 00 00 00 02 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 24 00 00 00 24 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0b 00 00 00 05 00 00 00 01 00 00 00 0b 00 00 00 0b 00 00 00 30 00 00 00 30 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 01 00 00 00 11 00 00 00 11 00 00 00 48 00 00 00 48 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 13 00 00 00 13 00 00 00 50 00 00 00 50 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 15 00 00 00 15 00 00 00 58 00 00 00 5a 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 10 00 00 00 04 00 00 00 01 00 00 00 1f 00 00 00 1f 00 00 00 80 00 00 00 bf 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 04 00 00 00 02 00 00 00 10 00 00 00 10 00 00 00 44 00 00 00 44 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 20 00 00 00 03 00 00 00 02 00 00 00 03 00 00 00 06 00 00 00 10 00 00 00 1f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 1f 00 00 00 03 00 00 00 02 00 00 00 1e 00 00 00 1e 00 00 00 7c 00 00 00 7c 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 12 00 00 00 05 00 00 00 02 00 00 00 09 00 00 00 09 00 00 00 28 00 00 00 28 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0a 00 00 00 05 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 34 00 00 00 34 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 15 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5d 00 00 00 5d 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 36 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5e 00 00 00 5e 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 37 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5f 00 00 00 5f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 19 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 05 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 29 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 06 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 07 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0c 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 09 00 00 00 09 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 14 00 00 00 04 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 0a 00 00 00 0a 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 27 00 00 00 04 00 00 00 03 00 00 00 1d 00 00 00 1d 00 00 00 78 00 00 00 78 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2c 00 00 00 03 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 64 00 00 00 64 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2f 00 00 00 02 00 00 00 03 00 00 00 1a 00 00 00 1a 00 00 00 6c 00 00 00 6c 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 26 00 00 00 05 00 00 00 03 00 00 00 1c 00 00 00 1c 00 00 00 74 00 00 00 74 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 23 00 00 00 00 00 00 00 03 00 00 00 14 00 00 00 14 00 00 00 54 00 00 00 54 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 28 00 00 00 00 00 00 00 03 00 00 00 1b 00 00 00 1b 00 00 00 70 00 00 00 70 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 31 00 00 00 0f 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 60 00 00 00 60 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 30 00 00 00 0e 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 61 00 00 00 61 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 33 00 00 00 11 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 62 00 00 00 62 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 34 00 00 00 12 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 63 00 00 00 63 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 35 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 68 00 00 00 68 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 1d 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 69 00 00 00 6b 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2d 00 00 00 0c 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 65 00 00 00 65 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 16 00 00 00 0b 00 00 00 03 00 00 00 ff ff 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00]; + apple,tunable-ptd-driver-version = [01 00 00 00]; + apple,tunable-afi-top-bw-threshold = [99 19 3a 00 00 00 00 00 00 00 00 00 00 00 00 00 51 78 4d 00 01 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 51 78 4c 00 00 00 00 00 00 00 00 00 00 00 00 00 33 33 73 00 00 00 00 00 00 00 00 00 00 00 00 00 b8 1e 7c 00 00 00 00 00]; + apple,tunable-region-size = [00 00 0d 00 00 00 00 00]; + apple,tunable-coredump-enable = [40 00 00 00]; + apple,tunable-soc-ni4-bw-threshold = [cc 0c 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 26 00 01 00 00 00 cc 0c 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 e1 ba 25 00 00 00 00 00 00 00 00 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 3d 00 00 00 00 00]; + apple,tunable-pm-ptd-ranges = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 00 00 00 00 0f 00 00 00]; + apple,tunable-fast-die-ctrl-ts-avg-temp = [00 00 64 00]; + apple,tunable-lts-ctrl-loop = [02 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 02 00 00 00 01 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 47 50 55 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-soc-device = [01 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 02 00 00 00 45 4e 4f 4e 01 00 00 00 16 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4d 43 43 00 00 00 00 17 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 00 18 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 00 19 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 00 1a 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 00 1b 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 00 1c 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 00 10 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 41 4f 4e 00 11 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 52 45 53 54 03 00 00 00 45 4e 4f 4e 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 45 41 43 43 30 00 00 00 04 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 30 00 00 00 05 00 00 00 5f 58 46 47 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 00 00 00 00 00 06 00 00 00 45 4e 4f 4e 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 30 00 00 00 00 07 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 49 53 50 00 00 00 00 00 08 00 00 00 45 4e 4f 4e 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 00 00 00 00 09 00 00 00 45 4e 4f 4e 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 30 0a 00 00 00 45 4e 4f 4e 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 31 0b 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 00 00 00 00 00 0c 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 14 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 44 00 00 00 00 00 0d 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 53 52 00 00 00 00 00 0e 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4a 50 45 47 00 00 00 00 0f 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 43 4f 44 45 43 00 00 1d 00 00 00 5f 52 54 4c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4c 54 52 00 00 00 00 00 1e 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 30 00 00 00 00 1f 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 31 00 00 00 00 20 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 32 00 00 00 00 21 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 33 00 00 00 00 22 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 00 00 00 00 00 23 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 43 49 45 00 00 00 00 24 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 42 52 00 00 00 00 00 25 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 45 50 00 00 00 00 00 26 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 49 4f 00 00 00 00 00]; + apple,tunable-fast-die-ctrl-ts = [00 00 05 00]; + apple,tunable-region-base = [00 00 50 80 03 00 00 00]; + apple,tunable-agx-af-wr-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 35 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 34 00 00 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 00 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 00 00 00 00 00 00 3a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 39 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 2d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 2c 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 5a 00 00 00 00 00 00 00 11 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 3c 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 89 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 88 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 33 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 7a 00 00 00 00 00 00 00 32 00 06 00 00 00 00 00 93 00 00 00 00 00 00 00 79 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 92 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 2c 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 8a 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 89 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3e 00 00 00 00 00 00 00 1d 00 06 00 00 00 00 00 6f 00 00 00 00 00 00 00 3d 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 6e 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 34 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 5a 00 00 00 00 00 00 00 33 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 59 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 2c 00 00 00 00 00 00 00 03 00 06 00 00 00 00 00 41 00 00 00 00 00 00 00 2b 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 40 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 3c 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 3b 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 37 00 00 00 00 00 00 00 18 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 36 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 13 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 31 00 06 00 00 00]; + apple,tunable-uuid = [30 41 37 44 41 39 43 33 2d 45 37 46 36 2d 33 31 35 31 2d 41 43 44 45 2d 44 41 30 31 37 42 45 42 45 36 34 35 00]; + apple,tunable-segment-names = [5f 5f 54 45 58 54 3b 5f 5f 44 41 54 41 00]; + apple,tunable-segment-ranges = [00 00 50 80 03 00 00 00 00 00 00 01 00 00 00 00 00 00 50 80 03 00 00 00 00 c0 03 00 03 00 00 00 00 c0 53 80 03 00 00 00 00 c0 03 01 00 00 00 00 00 c0 53 80 03 00 00 00 00 e0 06 00 06 00 00 00]; + apple,tunable-pre-loaded = [01 00 00 00]; +}; From 67d889ef91d9c3c4143f7c272e429b8a3a3561fc Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 093/117] arm64: dts: apple: t8142-j813: Describe the M5 GPU The M5 (T8142) GPU, laid out like the A18 Pro's: the sgx block, the gfx-asc and gfx1-asc firmware CPUs with their mailboxes, and the power domain, with addresses and interrupts from the J813 ADT. aura fills the uat-* regions from iBoot's raw-boot handoff, the gfx-data regions from the firmware DATA segments iBoot loaded, and the OPPs from sgx perf-states. apple,clpc links the GPU to the CLPC so that GPU work raises the DRAM and fabric floor. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi | 129 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 130 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi new file mode 100644 index 0000000000000..d1b8b540538bd --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-gpu.dtsi @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * M5 (T8142) GPU, laid out like the A18 Pro's (t8140.dtsi). Addresses and + * interrupts are from the J813 ADT (sgx, gfx-asc, gfx1-asc, pmgr reg[48]). + * The bootloader fills the uat-* regions from iBoot's raw-boot handoff, the + * gfx-data regions from the firmware DATA segments iBoot loaded, and the + * OPPs from sgx perf-states. + */ + +/ { + aliases { + gpu = &gpu; + }; + + reserved-memory { + /* gfx-asc and gfx1-asc __DATA (ADT sgx gfx-data, gfx1-data) */ + gfx_data: gfx-data { + reg = <0 0 0 0>; + }; + + gfx1_data: gfx1-data { + reg = <0 0 0 0>; + }; + + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + }; + + uat_l2_pagetables: uat-l2-pagetables { + reg = <0 0 0 0>; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + }; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + }; + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* 13 entries in perf-state order, filled in by the bootloader */ + opp00 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp01 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp02 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp03 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp04 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp05 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp06 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp07 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp08 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp09 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp10 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp11 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + opp12 { opp-hz = /bits/ 64 <0>; opp-microvolt = <0>; }; + }; +}; + +&{/soc} { + pmgr_gpu: power-management@580e80000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <5 0x80e80000 0 0x30000>; + #address-cells = <1>; + #size-cells = <1>; + + ps_gpx: power-controller@0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gpx"; + apple,always-on; /* critical */ + }; + + ps_gfx: power-controller@110 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + power-domains = <&ps_gpx>; + }; + }; + + gpu: gpu@582600000 { + compatible = "apple,agx-t8142"; + reg = <0x5 0x82600000 0x0 0x88000>, + <0x5 0x80000000 0x0 0x4000000>; + reg-names = "asc", "sgx"; + mboxes = <&gfx_mbox>, <&gfx1_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, + <&uat_l2_pagetables>, <&gfx_data>, <&gfx1_data>; + memory-region-names = "ttbs", "pagetables", "handoff", + "shared-l2", "gfx-data", "gfx1-data"; + operating-points-v2 = <&gpu_opp>; + /* Holds the DCS and fabric floor while GPU work is in flight. */ + apple,clpc = <&clpc>; + }; + + gfx_mbox: mailbox@582600000 { + compatible = "apple,t8142-ascwrap-v6"; + reg = <0x5 0x82600000 0x0 0x88000>, + <0x5 0x82050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + }; + + gfx1_mbox: mailbox@582e00000 { + compatible = "apple,t8142-ascwrap-v6"; + reg = <0x5 0x82e00000 0x0 0x88000>, + <0x5 0x82850000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX1"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index f1378b8651878..2c3f2b35dd8c7 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -394,5 +394,6 @@ }; #include "t8142-j813-display.dtsi" +#include "t8142-j813-gpu.dtsi" #include "t8142-j813-peripherals.dtsi" #include "t8142-j813-usb.dtsi" From 3c1ee27e048c5e79cb5d6fc828435975716ab44a Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 094/117] arm64: dts: apple: t8142-j813: AVD video decoder The M5 AVD ("Hibiscus", avd-version 4, avd-fw v5 tier 2 tunables): CM3 code and SRAM, the mailbox, its DART (SIDs 0-2) and the AVD_SYS power state, from the J813 ADT. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi | 45 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 46 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi new file mode 100644 index 0000000000000..5672ebb885a51 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-avd.dtsi @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * M5 (T8142) AVD video decoder: a "Hibiscus" AVD (macOS CAvdMcpuHibiscus, + * avd-version 4, tunables = avd-fw v5 tier 2). From the J813 ADT: avd reg[0] + * 0x2f0000000 + 0x1404000, here at the arm-io offset 0x500000000; 0x14000 each + * of CM3 code and SRAM, mailbox at +0x10a8000 (CAvdApCommHibiscus); + * IRQs 1077/1078; dart-avd at +0x1020000, IRQ 1084, SIDs 0-2; power: PMGR + * device AVD-SYS-V -> AVD_SYS (+0x310). + */ + +&pmgr { + ps_avd_sys: power-controller@310 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "avd_sys"; + }; +}; + +&{/soc} { + avd: avd@501080000 { + compatible = "apple,t8142-avd"; + interrupt-parent = <&aic>; + interrupts = , + ; + power-domains = <&ps_avd_sys>; + resets = <&ps_avd_sys>; + reg = <0x5 0x01080000 0x0 0x14000>, + <0x5 0x01094000 0x0 0x14000>, + <0x5 0x010a8000 0x0 0x4000>, + <0x5 0x01100000 0x0 0x10000>; + reg-names = "code", "sram", "mbox", "ctrl"; + iommus = <&avd_dart 0>, <&avd_dart 1>, <&avd_dart 2>; + }; + + avd_dart: iommu@501020000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x5 0x01020000 0x0 0x20000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_avd_sys>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 2c3f2b35dd8c7..ae576743895cf 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -397,3 +397,4 @@ #include "t8142-j813-gpu.dtsi" #include "t8142-j813-peripherals.dtsi" #include "t8142-j813-usb.dtsi" +#include "t8142-j813-avd.dtsi" From 4a6b03722050c4aaf06f9a61385681155a999b40 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 095/117] arm64: dts: apple: t8142-j813: ISP (FaceTime camera) The M5 ISP is the A18 Pro's ISP17a at another base: the ISP block and its interrupts, the ISP DART with the remapped streams, the PMGR power states the firmware does not lock, and the J813 camera presets (1280x720, 1440x1080, 1080x1080 and 1920x1080), from the J813 ADT. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi | 223 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 224 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi new file mode 100644 index 0000000000000..accb5629a7e65 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * M5 (T8142) ISP: the Neo's (T8140) ISP17a at another base. From the J813 + * ADT (arm-io offset 0x210000000): isp reg 0x4a0000000 + 0x4000000, IRQs + * [425 426 427 428 438] (425 carries the IPC mailbox), sensor-type 0x71; + * dart-isp at +0x28c0000/+0x28e0000/+0x2900000, IRQ 443, vm-size 0xe0000000, + * remap streams 1 2 3 6 7 10 11 -> 0; EIC mmio-wd-isp 0x4e3b10000 + 0x8000. + * PMGR (0x380700000): ISP_SYS +0x2f8 (ADT id 75), the ISP17a slots + * +0x4000..+0x4060 (ids 162-174, the Neo's offsets), ISPSENS0/1 +0x190/+0x198. + * aura carves the firmware heap and adds it as "isp-heap"; without it, aura + * disables the node. + */ + +/ { + aliases { + isp = &isp; + }; +}; + +&pmgr { + ps_isp_sys: power-controller@2f8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sys"; + }; + + ps_ispsens0: power-controller@190 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens0"; + }; + + ps_ispsens1: power-controller@198 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens1"; + }; + + ps_isp_cpu: power-controller@4000 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cpu"; + power-domains = <&ps_isp_sys>; + }; + + ps_isp_cpu_core0: power-controller@4008 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cpu_core0"; + power-domains = <&ps_isp_cpu>; + }; + + /* + * Not here: ISP_CPU_CORE1 (+0x4010), ISP_SECURE (+0x4028), ISP_STS1 + * (+0x4038), ISP_PEARL (+0x4040), ISP_CVD (+0x4058) and ISP_CVM + * (+0x4060). On J813 their target fields ignore writes from the AP and + * read back 0. The dart-isp DAPF passes this block to the ISP. + */ + + ps_isp_fe: power-controller@4018 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + power-domains = <&ps_isp_sys>; + }; + + /* Unnamed in the ADT, as on the Neo: the firmware's ISP_DPRX. */ + ps_isp_dprx: power-controller@4020 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_dprx"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_sts0: power-controller@4030 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sts0"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_be: power-controller@4048 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_be"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_clr: power-controller@4050 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_clr"; + power-domains = <&ps_isp_be>; + }; + +}; + +&{/soc} { + isp_dart0: iommu@4a28c0000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x4 0xa28c0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + /* As on the Neo: 3-level tables, the ISP fetches at vm-base + * 0x10000000000 and the walk ignores the high bits. */ + apple,dma-range = <0x0 0x0 0x8 0x0>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + }; + + isp_dart1: iommu@4a28e0000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x4 0xa28e0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,dma-range = <0x0 0x0 0x8 0x0>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + }; + + isp_dart2: iommu@4a2900000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x4 0xa2900000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,dma-range = <0x0 0x0 0x8 0x0>; + apple,remap-streams = <1 0>, <2 0>, <3 0>, <6 0>, <7 0>, + <10 0>, <11 0>; + }; + + /* + * As on the Neo: "coproc" stops short of the secure route table at + * +0x1804000 (a read takes a sticky SError) and of the IRQ block + * (+0x2920000, "mbox") and the GPIO words (+0x2921000). + */ + isp: isp@4a0000000 { + compatible = "apple,t8142-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "wdt"; + reg = <0x4 0xa0000000 0x0 0x1801000>, + <0x4 0xa2920000 0x0 0x1000>, + <0x4 0xa2921000 0x0 0x100>, + <0x4 0xe3b10000 0x0 0x8000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, + <&ps_isp_cpu_core0>, <&ps_isp_fe>, + <&ps_isp_dprx>, <&ps_isp_sts0>, + <&ps_isp_be>, <&ps_isp_clr>; + apple,dart-vm-size = <0x0 0xe0000000>; + apple,platform-id = <0x71>; + apple,temporal-filter = <1>; + }; +}; + +/* + * The J813 IMX958 (sensor version 0x958) as its firmware lists it: 13 configs, + * among them 3040x2880 (config 7) and a binned 1520x1440 (config 10); half + * of them report doubled heights and are left out. Its setfile is + * apple/isp_2224_01XX.dat (from macOS appleh16camerad). + */ +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <1>; + + sensor-presets { + /* 1280x720 */ + preset0 { + apple,config-index = <10>; + apple,input-size = <1520 1440>; + apple,output-size = <1280 720>; + apple,crop = <0 292 1520 856>; + }; + /* 1440x1080 (4:3) */ + preset1 { + apple,config-index = <10>; + apple,input-size = <1520 1440>; + apple,output-size = <1440 1080>; + apple,crop = <40 180 1440 1080>; + }; + /* 1080x1080 (1:1) */ + preset2 { + apple,config-index = <10>; + apple,input-size = <1520 1440>; + apple,output-size = <1080 1080>; + apple,crop = <220 180 1080 1080>; + }; + /* 1920x1080 */ + preset3 { + apple,config-index = <7>; + apple,input-size = <3040 2880>; + apple,output-size = <1920 1080>; + apple,crop = <0 584 3040 1710>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index ae576743895cf..4732e92f7f270 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -398,3 +398,4 @@ #include "t8142-j813-peripherals.dtsi" #include "t8142-j813-usb.dtsi" #include "t8142-j813-avd.dtsi" +#include "t8142-j813-isp.dtsi" From 2c6799798c725c2bb53d9b22b20af1fb59993ffd Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 096/117] arm64: dts: apple: t8142-j813: CPU idle through PSCI aura runs the OS at EL1 under its monitor and adds a /psci node with enable-method psci for every CPU. Its CPU_SUSPEND is a WFI at EL2 that preserves the registers WFI loses on these cores, so one retention state for both clusters replaces idle=nop. Without a /psci node (raw aura, spin-table) cpuidle-psci does not probe. Signed-off-by: Thomas Glanzmann --- .../boot/dts/apple/t8142-j813-psci-idle.dtsi | 62 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 63 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-psci-idle.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-psci-idle.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-psci-idle.dtsi new file mode 100644 index 0000000000000..9c9cf88a0267b --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-psci-idle.dtsi @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * CPU idle through PSCI CPU_SUSPEND when aura runs the OS at EL1 under its + * monitor (aura's /psci node, "method = smc", enable-method = "psci"). The + * monitor's CPU_SUSPEND is a WFI at EL2 that preserves the registers WFI + * wipes on these cores, so this one retention state replaces idle=nop. + * Without a /psci node (raw aura, spin-table) cpuidle-psci does not probe. + */ + +&{/cpus} { + idle-states { + entry-method = "psci"; + + cpu_retention: cpu-retention { + compatible = "arm,idle-state"; + arm,psci-suspend-param = <0x0>; + entry-latency-us = <10>; + exit-latency-us = <10>; + min-residency-us = <100>; + }; + }; +}; + +&cpu0 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu1 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu2 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu3 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu4 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu5 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu6 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu7 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu8 { + cpu-idle-states = <&cpu_retention>; +}; + +&cpu9 { + cpu-idle-states = <&cpu_retention>; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 4732e92f7f270..4e19d8c3cd052 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -399,3 +399,4 @@ #include "t8142-j813-usb.dtsi" #include "t8142-j813-avd.dtsi" #include "t8142-j813-isp.dtsi" +#include "t8142-j813-psci-idle.dtsi" From cf0704b6daffe9a7cfb196647552811ebacc6fd4 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 097/117] arm64: dts: apple: t8142-j813: Internal NVMe ANS2 with its mailbox, the coastguard SART (sart-version 3) and the apcie and ANS power states, from the J813 ADT. As on T8140 the NVMe registers are used through the non-secure alias and the NVMMU through the 0x60000 window; aura leaves ANS running for Linux to adopt. Signed-off-by: Thomas Glanzmann --- .../arm64/boot/dts/apple/t8142-j813-nvme.dtsi | 105 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 106 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-nvme.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-nvme.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-nvme.dtsi new file mode 100644 index 0000000000000..82f2bce059d17 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-nvme.dtsi @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (13-inch, M5) J813 with the internal ANS2 NVMe SSD. + * + * Addresses from the J813 ADT (/arm-io/ans, /arm-io/sart-ans, arm-io offset + * 0x210000000). ANS is an ascwrap-v6 ASC with its mailbox at +0x8000, like the + * SMC; the SART is coastguard sart-version 3. As on T8140, the NVMe registers + * are used through the non-secure alias (ADT reg[9]) and the NVMMU through + * the 0x60000 window (ADT reg[3]). iBoot leaves ANS running with the + * controller disabled, so the driver wakes it with a fresh RTKit handshake. + */ + + +&pmgr { + ps_apcie_gp: power-controller@2f0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_gp"; + }; + + ps_ans: power-controller@300 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ans"; + apple,always-on; /* handed over running, cannot be reset */ + }; + + ps_apcie_sys_gp: power-controller@408 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_gp"; + power-domains = <&ps_apcie_gp>; + }; + + ps_apcie_st: power-controller@410 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x410 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_st"; + power-domains = <&ps_ans>; + }; + + ps_apcie_sys_st: power-controller@520 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x520 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_st"; + power-domains = <&ps_ans>, <&ps_apcie_st>; + }; + + ps_apcie_phy_sw: power-controller@560 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x560 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_phy_sw"; + power-domains = <&ps_apcie_sys_st>, <&ps_apcie_sys_gp>; + }; +}; + +&{/soc} { + ans_mbox: mailbox@481608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x81608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_ans>; + #mbox-cells = <0>; + }; + + sart: iommu@485c50000 { + compatible = "apple,t8142-sart", "apple,t6000-sart"; + reg = <0x4 0x85c50000 0x0 0xc000>; + power-domains = <&ps_ans>; + }; + + nvme: nvme@4c5cc0000 { + compatible = "apple,t8142-nvme-ans2", "apple,t8140-nvme-ans2"; + reg = <0x4 0xc5cc0000 0x0 0x30000>, + <0x4 0x81600000 0x0 0x4000>, + <0x4 0x85cc0000 0x0 0x60000>; + reg-names = "nvme", "ans", "nvmmu"; + interrupt-parent = <&aic>; + interrupts = ; + mboxes = <&ans_mbox>; + apple,sart = <&sart>; + power-domains = <&ps_ans>, <&ps_apcie_sys_st>, + <&ps_apcie_phy_sw>; + power-domain-names = "ans", "apcie0", "apcie_phy"; + resets = <&ps_ans>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 4e19d8c3cd052..581eb10f876f4 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -400,3 +400,4 @@ #include "t8142-j813-avd.dtsi" #include "t8142-j813-isp.dtsi" #include "t8142-j813-psci-idle.dtsi" +#include "t8142-j813-nvme.dtsi" From 7c13636e52ce5e1bd2388c06b2b155fcbebacebb Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 098/117] arm64: dts: apple: t8142-j813: N1 Wi-Fi and Bluetooth on apcie port 0 aura trains the apcie port 0 link to the N1 (AUS5 PHY, refclk buffer, N1 reset). Linux uses the root port as a generic ECAM host with port 0's 32 MSIs (apple,t8142-pcie-msi) and owns the DART and the endpoint. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi | 47 ++++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 48 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi new file mode 100644 index 0000000000000..43a87da373a91 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-n1.dtsi @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * N1 Wi-Fi/Bluetooth on apcie port 0. The bootloader trains the link (AUS5 + * PHY, refclk buffer, N1 reset, port; see the m5air FINDINGS); Linux uses the + * root port as a generic ECAM host and owns the DART and the endpoint. + * Ranges and the DART IRQ from the J813 ADT, as in t8142-j813-wifi.dts. + */ +&{/} { + pcie0: pcie@1cb0000000 { + compatible = "pci-host-ecam-generic"; + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + bus-range = <0 1>; + reg = <0x1c 0xb0000000 0 0x200000>; + ranges = <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0 0x20000000>, + <0x02000000 0 0x80000000 0xb 0x80000000 0 0x40000000>; + dma-coherent; + msi-parent = <&pcie0_msi>; + /* N1's three ADT mapper nodes use stream zero. */ + iommu-map = <0x100 &dart_apcie0 0 1>, + <0x101 &dart_apcie0 0 1>, + <0x102 &dart_apcie0 0 1>; + }; +}; + +&{/soc} { + /* + * Port 0's MSI map (ADT apcie reg[8]). ADT msi-address 0xfffff000, + * msi-vector-offset 1741, #msi-vectors 96 for three ports: port 0 + * owns the first 32. + */ + pcie0_msi: msi-controller@488028000 { + compatible = "apple,t8142-pcie-msi"; + reg = <4 0x88028000 0 0x8000>; + msi-controller; + msi-ranges = <&aic AIC_IRQ 1741 IRQ_TYPE_EDGE_RISING 32>; + }; + + dart_apcie0: iommu@488000000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x88000000 0 0x20000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index 581eb10f876f4..cba25ff51274c 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -401,3 +401,4 @@ #include "t8142-j813-isp.dtsi" #include "t8142-j813-psci-idle.dtsi" #include "t8142-j813-nvme.dtsi" +#include "t8142-j813-n1.dtsi" From 84ac824b9cb1b3d8a73bccfe88f274fae5d68b3e Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 099/117] arm64: dts: apple: t8142-j813: SMC power off and restart, the watchdog The watchdog for restart (and as the SMC's fallback), the PMU on SPMI with the scratchpad cells the SMC uses for power off, and the SMC reboot node. Signed-off-by: Thomas Glanzmann --- .../boot/dts/apple/t8142-j813-power.dtsi | 72 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 73 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-power.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi new file mode 100644 index 0000000000000..fa080be75e8fa --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-power.dtsi @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (13-inch, M5) J813: SMC power off / restart, the PMU on + * SPMI and the watchdog. + */ + +&{/soc} { + /* ADT /arm-io/wdt (wdt,t8142): restart (and the SMC's fallback). */ + wdt: watchdog@3882b0000 { + compatible = "apple,t8142-wdt", "apple,t8103-wdt"; + reg = <0x3 0x882b0000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; +}; + +&{/soc} { + /* + * ADT /arm-io/nub-spmi0 (gen 4) and its primary PMU, pmu-main + * ("abbey", slave 0xe). The boot flags live in the legacy scratchpad + * (info-leg_scrpad = 0xf800), laid out as on T8103 and J873. + */ + nub_spmi0: spmi@388714000 { + compatible = "apple,t8142-spmi"; + reg = <0x3 0x88714000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + + pmic@e { + compatible = "apple,abbey-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + boot_stage: boot-stage@f801 { + reg = <0xf801 0x1>; + }; + + boot_error_count: boot-error-count@f802,0 { + reg = <0xf802 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@f802,4 { + reg = <0xf802 0x1>; + bits = <4 4>; + }; + + /* Set before off1, or the machine powers back on. */ + shutdown_flag: shutdown-flag@f80f,3 { + reg = <0xf80f 0x1>; + bits = <3 1>; + }; + }; + }; + }; +}; + +&smc { + /* Power off (MBSE = off1) and restart (MBSE = phra). */ + reboot { + compatible = "apple,smc-reboot"; + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index cba25ff51274c..b9c71fd75dacf 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -402,3 +402,4 @@ #include "t8142-j813-psci-idle.dtsi" #include "t8142-j813-nvme.dtsi" #include "t8142-j813-n1.dtsi" +#include "t8142-j813-power.dtsi" From 3181b067bdd93d67dddd8adbcaf2a554e64ecbf5 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:22:18 +0100 Subject: [PATCH 100/117] arm64: dts: apple: t8142-j813: Sound through the AOP As on J873 the AOP's audio service owns the audio fabric and runs the MCA serializers: the CS42L84 headphone jack on admac-base-ns TX2/RX2, the four SN012776 speaker amplifiers on TX0 with their V/I sense on RX0. The AOP is laid out like T8140's (ascwrap-v6, asc-mailbox-v4, setup port). Addresses are from the J813 ADT. Signed-off-by: Thomas Glanzmann --- .../boot/dts/apple/t8142-j813-audio.dtsi | 417 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + 2 files changed, 418 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-audio.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-audio.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-audio.dtsi new file mode 100644 index 0000000000000..892155da27ee0 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-audio.dtsi @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Air (13-inch, M5) J813: sound through the AOP firmware. + * + * As on J873 (T8152), the AOP's AOPAudioService owns the audio fabric and + * runs the MCA serializers: "cout"/"cin" (CS42L84 headphone jack) on + * admac-base-ns TX2/RX2 and "spkr" (four SN012776) on TX0, their V/I sense + * on RX0 (ADT iop-audio-controller; RX1 is the firmware's speaker tap). + * The AOP itself is laid out like T8140's (ascwrap-v6, asc-mailbox-v4), + * including the setup port. + * Addresses are from the J813 ADT (arm-io offset 0x210000000). + */ + +#include + +&pmgr { + ps_i2c1: power-controller@350 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c1"; + }; + + ps_i2c3: power-controller@360 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c3"; + }; +}; + +&{/soc} { + pinctrl_ap: pinctrl@39a000000 { + compatible = "apple,t8142-pinctrl", "apple,t8140-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x9a000000 0x0 0x100000>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 224>; + apple,npins = <224>; + /* + * Not every pin is the AP's: reading pin 78's register raises + * an SError. Expose only the pins sound uses: 100 (jack IRQ), + * 150 (amp IRQ) and 151 (amp shutdown). + */ + gpio-reserved-ranges = <0 100>, <101 49>, <152 72>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pinctrl_nub: pinctrl@3881f0000 { + compatible = "apple,t8142-pinctrl", "apple,t8140-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x881f0000 0x0 0x4000>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_nub 0 0 40>; + apple,npins = <40>; + /* Only pin 18 (jack codec reset), as on pinctrl_ap. */ + gpio-reserved-ranges = <0 18>, <19 21>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pmgr_aop: power-management@388284000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x3 0x88284000 0x0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + + /* + * Only the MCA serializer leaves (ADT AUDIO_MCA0_M..AUDIO_MCA5_M) + * are the AP's, as on J873. Their parents AUDIO_A, AUDIO_FR, + * AUDIO_P and AUDIO_BASE_NS_A (0x80, 0xb8, 0xd8, 0xf8) are the AOP + * firmware's: iBoot leaves them off, the firmware's audio init + * switches them off and polls for that (an AP-held auto-enable + * made it time out with 0xe00002d6, so "aop-audio" never + * registered and the firmware panicked in systemReady), and a + * service's pw0 request raises them again. + * + * Likewise the microphone array's LEAP leaves (AUDIO_LEAP_MCA, + * AUDIO_LEAP_S_A, AUDIO_LEAP_RX0) without their parents + * AUDIO_LEAP_C and AUDIO_LEAP_A (0x138, 0xe0); hpai's pw0 raises + * those. + */ + ps_audio_leap_mca: power-controller@100 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_mca"; + }; + + ps_audio_mca0_m: power-controller@108 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca0_m"; + }; + + ps_audio_mca1_m: power-controller@110 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca1_m"; + }; + + ps_audio_mca2_m: power-controller@118 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca2_m"; + }; + + ps_audio_mca3_m: power-controller@120 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca3_m"; + }; + + ps_audio_mca4_m: power-controller@128 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca4_m"; + }; + + ps_audio_mca5_m: power-controller@130 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_mca5_m"; + }; + + ps_audio_leap_s_a: power-controller@140 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_s_a"; + }; + + ps_audio_leap_rx0: power-controller@150 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_leap_rx0"; + }; + }; + + aop_mbox: mailbox@390608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x90608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + /* + * The ADT's aop-exclave-mailbox, flagged exclave-assigned as on T8140, + * is the AOP setup port the host drives. Its ADT reg (0x3d067c000) is + * not behind the arm-io translation: AOP ASC + 0x4007c000, as on + * T8140. ADT interrupts [497, 494], T8140's +100. + */ + aop_setup_mbox: mailbox@3d067c000 { + compatible = "apple,t8142-aop-setup-mailbox", "apple,t8140-aop-setup-mailbox"; + reg = <0x3 0xd067c000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + /* dart-aop stream 1: ADT vm-base-1 0x20000000000, as on T8140. */ + iommus = <&aop_dart 1>; + apple,dma-range = <0x200 0x0 0x10 0x0>; + #mbox-cells = <0>; + }; + + /* + * dart-aop: streams 0 (AOP), 3, 8 (admac-base-ns), 10 and 16 are the + * AP's; 1, 9 and 11 are exclave streams, of which the host drives 1 + * (setup port) and 11 (admac-leap-s) on the raw boot. As on T8140, the + * firmware's own mappings own the first slot, host buffers go above it. + */ + aop_dart: iommu@390fc0000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x3 0x90fc0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + admac_base_ns: dma-controller@393800000 { + compatible = "apple,t8140-admac"; + reg = <0x3 0x93800000 0x0 0x34000>; + dma-channels = <13>; + interrupts-extended = <&aic AIC_IRQ 549 IRQ_TYPE_LEVEL_HIGH>, + <0>, <0>, <0>; + #dma-cells = <1>; + iommus = <&aop_dart 8>; + apple,dma-range = <0x110 0x00000000 0x0 0x10000000>; + }; + + /* + * admac-leap-s, for the microphone array (ADT audio-hp-mic -> + * leap-s-adma-rx-0), through its admac-leap-s-rx-0-proxy aperture as + * on T8140: the direct window (0x393200000) faults on per-channel + * registers. dart-aop stream 11 with the ADT's vm-base-11/vm-size-11. + * No power domain: the channel registers answer only with the LEAP + * leaves up, which the audio driver raises (after hpai's pw0) before + * it requests the channel; the controller touches no register at + * probe. No resets: the firmware owns the LEAP. + */ + admac_leap_s: dma-controller@3d3200000 { + compatible = "apple,t8140-admac"; + reg = <0x3 0xd3200000 0x0 0x34000>; + dma-channels = <9>; + interrupts-extended = <&aic AIC_IRQ 551 IRQ_TYPE_LEVEL_HIGH>, + <0>, <0>, <0>; + #dma-cells = <1>; + iommus = <&aop_dart 11>; + apple,dma-range = <0x100 0x40000000 0x0 0xffff0000>; + }; + + i2c1: i2c@3a5014000 { + compatible = "apple,t8142-i2c", "apple,t8103-i2c"; + reg = <0x3 0xa5014000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c1>; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 18 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 100 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + sound-name-prefix = "Jack"; + }; + }; + + /* + * Four SN012776: left woofer (0x38, carries the shared shutdown and + * interrupt lines), left tweeter, right woofer, right tweeter. The ADT + * speaker-config lists them as 0x38, then its speaker1..3 (right + * woofer, left tweeter, right tweeter): playback slots 0-3 and V/I + * sense in 8-bit slots 0/2, 4/6, 8/10, 12/14 in that order. The sense + * returns on two lines, the left pair and the right pair; the woofers + * drive the other pair's slots to zero (ADT amp-tx-zd-config, see the + * speaker dai-link). The firmware's speaker frame is 250 BCLKs of + * 12 MHz (ADT audio-speaker stream), which the amplifiers' ratio + * detection rejects. + */ + i2c3: i2c@3a501c000 { + compatible = "apple,t8142-i2c", "apple,t8103-i2c"; + reg = <0x3 0xa501c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c3>; + + speaker_left_woofer: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 150 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer"; + ti,imon-slot-no = <2>; + ti,vmon-slot-no = <0>; + ti,bclk-ratio = <250>; + }; + + speaker_left_tweeter: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + ti,imon-slot-no = <10>; + ti,vmon-slot-no = <8>; + ti,bclk-ratio = <250>; + }; + + speaker_right_woofer: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer"; + ti,imon-slot-no = <6>; + ti,vmon-slot-no = <4>; + ti,bclk-ratio = <250>; + }; + + speaker_right_tweeter: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + shutdown-gpios = <&pinctrl_ap 151 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + ti,imon-slot-no = <14>; + ti,vmon-slot-no = <12>; + ti,bclk-ratio = <250>; + }; + }; + + aop: aop@390c00000 { + compatible = "apple,t8142-aop", "apple,t8152-aop"; + reg = <0x3 0x90c00000 0x0 0x1e0000>, + <0x3 0x90600000 0x0 0x4000>; + reg-names = "mmio", "asc"; + mboxes = <&aop_mbox>, <&aop_setup_mbox>; + mbox-names = "mbox", "setup"; + iommus = <&aop_dart 0>; + apple,dma-range = <0x110 0x0 0x10 0x0>; + + aop_audio: audio { + compatible = "apple,t8142-aop-audio", "apple,t8152-aop-audio"; + /* The microphone card's name: AppleJ813HPAI. */ + apple,chassis-name = "J813"; + apple,machine-kind = "MacBook Air"; + /* hpai: leap-s RX0 (Linux channel 1, odd = DEV_TO_MEM). */ + dmas = <&admac_base_ns 4>, <&admac_base_ns 5>, + <&admac_base_ns 0>, <&admac_base_ns 1>, + <&admac_leap_s 1>; + dma-names = "cout", "cin", "spkr", "sense", "hpai"; + power-domains = <&ps_audio_mca0_m>, <&ps_audio_mca1_m>, + <&ps_audio_mca2_m>, <&ps_audio_mca3_m>, + <&ps_audio_mca4_m>, <&ps_audio_mca5_m>, + <&ps_audio_leap_s_a>, <&ps_audio_leap_rx0>, + <&ps_audio_leap_mca>; + power-domain-names = "mca0", "mca1", "mca2", + "mca3", "mca4", "mca5", + "leap_s_a", "leap_rx0", "leap_mca"; + #sound-dai-cells = <1>; + }; + }; +}; + +/ { + sound { + compatible = "apple,j813-macaudio", "apple,macaudio"; + model = "MacBook Air J813"; + #address-cells = <1>; + #size-cells = <0>; + + dai-link@0 { + reg = <0>; + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&aop_audio 0>; + }; + + codec { + sound-dai = <&jack_codec>; + }; + }; + + dai-link@1 { + reg = <1>; + link-name = "Speaker"; + /* + * Codecs in ADT speaker-config order, so that macaudio + * gives them playback slots 0-3 as macOS does (left + * channels in slots 0 and 2, right in 1 and 3). + */ + dai-tdm-idle-mode-0 = "zero"; + dai-tdm-slot-tx-idle-mask-0 = <0xfff0f0f0>; + dai-tdm-idle-mode-1 = "zero"; + dai-tdm-slot-tx-idle-mask-1 = <0xff0f0f0f>; + + cpu { + sound-dai = <&aop_audio 3>; + }; + + codec { + sound-dai = <&speaker_left_woofer>, <&speaker_right_woofer>, + <&speaker_left_tweeter>, <&speaker_right_tweeter>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index b9c71fd75dacf..dd1526e6caad1 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -403,3 +403,4 @@ #include "t8142-j813-nvme.dtsi" #include "t8142-j813-n1.dtsi" #include "t8142-j813-power.dtsi" +#include "t8142-j813-audio.dtsi" From a175068da072209c552b2c46d5d9a63a787cd038 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 13:22:56 +0100 Subject: [PATCH 101/117] arm64: dts: apple: t8142-j813: Run the camera presets from the full sensor mode The 1280x720, 1440x1080 and 1080x1080 presets used the IMX958's binned mode (config 10, 1520x1440). In it the ISP firmware's AWB never leaves the setfile's default white point and the picture is magenta (mean U/V 160/168 on a grey room). In the full-resolution mode (config 7, 3040x2880), which 1920x1080 already used, it converges. Scale all presets from config 7, the crops doubled. With the firmware's AWB alone the J813 streams a neutral picture at all four sizes (mean U/V 127/127, two streams each), at 30 fps. Signed-off-by: Thomas Glanzmann --- arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi index accb5629a7e65..f3850c4161925 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-isp.dtsi @@ -190,27 +190,33 @@ &isp { apple,temporal-filter = <1>; + /* + * All presets scale from the full-resolution sensor mode (config 7). + * In the binned mode (config 10, 1520x1440) the firmware's AWB keeps + * the setfile's default white point and the picture is magenta; in + * config 7 it converges. + */ sensor-presets { /* 1280x720 */ preset0 { - apple,config-index = <10>; - apple,input-size = <1520 1440>; + apple,config-index = <7>; + apple,input-size = <3040 2880>; apple,output-size = <1280 720>; - apple,crop = <0 292 1520 856>; + apple,crop = <0 584 3040 1710>; }; /* 1440x1080 (4:3) */ preset1 { - apple,config-index = <10>; - apple,input-size = <1520 1440>; + apple,config-index = <7>; + apple,input-size = <3040 2880>; apple,output-size = <1440 1080>; - apple,crop = <40 180 1440 1080>; + apple,crop = <80 360 2880 2160>; }; /* 1080x1080 (1:1) */ preset2 { - apple,config-index = <10>; - apple,input-size = <1520 1440>; + apple,config-index = <7>; + apple,input-size = <3040 2880>; apple,output-size = <1080 1080>; - apple,crop = <220 180 1080 1080>; + apple,crop = <440 360 2160 2160>; }; /* 1920x1080 */ preset3 { From e5f0dad9346cc2f40e6e42a6d6766cf9eb04ce54 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:39:33 +0100 Subject: [PATCH 102/117] soc: apple: rtkit: Take a crash buffer the firmware maps itself The T8142 AOP keeps its crash buffer in its own slot of its DART stream and asks for it on endpoint 1 with that IOVA. A client without shmem_setup refused the request, so the firmware never got its crash buffer acknowledged. APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG takes such a buffer as the firmware's, without a CPU mapping, and acknowledges the request unchanged; a later crash is reported without a log. RtKit::new_with_firmware_crashlog() selects it at runtime, so a driver can use it for one SoC only. Signed-off-by: Thomas Glanzmann --- drivers/soc/apple/rtkit.c | 15 +++++++++++++++ include/linux/soc/apple/rtkit.h | 7 +++++++ rust/bindings/bindings_helper.h | 1 + rust/kernel/soc/apple/rtkit.rs | 16 ++++++++++++++++ 4 files changed, 39 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index d352083aaa26c..9eaa28d16c46d 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -410,6 +410,13 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, dev_dbg(rtk->dev, "RTKit: buffer request for 0x%zx bytes at %pad\n", request.size, &request.iova); + if ((rtk->ops->flags & APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG) && + ep == APPLE_RTKIT_EP_CRASHLOG && request.iova) { + *buffer = request; + buffer->is_mapped = true; + return apple_rtkit_send_message(rtk, ep, msg, NULL, false); + } + if (request.iova && !rtk->ops->shmem_setup && !rtk->ops->shmem_setup_ep) { err = -EINVAL; goto error; @@ -734,6 +741,14 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) dev_err(rtk->dev, "RTKit: co-processor has crashed\n"); + /* A firmware-owned crash buffer has no CPU mapping to read the log from. */ + if (!rtk->crashlog_buffer.buffer && !rtk->crashlog_buffer.iomem) { + rtk->crashed = true; + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); + return; + } + /* * create a shadow copy here to make sure the co-processor isn't able * to change the log while we're dumping it. this also ensures diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 65bd28a6f0240..4c27e012c2c75 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -57,6 +57,13 @@ struct apple_rtkit_inherited_buffer { /* This client requires endpoint 7 setup before IOP power can be acknowledged. */ #define APPLE_RTKIT_NEEDS_COREANALYTICS BIT(1) +/* + * The co-processor keeps its crash buffer in memory it maps itself (the T8142 + * AOP) and asks for it with that IOVA: take the buffer as the firmware's, + * without a CPU mapping, and acknowledge the request unchanged. + */ +#define APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG BIT(2) + /* * Struct to represent implementation-specific RTKit operations. * diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 56f9f84689804..2731a39a0255c 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -158,6 +158,7 @@ const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPROC_PLACES_BUFFERS; const u32 RUST_CONST_HELPER_APPLE_RTKIT_NEEDS_COREANALYTICS = APPLE_RTKIT_NEEDS_COREANALYTICS; +const u32 RUST_CONST_HELPER_APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG = APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG; const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 73896fea53e00..ef9c9e7f27b0f 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -245,6 +245,11 @@ impl RtKit { ..Self::VTABLE }; + const FIRMWARE_CRASHLOG_VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + flags: Self::VTABLE.flags | bindings::APPLE_RTKIT_FIRMWARE_OWNED_CRASHLOG, + ..Self::VTABLE + }; + /// Creates a new RTKit client for a given device and optional mailbox name or index. pub fn new( dev: &device::Device, @@ -268,6 +273,17 @@ impl RtKit { Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::COREANALYTICS_VTABLE) } + /// Creates a client whose co-processor keeps its crash buffer in memory it + /// maps itself and asks for it with that IOVA (the T8142 AOP). + pub fn new_with_firmware_crashlog( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ) -> Result { + Self::new_with_ops(dev, mbox_name, mbox_idx, data, &Self::FIRMWARE_CRASHLOG_VTABLE) + } + fn new_with_ops( dev: &device::Device, mbox_name: Option<&'static CStr>, From ae7ce6378b2e86c09ee7fb7e1fdd2d67d691d46f Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 07:26:19 +0100 Subject: [PATCH 103/117] drm/asahi: mmu: Map at a fixed address with a guard of any size Vm::map_at() reserves at most one guard page after a mapping. The M5 backend maps the driver's pools at the same address in each client VM with two guard pages after each, as it does when it allocates them. Add Vm::map_at_with_guard_size() and express map_at() through it. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/mmu.rs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 122b42ccd1b6d..cffb0e2306d4d 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2551,9 +2551,24 @@ impl Vm { prot: Prot, guard: bool, ) -> Result { - let reserved_size = size - .checked_add(if guard { UAT_PGSZ } else { 0 }) - .ok_or(EOVERFLOW)?; + self.map_at_with_guard_size(addr, size, gem, prot, if guard { UAT_PGSZ } else { 0 }) + } + + /// Map a GEM object into this Vm at a specific address, followed by an unmapped guard of + /// `guard_size` bytes (a multiple of the page size). + #[inline(never)] + pub(crate) fn map_at_with_guard_size( + &self, + addr: u64, + size: usize, + gem: &gem::Object, + prot: Prot, + guard_size: usize, + ) -> Result { + if guard_size & UAT_PGMSK != 0 { + return Err(EINVAL); + } + let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; // End the bound-device lease before entering firmware/GPUVM waits. let (_bind_lease, vm_bo) = { From 588d462030a96495269574523f7c6e64afc0e3f4 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 07:26:20 +0100 Subject: [PATCH 104/117] drm/asahi: mmu: Keep the kernel range of a VM The kernel range of a VM, reserved at VM_CREATE for kernel-managed objects, is only handed to GPUVM. The M5 backend places its per-client objects in that range and checks that userspace commands stay out of it. Keep the range in the Vm and add an accessor. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/mmu.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index cffb0e2306d4d..ae4663134344c 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1046,6 +1046,9 @@ pub(crate) struct Vm { device_binding: Arc, dev: ARef, id: u64, + /// The range reserved for kernel-managed objects at creation. + #[cfg_attr(not(CONFIG_DRM_ASAHI_M5), allow(dead_code))] + kernel_range: Range, mapping_epoch: Option>, bind_sequence: Arc, lifetime: Option>, @@ -2172,7 +2175,7 @@ impl Vm { &dev, &*dummy_obj.gem, gpuvm_range, - kernel_range, + kernel_range.clone(), VmInner { dev: dev.clone(), is_kernel, @@ -2194,6 +2197,7 @@ impl Vm { device_binding, dev: dev.as_ref().as_ref().into(), id, + kernel_range, gpuvm, mapping_epoch, bind_sequence, @@ -2207,6 +2211,12 @@ impl Vm { }) } + /// The range reserved for kernel-managed objects when this VM was created. + #[cfg_attr(not(CONFIG_DRM_ASAHI_M5), allow(dead_code))] + pub(crate) fn kernel_range(&self) -> Range { + self.kernel_range.clone() + } + /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { self.status = Some(Arc::pin_init( From 44bfb4a02aa50aa1d4e392096e03e93671582348 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 07:26:20 +0100 Subject: [PATCH 105/117] drm/asahi: mmu: Add M5 application contexts The M5 (T8142) 25G76 firmware runs application work in a TTBAT context whose lower half is the client VM's root and whose upper half is an empty root of its own; the firmware's upper mappings are not shared with applications. The context is claimed when the work is set up and published explicitly once it is prepared, and a later client takes it over without giving up the context number. Add M5ExecutionContext for that, built only with the M5 backend. It claims numbers from the execution context mask, publishes and retracts under the handoff lock, and a mapping change reaches it through the existing invalidation of every context that names a VM's root. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/mmu.rs | 4 + drivers/gpu/drm/asahi/mmu/m5_context.rs | 173 ++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 drivers/gpu/drm/asahi/mmu/m5_context.rs diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index ae4663134344c..71f1a4b7ba269 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -113,11 +113,15 @@ const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; mod bind; mod context; mod lifetime; +#[cfg(CONFIG_DRM_ASAHI_M5)] +mod m5_context; mod ppl; mod residency; mod shared; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; +#[cfg(CONFIG_DRM_ASAHI_M5)] +pub(crate) use m5_context::M5ExecutionContext; pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; /// PPL magic number for the handoff region diff --git a/drivers/gpu/drm/asahi/mmu/m5_context.rs b/drivers/gpu/drm/asahi/mmu/m5_context.rs new file mode 100644 index 0000000000000..5548d5d0dab6c --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/m5_context.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! M5 (T8142) application contexts. +//! +//! The 25G76 firmware runs application work in a context whose lower half is the client VM's +//! root and whose upper half is a private, empty root: the firmware's own upper mappings are +//! not shared into application contexts. The context is claimed when a queue is set up and its +//! roots are published explicitly, once the work that names it has been prepared. Mapping +//! changes reach a published context through [`VmInner::tlbi_contexts_naming_root`]. + +use super::*; + +/// A claimed M5 application context and the VMs it names. +pub(crate) struct M5ExecutionContext { + inner: Arc, + id: u8, + low: Vm, + high: Vm, + published: bool, +} + +impl Uat { + /// Claims an application context for `vm`, a client VM of this UAT. Nothing is published. + pub(crate) fn new_m5_execution_context(&self, vm: &Vm) -> Result { + self.check_client_vm(vm)?; + let id = { + let mut inner = self.inner.lock(); + let free = !inner.execution_contexts & EXECUTION_CONTEXT_MASK; + if free == 0 { + return Err(ENOSPC); + } + let id = free.trailing_zeros() as usize; + inner.execution_contexts |= 1u64 << id; + id + }; + let high = match self.new_vm(0x8142_1000 + id as u64, self.geometry.user_range()) { + Ok(high) => high, + Err(error) => { + self.inner.lock().execution_contexts &= !(1u64 << id); + return Err(error); + } + }; + Ok(M5ExecutionContext { + inner: self.inner.clone(), + id: id as u8, + low: vm.clone(), + high, + published: false, + }) + } + + fn check_client_vm(&self, vm: &Vm) -> Result { + let owner = vm.gpuvm.lock_inner(); + if owner.is_kernel || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) + { + return Err(EINVAL); + } + Ok(()) + } +} + +impl M5ExecutionContext { + /// The client VM named as the lower half. + pub(crate) fn vm(&self) -> &Vm { + &self.low + } + + /// The hardware context number, as the firmware records name it. + pub(crate) fn context_id(&self) -> u8 { + self.id + } + + /// The TTBAT words of this context. + pub(crate) fn roots(&self) -> [u64; 2] { + let id = usize::from(self.id); + [tagged_root(self.low.ttb(), id), tagged_root(self.high.ttb(), id)] + } + + /// Publishes the roots. A slot that is not empty is never replaced. + pub(crate) fn publish(&mut self) -> Result { + if self.published { + return Err(EBUSY); + } + let [low, high] = self.roots(); + let id = usize::from(self.id); + { + let inner = self.inner.lock(); + inner.handoff_lock(); + let slot = &inner.ttbs()[id]; + let empty = + slot.ttb0.load(Ordering::Acquire) == 0 && slot.ttb1.load(Ordering::Acquire) == 0; + if empty { + slot.ttb0.store(low, Ordering::Release); + slot.ttb1.store(high, Ordering::Release); + inner.publish_slot(id); + } + inner.handoff_unlock(); + if !empty { + return Err(EBUSY); + } + } + self.published = true; + mem::sync(); + mem::tlbi_asid(self.id); + mem::sync(); + Ok(()) + } + + /// Whether this context's roots are published and still in place. + pub(crate) fn is_current(&self) -> bool { + if !self.published { + return false; + } + let [low, high] = self.roots(); + let inner = self.inner.lock(); + let slot = &inner.ttbs()[usize::from(self.id)]; + slot.ttb0.load(Ordering::Acquire) == low && slot.ttb1.load(Ordering::Acquire) == high + } + + /// Names another client VM of the same UAT, keeping the context number. Published roots + /// are retracted first; the caller has made sure no work of the previous VM is outstanding. + pub(crate) fn rebind(&mut self, vm: &Vm) -> Result { + { + let owner = vm.gpuvm.lock_inner(); + if owner.is_kernel + || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) + { + return Err(EINVAL); + } + } + self.retract(); + self.low = vm.clone(); + Ok(()) + } + + fn retract(&mut self) { + if !self.published { + return; + } + let [low, high] = self.roots(); + let id = usize::from(self.id); + { + let inner = self.inner.lock(); + inner.handoff_lock(); + let slot = &inner.ttbs()[id]; + // Only our own pair is retracted. + if slot.ttb0.load(Ordering::Acquire) == low && slot.ttb1.load(Ordering::Acquire) == high + { + slot.ttb0.store(0, Ordering::Release); + slot.ttb1.store(0, Ordering::Release); + inner.publish_slot(id); + } + inner.handoff_unlock(); + } + mem::sync(); + mem::tlbi_asid(self.id); + mem::sync(); + self.published = false; + } +} + +impl Drop for M5ExecutionContext { + fn drop(&mut self) { + self.retract(); + let id = usize::from(self.id); + let mut inner = self.inner.lock(); + let slot = &inner.ttbs()[id]; + // A slot someone else filled is not ours to hand out again: keep it claimed. + if slot.ttb0.load(Ordering::Acquire) == 0 && slot.ttb1.load(Ordering::Acquire) == 0 { + inner.execution_contexts &= !(1u64 << id); + } + } +} From 3488398d99576bcb01e062f89daa0803d15b2869 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 07:26:25 +0100 Subject: [PATCH 106/117] drm/asahi: regs: Add the M5 PMGR boot control The native 25G76 driver sets the low bit of the GPU's PMGR boot control word (SGX + 0xe90000) before either M5 firmware instance starts; the word reads 0x30000 on J813. Add the write, for the M5 backend only, refusing any other starting value. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/regs.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 2b04726e43878..7ed0d552a4879 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -770,6 +770,26 @@ impl G17Resources { self.sgx_read32::() } + /// M5: set the low bit of the PMGR boot control word at SGX+0xe90000, as the native + /// 25G76 driver does before either firmware role starts. The word reads 0x30000 (or + /// 0x30001 when already set) on J813; anything else is refused. + #[cfg(CONFIG_DRM_ASAHI_M5)] + pub(crate) fn m5_match_pmgr_boot_control(&self) -> Result { + const OFFSET: usize = 0xe90000; + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = (&*sgx).as_view(); + let before = sgx.try_read32(OFFSET)?; + if before != 0x30000 && before != 0x30001 { + dev_err!(self.dev.as_ref(), "M5: unexpected PMGR boot control {:#x}\n", before); + return Err(EIO); + } + sgx.try_write32(before | 1, OFFSET)?; + if sgx.try_read32(OFFSET)? != 0x30001 { + return Err(EIO); + } + Ok(()) + } + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. pub(crate) fn enable_g17_gating(&self) -> Result { From db779e90187743eba7344a2d963e784617ab4ac1 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 2 Nov 2026 07:27:08 +0100 Subject: [PATCH 107/117] drm/asahi: Add the M5 (T8142) backend Add a backend for the M5 GPU (J813, G17G) running the 25G76 firmware (RTKit-3255.160.4), as ProbeConfig::M5 for apple,agx-t8142 behind CONFIG_DRM_ASAHI_M5. It changes nothing for the other SoCs. The 25G76 firmware has two coprocessors, GFX and GFX1, as the A18 Pro has, but a HAL300 interface that differs from the 26A428 and 25G83 builds the shared G17 code runs. The backend therefore builds its own initialization graph and talks to the firmware over its own queues, while using the shared parts of the driver: - the UAT is the shared one with published roots, adopted and mirrored as on the A18 Pro, plus the M5 application contexts; - user VMs carry a status and a job lifetime, and the backend uses the tracked submission model of file.rs; - the UAPI is the shared one. The backend reports VM_STATUS and SOFT_FAULTS and one hardware command per submission. The boot loader (aura) loads the firmware, checks both TEXT images against the 25G76 build and says so in /chosen/aurora,m5-gpu-firmware-profile; the backend refuses to start without it. One render pair and one compute queue serve all clients and are handed from client VM to client VM; a submission is polled to retirement before the ioctl returns. Renders and compute are admitted only within the state the firmware programs are known to handle, and refused with ENOTSUPP otherwise. There is no recovery yet: a job that fails stops both coprocessors and retains every firmware-visible object until reboot. The driver's pools are mapped at their own address in each client VM the job is handed to, except in the VM that owns them, and the compute storage is reserved together with the parameter buffer, so that its address is free in every such VM. Tested on J813 (M5, 8 GPU cores) on the aura one-shot trial boot, with Mesa j813, against the same Mesa on the J813 kernel this code comes from: GLES2 16016 pass, 930 fail (identical per case); GLES3 40832 pass, 3547 fail, 2 timeouts (identical but for one known flake); GLES31 32813 pass, 4517 fail (t70: 32804/4520; the 32 differing cases are 9 fail->pass and flakes); the sway desktop renders on the GPU from boot, also during the CTS runs. This consolidates the M5 work of Acelogic and the G17P render, UAPI and initdata encoders of DjDeveloperr , with contributions by Ryan Murray, Eryk Wieliczko and Chris Kearney, from the J813 kernel tree, ported onto integration's driver. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/Kconfig | 8 + drivers/gpu/drm/asahi/asahi_core.rs | 6 + drivers/gpu/drm/asahi/driver.rs | 41 + drivers/gpu/drm/asahi/m5/channel.rs | 183 ++ drivers/gpu/drm/asahi/m5/command.rs | 1233 +++++++++++++ drivers/gpu/drm/asahi/m5/completion.rs | 61 + drivers/gpu/drm/asahi/m5/compute.rs | 246 +++ drivers/gpu/drm/asahi/m5/drm.rs | 386 ++++ drivers/gpu/drm/asahi/m5/fw_msg.rs | 57 + drivers/gpu/drm/asahi/m5/initdata.rs | 1630 +++++++++++++++++ drivers/gpu/drm/asahi/m5/live_boot.rs | 527 ++++++ drivers/gpu/drm/asahi/m5/mapped.rs | 145 ++ drivers/gpu/drm/asahi/m5/mod.rs | 91 + drivers/gpu/drm/asahi/m5/provider.rs | 199 ++ drivers/gpu/drm/asahi/m5/render.rs | 508 +++++ drivers/gpu/drm/asahi/m5/render_desc.rs | 1138 ++++++++++++ drivers/gpu/drm/asahi/m5/render_target.rs | 72 + drivers/gpu/drm/asahi/m5/resources.rs | 631 +++++++ .../asahi/m5/resources/application_compute.rs | 305 +++ .../drm/asahi/m5/resources/firmware_view.rs | 75 + .../drm/asahi/m5/resources/graphics_job.rs | 512 ++++++ .../drm/asahi/m5/resources/graphics_memory.rs | 198 ++ .../drm/asahi/m5/resources/graphics_queues.rs | 261 +++ .../drm/asahi/m5/resources/graphics_usc.rs | 203 ++ .../asahi/m5/resources/parameter_buffer.rs | 139 ++ .../drm/asahi/m5/resources/target_memory.rs | 75 + drivers/gpu/drm/asahi/m5/sksm.rs | 271 +++ drivers/gpu/drm/asahi/m5/submission.rs | 51 + drivers/gpu/drm/asahi/m5/vm_layout.rs | 38 + include/uapi/drm/asahi_drm.h | 5 +- 30 files changed, 9293 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/asahi/m5/channel.rs create mode 100644 drivers/gpu/drm/asahi/m5/command.rs create mode 100644 drivers/gpu/drm/asahi/m5/completion.rs create mode 100644 drivers/gpu/drm/asahi/m5/compute.rs create mode 100644 drivers/gpu/drm/asahi/m5/drm.rs create mode 100644 drivers/gpu/drm/asahi/m5/fw_msg.rs create mode 100644 drivers/gpu/drm/asahi/m5/initdata.rs create mode 100644 drivers/gpu/drm/asahi/m5/live_boot.rs create mode 100644 drivers/gpu/drm/asahi/m5/mapped.rs create mode 100644 drivers/gpu/drm/asahi/m5/mod.rs create mode 100644 drivers/gpu/drm/asahi/m5/provider.rs create mode 100644 drivers/gpu/drm/asahi/m5/render.rs create mode 100644 drivers/gpu/drm/asahi/m5/render_desc.rs create mode 100644 drivers/gpu/drm/asahi/m5/render_target.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/application_compute.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/firmware_view.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/graphics_job.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/graphics_memory.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/graphics_queues.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/graphics_usc.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/parameter_buffer.rs create mode 100644 drivers/gpu/drm/asahi/m5/resources/target_memory.rs create mode 100644 drivers/gpu/drm/asahi/m5/sksm.rs create mode 100644 drivers/gpu/drm/asahi/m5/submission.rs create mode 100644 drivers/gpu/drm/asahi/m5/vm_layout.rs diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 6b28266b6b4d1..af5e7e9e8cb87 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -43,6 +43,14 @@ config DRM_ASAHI_G16 An aborted receipt disables that command class until reboot. Suspend, hibernation and kexec are blocked while native graphics owns DMA. +config DRM_ASAHI_M5 + bool "Apple M5 (G17G, T8142) support (experimental)" + depends on DRM_ASAHI && ARM64 + help + Backend for the M5 GPU (J813 MacBook Air) running the 25G76 + firmware (RTKit-3255.160.4), as loaded and published by the aura + boot loader (/chosen/aurora,m5-gpu-firmware-profile). + config DRM_ASAHI_DEBUG_ALLOCATOR bool "Use debug allocator" depends on DRM_ASAHI diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index decf16ea09647..17b64ec578d5f 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -22,6 +22,8 @@ mod gem; #[cfg(CONFIG_DRM_ASAHI_G16)] mod g16; +#[cfg(CONFIG_DRM_ASAHI_M5)] +mod m5; mod gpu; mod hw; mod initdata; @@ -198,6 +200,10 @@ module! { default: 1, description: "Count held GPU memory in its creator's RSS for the OOM killer (1 = on)", }, + m5_fw_trace: u32 { + default: 0, + description: "M5: firmware trace classes (power bit 0, submission bit 1, recovery classifier bit 9)", + }, g16_deadline_ms: u32 { default: 5000, description: "G16: administrative completion-stamp deadline, ms (0 disables; long running work can exceed it)", diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index ff82d8c118e85..554a82285c865 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -142,6 +142,8 @@ pub(crate) enum ProbeConfig { Legacy(&'static hw::HwConfig), G17(&'static g17::Config), M3(crate::m3::Chip), + /// The M5 (T8142) backend, refused unless it is built. + M5, } // OF Device ID table. Linux 7.3's of_device_table!() takes no attributes on @@ -165,6 +167,10 @@ macro_rules! asahi_of_device_table { of::DeviceId::new(c"apple,agx-t6041"), ProbeConfig::G17(&hw::t6041::CONFIG) ), + ( + of::DeviceId::new(c"apple,agx-t8142"), + ProbeConfig::M5 + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) @@ -253,6 +259,10 @@ impl platform::Driver for AsahiPlatformDriver { ProbeConfig::Legacy(cfg) => *cfg, ProbeConfig::G17(cfg) => return AsahiPlatformDriver::probe_g17(pdev, cfg), ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), + #[cfg(CONFIG_DRM_ASAHI_M5)] + ProbeConfig::M5 => return AsahiPlatformDriver::probe_m5(pdev), + #[cfg(not(CONFIG_DRM_ASAHI_M5))] + ProbeConfig::M5 => return Err(ENODEV), }; // SAFETY: The selected SoC's UAT output width bounds all device DMA. @@ -444,6 +454,37 @@ impl AsahiPlatformDriver { } } +#[cfg(CONFIG_DRM_ASAHI_M5)] +impl AsahiPlatformDriver { + fn probe_m5<'bound>( + pdev: &'bound platform::Device>, + ) -> Result> { + // SAFETY: The M5 UAT has 42 output bits. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; + let resources = regs::Resources::new(pdev)?; + let res = regs::G17Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let gpu = crate::m5::probe(pdev, &drm, res)?; + let data = try_pin_init!(AsahiDrmRegistrationData { + gpu: gpu as Arc, + resources, + }); + // SAFETY: The registration is stored and unregistered by platform data destruction. + // The M5 firmware session retains its firmware-visible state until reboot. + let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + Ok(AsahiPlatformDriverData { + _device: reg.device().into(), + _reg: reg, + g17: None, + #[cfg(CONFIG_DRM_ASAHI_G16)] + g16: None, + raw_dev: core::ptr::null_mut(), + raw_stats: core::ptr::null_mut(), + stats: None, + }) + } +} + #[cfg(CONFIG_DRM_ASAHI_G16)] impl AsahiPlatformDriver { pub(crate) fn probe_g16<'bound>( diff --git a/drivers/gpu/drm/asahi/m5/channel.rs b/drivers/gpu/drm/asahi/m5/channel.rs new file mode 100644 index 0000000000000..259d79209a0f7 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/channel.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8142/25G76 channel configuration, separate from Neo's HAL200 path. +//! +//! Native AGXCLChannelSKSM::submitBuffer inlines this packed 0x86-byte +//! record at 0xfffffe000898488c..0x8984bd8. The controlled native sequence +//! is a selector-3 register program, this tag-15 record, then a tag-14 kick. +//! These builders do not map memory, install a queue or publish a producer. +//! Every pointer and event identifier must belong to the retained session. + +pub(crate) const M5_CHANNEL_CONFIG_BYTES: usize = 0x86; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5ChannelError { + QueueId, + Priority, + Address, + Context, + DataMaster, +} + +/// Optional TA parameter-buffer event, separate from the first event tuple. +/// The native TA writer at 25G76 0x8a249ac..0x8a249e4 obtains these values +/// from its retained PM manager and command descriptor. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ParameterBufferEvent { + pub(crate) address: u64, + pub(crate) timestamp: u64, + /// TA descriptor +0x1218, independently selected by the owned PB table. + pub(crate) hardware_buffer_id: u32, + /// Native PM manager +0x79c. This is not the SKSM queue ID. + pub(crate) manager_id: u32, +} + +/// Optional retained event-machine object, with the native timestamp and +/// index. None emits the native disabled branch's full-width sentinels. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ChannelEvent { + pub(crate) address: u64, + pub(crate) timestamp: u64, + pub(crate) index: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ChannelConfig { + pub(crate) ring_low: u64, + pub(crate) ring_high: u64, + pub(crate) queue_id: u8, + pub(crate) priority: u8, + /// Native descriptor +0x181 bit 0. Mode 3 validates the first queue + /// installation on M5; an already-installed native capture has 0. + pub(crate) install_queue: bool, + /// Native descriptor +0x144, independently matched to the UAT context. + pub(crate) gpu_context: u8, + pub(crate) event: Option, + /// AGXCommandQueue +0x8b8: retained scheduler/control object. + pub(crate) channel_control: u64, + /// Native descriptor +0x2dc bit 0. + pub(crate) update_scheduler: bool, + /// AGXCommandQueue +0x898 object's +0x10. Not the GPU context ID. + pub(crate) scheduler_index: u32, + /// getStampIndex on descriptor +0x170. Not a fixed target constant. + pub(crate) channel_stamp_index: u32, + /// Native descriptor +0x180 bit 0. + pub(crate) update_context: bool, + /// Descriptor +0x58 owner's virtual context value. Distinct from UAT. + pub(crate) owner_context: u32, + /// Native channel-state +0x84 is nonzero. + pub(crate) extended_command: bool, +} + +fn firmware_address(address: u64) -> bool { + address >= 0xffff_fc00_0000_0000 && address & 7 == 0 +} + +fn put32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +pub(crate) fn encode_m5_channel_config( + p: M5ChannelConfig, +) -> Result<[u8; M5_CHANNEL_CONFIG_BYTES], M5ChannelError> { + encode_m5_channel_config_for(p, 2, 2, None, None) +} + +/// The two selectors are independent: native TA/fragment use 0/1 at +0x22 +/// while both retain 2 at +0x5e. Changing both by stage breaks that contract. +/// gpu_state_update retains the separate +0x26 flag/+0x2a value pair; +/// the native TA writer copies descriptor +0xe41/+0xe48 to these fields. +pub(crate) fn encode_m5_channel_config_for( + p: M5ChannelConfig, + data_master: u32, + channel_data_master: u32, + gpu_state_update: Option, + parameter_buffer: Option, +) -> Result<[u8; M5_CHANNEL_CONFIG_BYTES], M5ChannelError> { + if data_master > 2 + || channel_data_master > 2 + || (parameter_buffer.is_some() && data_master != 0) + { + return Err(M5ChannelError::DataMaster); + } + if p.queue_id > 127 { + return Err(M5ChannelError::QueueId); + } + if p.priority > 3 { + return Err(M5ChannelError::Priority); + } + if p.gpu_context == 0 { + return Err(M5ChannelError::Context); + } + if p.ring_low == 0 + || p.ring_low >= (1 << 42) + || p.ring_low & 0x3fff != 0 + || !firmware_address(p.ring_high) + || p.ring_high & 0x3fff != 0 + || !firmware_address(p.channel_control) + { + return Err(M5ChannelError::Address); + } + if let Some(event) = p.event { + if !firmware_address(event.address) { + return Err(M5ChannelError::Address); + } + } + if let Some(event) = parameter_buffer { + if !firmware_address(event.address) { + return Err(M5ChannelError::Address); + } + } + let mut bytes = [0; M5_CHANNEL_CONFIG_BYTES]; + put32(&mut bytes, 0, 15); + put64(&mut bytes, 8, p.ring_low); + put64(&mut bytes, 0x10, p.ring_high); + bytes[0x18..0x1a].copy_from_slice(&u16::from(p.queue_id).to_le_bytes()); + for (offset, value) in [ + (0x1a, u32::from(p.install_queue)), + (0x1e, u32::from(p.priority)), + (0x22, data_master), + (0x32, u32::from(p.gpu_context)), + (0x52, u32::from(p.update_scheduler)), + (0x56, p.scheduler_index), + (0x5a, p.channel_stamp_index), + (0x5e, channel_data_master), + (0x62, u32::from(p.update_context)), + (0x66, p.owner_context), + (0x6a, u32::from(p.extended_command)), + ] { + put32(&mut bytes, offset, value); + } + if let Some(value) = gpu_state_update { + put32(&mut bytes, 0x26, 1); + put64(&mut bytes, 0x2a, value); + } + match p.event { + Some(event) => { + put64(&mut bytes, 0x36, event.address); + put64(&mut bytes, 0x3e, event.timestamp); + put32(&mut bytes, 0x46, event.index); + } + None => { + put64(&mut bytes, 0x3e, u64::MAX); + put32(&mut bytes, 0x46, u32::MAX); + } + } + put64(&mut bytes, 0x4a, p.channel_control); + if let Some(event) = parameter_buffer { + put64(&mut bytes, 0x6e, event.address); + put64(&mut bytes, 0x76, event.timestamp); + put32(&mut bytes, 0x7e, event.hardware_buffer_id); + put32(&mut bytes, 0x82, event.manager_id); + } else { + bytes[0x76..0x86].fill(0xff); + } + Ok(bytes) +} + diff --git a/drivers/gpu/drm/asahi/m5/command.rs b/drivers/gpu/drm/asahi/m5/command.rs new file mode 100644 index 0000000000000..c4555602c781e --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/command.rs @@ -0,0 +1,1233 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +use crate::m5::{render_desc, submission}; + +use render_desc::{ + G17pRenderParameters, RenderBuildError, +}; + +pub(crate) const UAPI_COMMAND_HEADER_SIZE: usize = 8; + +pub(crate) const UAPI_RENDER_SIZE: usize = 272; + +pub(crate) const UAPI_COMPUTE_SIZE: usize = 88; + +pub(crate) const UAPI_G18_COMPUTE_SIZE: usize = 136; + +pub(crate) const COMPUTE_G18_RESOURCES: u32 = 1 << 2; + +/// `drm_asahi_cmd_render_g18`: the render payload plus its 80-byte suffix. +pub(crate) const UAPI_G18_RENDER_SIZE: usize = UAPI_RENDER_SIZE + 80; + +/// `DRM_ASAHI_RENDER_G18_RESOURCES`. Only the G18 adapter may consume and +/// strip it; the shared translator's RENDER_FLAGS keeps refusing it. +pub(crate) const RENDER_G18_RESOURCES: u32 = 1 << 21; + +pub(crate) const UAPI_ATTACHMENT_SIZE: usize = 24; + +pub(crate) const UAPI_MAX_ATTACHMENTS: usize = 16; + +pub(crate) const UAPI_MAX_HARDWARE_COMMANDS: u32 = 64; + +pub(crate) const M5_MAX_RENDER_DIMENSION: u32 = 16384; + +pub(crate) const UAPI_BARRIER_NONE: u16 = 0xffff; + +pub(crate) const USC_WINDOW_SIZE: u64 = 1 << 32; + +pub(crate) const COMPUTE_G17P_ADD3_PROOF: u32 = + kernel::uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF as u32; + +const G17P_ADD3_BUFFER_BYTES: u64 = 64 * 4; + +const COMMAND_RENDER: u16 = 0; + +const COMMAND_COMPUTE: u16 = 1; + +const SET_VERTEX_ATTACHMENTS: u16 = 2; + +const SET_FRAGMENT_ATTACHMENTS: u16 = 3; + +const SET_COMPUTE_ATTACHMENTS: u16 = 4; + +const RENDER_VERTEX_SCRATCH: u32 = 1 << 0; + +const RENDER_PROCESS_EMPTY_TILES: u32 = 1 << 1; + +const RENDER_NO_VERTEX_CLUSTERING: u32 = 1 << 2; + +const RENDER_RSRC_SPEC_HI: u32 = 1 << 4; + +const RENDER_DBIAS_IS_INT: u32 = 1 << 18; + +const RENDER_FLAGS: u32 = RENDER_VERTEX_SCRATCH + | RENDER_PROCESS_EMPTY_TILES + | RENDER_NO_VERTEX_CLUSTERING + | RENDER_RSRC_SPEC_HI + | RENDER_DBIAS_IS_INT + // Private, opt-in matched-workload diagnostic. Ordinary builds reject it. + | if cfg!(g17p_aux_fb_small_probe) { 1 << 31 } else { 0 } + | if cfg!(g17p_mcache_aperture_probe) { 1 << 30 } else { 0 }; + +const SAMPLER_SIZE: u64 = 8; + +const MAX_SAMPLERS: u32 = 1024; + +const PAGE_SIZE: u64 = 0x4000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum UapiTranslateError { + HeaderTruncated, + PayloadTruncated, + PayloadTail, + CommandType, + AttachmentPayloadSize, + AttachmentCount, + AttachmentFields, + AttachmentBarrier, + HardwareCommandCount, + BarrierFuture, + QueueUscAlignment, + QueueUscRange, + AddressRange, + AddressAlignment, + FlagsUnknownBits, + FlagsUnsupported, + Dimensions, + Layers, + UtileDimensions, + Samples, + TilebufferSize, + SamplerCount, + SamplerPair, + HelperProgram, + Add3ProofAttachments, + ProgramOffset, + ZlsFields, + RenderBuild(RenderBuildError), +} + +impl From for UapiTranslateError { + fn from(value: RenderBuildError) -> Self { + Self::RenderBuild(value) + } +} + +impl UapiTranslateError { +} + +fn get_u16(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) +} + +fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) +} + +fn padded_payload(raw: &[u8]) -> Result<[u8; N], UapiTranslateError> { + if raw.len() > N && raw[N..].iter().any(|byte| *byte != 0) { + return Err(UapiTranslateError::PayloadTail); + } + let mut out = [0u8; N]; + let count = core::cmp::min(raw.len(), N); + out[..count].copy_from_slice(&raw[..count]); + Ok(out) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiCommandHeader { + pub(crate) command_type: u16, + pub(crate) size: u16, + pub(crate) render_barrier: u16, + pub(crate) compute_barrier: u16, +} + +impl UapiCommandHeader { + fn parse(raw: &[u8]) -> Result { + if raw.len() < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + Ok(Self { + command_type: get_u16(raw, 0), + size: get_u16(raw, 2), + render_barrier: get_u16(raw, 4), + compute_barrier: get_u16(raw, 6), + }) + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct UapiAttachment { + pub(crate) address: u64, + pub(crate) size: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiAttachmentList { + pub(crate) entries: [UapiAttachment; UAPI_MAX_ATTACHMENTS], + pub(crate) count: u8, +} + +impl UapiAttachmentList { + pub(crate) const EMPTY: Self = Self { + entries: [UapiAttachment { + address: 0, + size: 0, + }; UAPI_MAX_ATTACHMENTS], + count: 0, + }; + + pub(crate) fn as_slice(&self) -> &[UapiAttachment] { + &self.entries[..self.count as usize] + } + + fn parse(raw: &[u8]) -> Result { + if raw.len() % UAPI_ATTACHMENT_SIZE != 0 { + return Err(UapiTranslateError::AttachmentPayloadSize); + } + let count = raw.len() / UAPI_ATTACHMENT_SIZE; + if count > UAPI_MAX_ATTACHMENTS { + return Err(UapiTranslateError::AttachmentCount); + } + let mut out = Self::EMPTY; + for index in 0..count { + let offset = index * UAPI_ATTACHMENT_SIZE; + if get_u32(raw, offset + 0x10) != 0 || get_u32(raw, offset + 0x14) != 0 { + return Err(UapiTranslateError::AttachmentFields); + } + out.entries[index] = UapiAttachment { + address: get_u64(raw, offset), + size: get_u64(raw, offset + 8), + }; + } + out.count = count as u8; + Ok(out) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiHelperProgram { + pub(crate) binary: u32, + pub(crate) config: u32, + pub(crate) data: u64, +} + +fn helper(raw: &[u8], offset: usize) -> UapiHelperProgram { + UapiHelperProgram { + binary: get_u32(raw, offset), + config: get_u32(raw, offset + 4), + data: get_u64(raw, offset + 8), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiProgram { + pub(crate) usc: u32, + pub(crate) resource_spec: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamp { + pub(crate) handle: u32, + pub(crate) offset: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamps { + pub(crate) start: UapiTimestamp, + pub(crate) end: UapiTimestamp, +} + +fn timestamps(raw: &[u8], offset: usize) -> UapiTimestamps { + UapiTimestamps { + start: UapiTimestamp { + handle: get_u32(raw, offset), + offset: get_u32(raw, offset + 4), + }, + end: UapiTimestamp { + handle: get_u32(raw, offset + 8), + offset: get_u32(raw, offset + 12), + }, + } +} + +fn program(raw: &[u8], offset: usize, high_offset: usize) -> UapiProgram { + UapiProgram { + usc: get_u32(raw, offset), + resource_spec: get_u32(raw, offset + 4) as u64 + | (get_u32(raw, high_offset) as u64) << 32, + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiZlsBuffer { + pub(crate) base: u64, + pub(crate) compression_base: u64, + pub(crate) stride: u32, + pub(crate) compression_stride: u32, +} + +fn zls(raw: &[u8], offset: usize) -> UapiZlsBuffer { + UapiZlsBuffer { + base: get_u64(raw, offset), + compression_base: get_u64(raw, offset + 8), + stride: get_u32(raw, offset + 0x10), + compression_stride: get_u32(raw, offset + 0x14), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiRenderCommand { + pub(crate) flags: u32, + pub(crate) depth_dimensions: u32, + pub(crate) vdm_base: u64, + pub(crate) vertex_helper: UapiHelperProgram, + pub(crate) fragment_helper: UapiHelperProgram, + pub(crate) scissor_base: u64, + pub(crate) depth_bias_base: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth: UapiZlsBuffer, + pub(crate) stencil: UapiZlsBuffer, + pub(crate) zls_control: u64, + pub(crate) multisample_control: u64, + pub(crate) sampler_heap: u64, + pub(crate) ppp_control: u32, + pub(crate) width: u16, + pub(crate) height: u16, + pub(crate) layers: u16, + pub(crate) sampler_count: u16, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) samples: u8, + pub(crate) sample_size: u8, + pub(crate) merge_upper_x: u32, + pub(crate) merge_upper_y: u32, + pub(crate) background: UapiProgram, + pub(crate) end_of_tile: UapiProgram, + pub(crate) partial_background: UapiProgram, + pub(crate) partial_end_of_tile: UapiProgram, + pub(crate) depth_clear: u32, + pub(crate) stencil_clear: u32, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, + pub(crate) prior_vdm: u64, + pub(crate) prior_cdm: u64, + /// `drm_asahi_g18_render_resources` words; zero unless flagged. + pub(crate) g18_resources: [u32; 20], +} + +impl UapiRenderCommand { + fn parse(raw: &[u8]) -> Result { + // Legacy payloads are zero-extended exactly as before. A flagged G18 + // suffix must be complete; padding could hide a truncated field. + let has_resources = raw.len() >= 4 && get_u32(raw, 0) & RENDER_G18_RESOURCES != 0; + if has_resources && raw.len() < UAPI_G18_RENDER_SIZE { + return Err(UapiTranslateError::PayloadTruncated); + } + if !has_resources && raw.len() > UAPI_RENDER_SIZE + && raw[UAPI_RENDER_SIZE..].iter().any(|byte| *byte != 0) + { + return Err(UapiTranslateError::PayloadTail); + } + let raw = padded_payload::(raw)?; + let mut g18_resources = [0; 20]; + if has_resources { + for (i, word) in g18_resources.iter_mut().enumerate() { + *word = get_u32(&raw, UAPI_RENDER_SIZE + i * 4); + } + } + Ok(Self { + flags: get_u32(&raw, 0x00), + depth_dimensions: get_u32(&raw, 0x04), + vdm_base: get_u64(&raw, 0x08), + vertex_helper: helper(&raw, 0x10), + fragment_helper: helper(&raw, 0x20), + scissor_base: get_u64(&raw, 0x30), + depth_bias_base: get_u64(&raw, 0x38), + occlusion_query_base: get_u64(&raw, 0x40), + depth: zls(&raw, 0x48), + stencil: zls(&raw, 0x60), + zls_control: get_u64(&raw, 0x78), + multisample_control: get_u64(&raw, 0x80), + sampler_heap: get_u64(&raw, 0x88), + ppp_control: get_u32(&raw, 0x90), + width: get_u16(&raw, 0x94), + height: get_u16(&raw, 0x96), + layers: get_u16(&raw, 0x98), + sampler_count: get_u16(&raw, 0x9a), + utile_width: raw[0x9c], + utile_height: raw[0x9d], + samples: raw[0x9e], + sample_size: raw[0x9f], + merge_upper_x: get_u32(&raw, 0xa0), + merge_upper_y: get_u32(&raw, 0xa4), + background: program(&raw, 0xa8, 0xf8), + end_of_tile: program(&raw, 0xb0, 0xf0), + partial_background: program(&raw, 0xb8, 0xfc), + partial_end_of_tile: program(&raw, 0xc0, 0xf4), + depth_clear: get_u32(&raw, 0xc8), + stencil_clear: get_u32(&raw, 0xcc), + vertex_timestamps: timestamps(&raw, 0xd0), + fragment_timestamps: timestamps(&raw, 0xe0), + prior_vdm: get_u64(&raw, 0x100), + prior_cdm: get_u64(&raw, 0x108), + g18_resources, + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiComputeCommand { + pub(crate) flags: u32, + pub(crate) sampler_count: u32, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) sampler_heap: u64, + pub(crate) helper: UapiHelperProgram, + pub(crate) timestamps: UapiTimestamps, + pub(crate) scs_layout: u64, + pub(crate) g18_resources: [u32; 16], + pub(crate) prior_vdm: u64, + pub(crate) prior_cdm: u64, +} + +impl UapiComputeCommand { + fn parse(raw: &[u8]) -> Result { + // Legacy compute payloads are zero-extended as before. A flagged + // metadata payload must be complete; padding it could hide truncation + // of a valid zero-valued field or reserved word. + let has_resources = raw.len() >= 4 && get_u32(raw, 0) & COMPUTE_G18_RESOURCES != 0; + if has_resources && raw.len() < UAPI_G18_COMPUTE_SIZE { + return Err(UapiTranslateError::PayloadTruncated); + } + if !has_resources && raw.len() > UAPI_COMPUTE_SIZE + && raw[UAPI_COMPUTE_SIZE..].iter().any(|byte| *byte != 0) + { + return Err(UapiTranslateError::PayloadTail); + } + let raw = padded_payload::(raw)?; + let mut g18_resources = [0; 16]; + if has_resources { + for (i, word) in g18_resources.iter_mut().enumerate() { + *word = get_u32(&raw, 72 + i * 4); + } + } + Ok(Self { + flags: get_u32(&raw, 0), + sampler_count: get_u32(&raw, 4), + control_stream_base: get_u64(&raw, 8), + control_stream_end: get_u64(&raw, 0x10), + sampler_heap: get_u64(&raw, 0x18), + helper: helper(&raw, 0x20), + timestamps: timestamps(&raw, 0x30), + scs_layout: get_u64(&raw, 0x40), + g18_resources, + prior_vdm: if has_resources { 0 } else { get_u64(&raw, 0x48) }, + prior_cdm: if has_resources { 0 } else { get_u64(&raw, 0x50) }, + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ParsedHardwareCommand { + Render { + index: u32, + header: UapiCommandHeader, + payload: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + }, + Compute { + index: u32, + header: UapiCommandHeader, + payload: UapiComputeCommand, + attachments: UapiAttachmentList, + }, +} + +pub(crate) struct UapiCommandParser<'a> { + raw: &'a [u8], + offset: usize, + command_index: u32, + hardware_count: u32, + render_count: u16, + compute_count: u16, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + compute_attachments: UapiAttachmentList, +} + +impl<'a> UapiCommandParser<'a> { + pub(crate) const fn new(raw: &'a [u8]) -> Self { + Self { + raw, + offset: 0, + command_index: 0, + hardware_count: 0, + render_count: 0, + compute_count: 0, + vertex_attachments: UapiAttachmentList::EMPTY, + fragment_attachments: UapiAttachmentList::EMPTY, + compute_attachments: UapiAttachmentList::EMPTY, + } + } + + pub(crate) fn finish(&self) -> Result { + if self.offset != self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + if self.hardware_count == 0 || self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + Ok(self.hardware_count) + } + + pub(crate) fn next_hardware( + &mut self, + ) -> Result, UapiTranslateError> { + loop { + if self.offset == self.raw.len() { + return Ok(None); + } + if self.raw.len() - self.offset < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + let header = UapiCommandHeader::parse(&self.raw[self.offset..])?; + self.offset += UAPI_COMMAND_HEADER_SIZE; + let end = self + .offset + .checked_add(header.size as usize) + .ok_or(UapiTranslateError::PayloadTruncated)?; + if end > self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + let payload = &self.raw[self.offset..end]; + self.offset = end; + self.command_index += 1; + + match header.command_type { + COMMAND_RENDER | COMMAND_COMPUTE => { + if header.render_barrier != UAPI_BARRIER_NONE + && header.render_barrier > self.render_count + { + return Err(UapiTranslateError::BarrierFuture); + } + if header.compute_barrier != UAPI_BARRIER_NONE + && header.compute_barrier > self.compute_count + { + return Err(UapiTranslateError::BarrierFuture); + } + self.hardware_count += 1; + if self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + if header.command_type == COMMAND_RENDER { + self.render_count += 1; + return Ok(Some(ParsedHardwareCommand::Render { + index: self.command_index, + header, + payload: UapiRenderCommand::parse(payload)?, + vertex_attachments: self.vertex_attachments, + fragment_attachments: self.fragment_attachments, + })); + } + self.compute_count += 1; + return Ok(Some(ParsedHardwareCommand::Compute { + index: self.command_index, + header, + payload: UapiComputeCommand::parse(payload)?, + attachments: self.compute_attachments, + })); + } + SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS | SET_COMPUTE_ATTACHMENTS => { + if header.render_barrier != UAPI_BARRIER_NONE + || header.compute_barrier != UAPI_BARRIER_NONE + { + return Err(UapiTranslateError::AttachmentBarrier); + } + let attachments = UapiAttachmentList::parse(payload)?; + match header.command_type { + SET_VERTEX_ATTACHMENTS => self.vertex_attachments = attachments, + SET_FRAGMENT_ATTACHMENTS => self.fragment_attachments = attachments, + SET_COMPUTE_ATTACHMENTS => self.compute_attachments = attachments, + _ => unreachable!(), + } + } + _ => return Err(UapiTranslateError::CommandType), + } + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum GpuAccess { + Read, + Write, + ReadWrite, +} + +pub(crate) trait GpuAddressSpace { + fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool; +} + +fn require_range( + address_space: &A, + address: u64, + size: u64, + access: GpuAccess, +) -> Result<(), UapiTranslateError> { + let end = address + .checked_add(size) + .ok_or(UapiTranslateError::AddressRange)?; + if size == 0 || end <= address || !address_space.covers(address, size, access) { + return Err(UapiTranslateError::AddressRange); + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueueUscWindow { + pub(crate) base: u64, + pub(crate) user_start: u64, + pub(crate) user_end: u64, +} + +impl QueueUscWindow { + pub(crate) fn validate(self) -> Result<(), UapiTranslateError> { + if self.base & (USC_WINDOW_SIZE - 1) != 0 { + return Err(UapiTranslateError::QueueUscAlignment); + } + let end = self + .base + .checked_add(USC_WINDOW_SIZE) + .ok_or(UapiTranslateError::QueueUscRange)?; + if self.base < self.user_start || end > self.user_end || end <= self.base { + return Err(UapiTranslateError::QueueUscRange); + } + Ok(()) + } + + fn full_program( + self, + address_space: &A, + tagged_offset: u32, + ) -> Result { + self.validate()?; + if tagged_offset == 0 { + return Err(UapiTranslateError::ProgramOffset); + } + let allocation = self + .base + .checked_add((tagged_offset & !7) as u64) + .ok_or(UapiTranslateError::ProgramOffset)?; + require_range(address_space, allocation, 4, GpuAccess::Read)?; + // Honeykrisp carries resource metadata in the low three UAPI bits, + // but G17P's native 3D command strips those bits before programming + // every BG/EOT/partial USC address. Preserving bit 2 launched four + // bytes into the instruction stream and contradicted all complete + // Metal register-list captures. + Ok(allocation) + } +} + +fn validate_sampler( + address_space: &A, + heap: u64, + count: u32, +) -> Result<(), UapiTranslateError> { + if count > MAX_SAMPLERS { + return Err(UapiTranslateError::SamplerCount); + } + if (heap == 0) != (count == 0) { + return Err(UapiTranslateError::SamplerPair); + } + if count != 0 { + if heap & (SAMPLER_SIZE - 1) != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + heap, + count as u64 * SAMPLER_SIZE, + GpuAccess::Read, + )?; + } + Ok(()) +} + +fn validate_attachments( + address_space: &A, + attachments: &UapiAttachmentList, +) -> Result<(), UapiTranslateError> { + for attachment in attachments.as_slice() { + require_range( + address_space, + attachment.address, + attachment.size, + GpuAccess::Write, + )?; + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderInternalState { + pub(crate) parameters: G17pRenderParameters, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedRenderCommand { + /// Exact logical render prefix assigned by the DRM ioctl dispatcher. + pub(crate) render_order: submission::G17PRenderOrder, + /// Original UAPI render index, retained through admission. + pub(crate) render_barrier: u16, + /// Original UAPI compute index, retained through admission. + pub(crate) compute_barrier: u16, + /// Exact compute prefix resolved at ioctl admission. Later independent + /// compute commands must not become parents of this render. + pub(crate) compute_wait_through: Option, + pub(crate) flags: u32, + pub(crate) vdm_base: u64, + pub(crate) parameters: G17pRenderParameters, + pub(crate) vertex_attachments: UapiAttachmentList, + pub(crate) fragment_attachments: UapiAttachmentList, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, +} + +fn validate_zls( + address_space: &A, + value: UapiZlsBuffer, + layers: u16, + g18_subpage: bool, +) -> Result<(), UapiTranslateError> { + if value.base == 0 + && (value.compression_base != 0 || value.stride != 0 || value.compression_stride != 0) + { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_base == 0 && value.compression_stride != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if !g18_subpage && layers > 1 && value.base != 0 && value.stride == 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.stride != 0 && value.stride & 0x3fff != 1 + && !(g18_subpage && value.stride & 0x3fff == 0) { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_stride & 0x3fff != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.base != 0 { + let layer_stride = if g18_subpage && layers > 1 { + let unit = if value.stride & 1 != 0 { PAGE_SIZE } else { 0x80 }; + ((value.stride as u64 >> 14) + 1) * unit + } else if value.stride == 0 { + 0 + } else { + ((value.stride as u64 >> 14) + 1) * PAGE_SIZE + }; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range(address_space, value.base, size, GpuAccess::ReadWrite)?; + } + if value.compression_base != 0 { + let layer_stride = ((value.compression_stride as u64 >> 14) + 1) * 0x80; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range( + address_space, + value.compression_base, + size, + GpuAccess::ReadWrite, + )?; + } + Ok(()) +} + +/// Bounded M5 D32 clear/store envelope, alongside the original color-only +/// profile. Native 25G76 supplies ff00ff dimensions and 80044 with compression; +/// bits2/6 select compression, while bit19 stores the depth plane. Geometry, +/// stencil absence, and application programs are checked by the M5 caller. +pub(crate) fn validate_m5_reference_depth( + command: &UapiRenderCommand, + address_space: &A, +) -> Result<(), UapiTranslateError> { + let depth = command.depth; + if depth.base == 0 { + // No depth plane. The surface may still carry an inert stencil one, + // which contributes nothing but its compression pair and the shared + // pixel dimensions; a render that transfers it is refused with the + // rest of the stencil profile. + let stencil_only = command.stencil.base != 0 + && command.zls_control + == if command.stencil.compression_base != 0 { 0x110 } else { 0 } + && command.depth_dimensions + == ((u32::from(command.width) - 1) + | ((u32::from(command.height) - 1) << 16)); + return if depth.compression_base == 0 && depth.stride == 0 + && depth.compression_stride == 0 + && (stencil_only + || (command.zls_control == 0 && command.depth_dimensions == 0)) { + Ok(()) + } else { + Err(UapiTranslateError::ZlsFields) + }; + } + let compressed = depth.compression_base != 0; + // The control word's load and store bits are the only part a render + // chooses: bit15 loads the depth plane and bit19 stores it, and a render + // that neither tests nor writes depth sets neither, which is how a + // conformance surface with an unused depth buffer arrives. Everything + // else -- the compression pair in bits 2 and 6, the GPU tiling, the 32F + // format -- is the witnessed profile and stays exact. depth_flags reaches + // register 0x15319 unchanged either way, so the admitted envelope is the + // set of load/store combinations over that fixed remainder. + let compression_bits = if compressed { 0x44 } else { 0 }; + // Bits 25-26 select the depth format: 0 is 32-bit float, 2 is 16-bit + // unorm. The narrow plane was captured on 25G76 alongside the wide one + // and moves nothing else in the register program except the clear value + // and one ISP flag, both of which the caller derives. Its compression + // metadata has no witness, so a compressed 16-bit plane stays out. + let z16 = command.zls_control & (3 << 25) == (2 << 25); + if z16 && compressed { + return Err(UapiTranslateError::ZlsFields); + } + let format_bits = if z16 { 2 << 25 } else { 0 }; + // A stencil plane may be attached without being transferred: a surface + // with a depth-stencil buffer carries one for every render, whether or + // not the render uses it. Its compression pair in bits 4 and 8 describes + // the attached layout and is the only stencil state admitted here -- + // loading or storing the plane (bits 14 and 18) needs its own witnesses, + // as does the stencil unit in the graphics record. + let stencil_compression_bits = + if command.stencil.compression_base != 0 { 0x110 } else { 0 }; + let control_transfers = command.zls_control & 0x88000; + let expected_control = + compression_bits | stencil_compression_bits | format_bits | control_transfers; + let width = u32::from(command.width); + let height = u32::from(command.height); + if !(1..=M5_MAX_RENDER_DIMENSION).contains(&width) + || !(1..=M5_MAX_RENDER_DIMENSION).contains(&height) + || command.layers != 1 || (compressed && (width < 16 || height < 16)) { + return Err(UapiTranslateError::ZlsFields); + } + // AIL tiles are 16 KiB whatever the format: 64x64 at four bytes a pixel, + // 128x64 at two. While both axes fit one tile the level is a whole + // number of them; if either axis is smaller, the level uses a + // power-of-two miptail aligned to a 128-byte cacheline. Single-layer M5 + // surfaces are not page-padded. + let (tile_width, bytes_per_pixel) = if z16 { (128u32, 2u64) } else { (64u32, 4u64) }; + let data_bytes = if width >= tile_width && height >= 64 { + u64::from(width.div_ceil(tile_width)) * u64::from(height.div_ceil(64)) * 0x4000 + } else { + (u64::from(width.next_power_of_two()) + * u64::from(height.next_power_of_two()) * bytes_per_pixel) + .div_ceil(128) * 128 + }; + let metadata_bytes = (u64::from(width.next_power_of_two().div_ceil(16)) + * u64::from(height.next_power_of_two().div_ceil(16)) * 8).div_ceil(128) * 128; + let dimensions = (width - 1) | ((height - 1) << 16); + if command.depth_dimensions != dimensions + || command.zls_control != expected_control + // The clear value is whatever the application asked for; it reaches + // register 0x15301 verbatim and is only consumed when the render + // actually clears the plane. + || (command.depth_clear & 0x7fffffff) > 0x7f800000 + // Image suballocations and small miptails use cacheline alignment; + // the backing VM pages remain independently validated below. + || depth.base & 0x7f != 0 || depth.compression_base & 0x7f != 0 + // Both strides are array strides, and the M5 caller zeroes them + // because native single-layer ZLS does not encode one. Every render + // admitted here is single-layer -- layers != 1 is refused above -- so + // the value that arrives is whatever the userspace packer computed + // and the hardware never sees it: for a surface smaller than a page + // Mesa's (pages - 1) << 14 | 1 underflows to 0xffffc001. Comparing it + // would refuse a valid render over a field that is discarded. The + // backing store is still validated in full below, from the + // dimensions rather than from either stride. + || (!compressed && depth.compression_stride != 0) { + return Err(UapiTranslateError::ZlsFields); + } + // Validate the entire resized surface and metadata before the + // M5 caller removes the unused single-layer array stride fields. + require_range(address_space, depth.base, data_bytes, GpuAccess::ReadWrite)?; + if compressed { + require_range(address_space, depth.compression_base, metadata_bytes, + GpuAccess::ReadWrite)?; + } + Ok(()) +} + +pub(crate) fn translate_render_command( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, +) -> Result { + translate_render_command_with_capacity(command, vertex_attachments, + fragment_attachments, queue, address_space, internal, 32_768, false) +} + +fn translate_render_command_with_capacity( + command: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, + internal: RenderInternalState, + tile_capacity: u64, + g18_subpage: bool, +) -> Result { + if command.flags & !RENDER_FLAGS != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + // Genuinely unimplemented on G17P rather than merely unvalidated: + // `G17pRenderParameters` carries no helper-program fields at all, so + // VERTEX_SCRATCH has nowhere to put the scratch binding, and there is no + // cluster-count control for NO_VERTEX_CLUSTERING. A clean EINVAL beats + // rendering with the request silently dropped. + if command.flags & (RENDER_VERTEX_SCRATCH | RENDER_NO_VERTEX_CLUSTERING) != 0 { + return Err(UapiTranslateError::FlagsUnsupported); + } + // RSRC_SPEC_HI is *declarative*, and honouring it must not mean rejecting. + // The UAPI defines it as "the appended resource-specifier high dwords are + // present", and states its purpose: "This flag lets old kernels report an + // unsupported render payload instead of silently truncating 64-bit G17 + // values." It exists for kernels that do NOT understand the dwords. + // + // This kernel does understand them -- `program()` ORs the high dword in + // unconditionally, with no reference to the flag -- so the flag cannot + // change what we do with a payload that sets it. The old check therefore + // refused payloads it would otherwise have handled identically, protecting + // nothing; it is the same bring-up leftover as the stencil-clear mask. + // + // Honour the documented meaning instead: when the flag is clear the high + // dwords are "not present", so mask them off rather than refuse the + // submission. A userspace built against the 240-byte upstream struct (whose + // payload `padded_payload` zero-fills) and one that clears the flag now + // behave identically, which is what "not present" has to mean. + let resource_spec_mask = if command.flags & RENDER_RSRC_SPEC_HI != 0 { + u64::MAX + } else { + u32::MAX as u64 + }; + if !(1..=16_384).contains(&command.width) || !(1..=16_384).contains(&command.height) { + return Err(UapiTranslateError::Dimensions); + } + if !(1..=2048).contains(&command.layers) { + return Err(UapiTranslateError::Layers); + } + if !matches!( + (command.utile_width, command.utile_height), + (32, 32) | (32, 16) | (16, 16) + ) { + return Err(UapiTranslateError::UtileDimensions); + } + let samples_log2 = match command.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err(UapiTranslateError::Samples), + }; + let utile_bytes = command.sample_size as u64 + * command.utile_width as u64 + * command.utile_height as u64 + * command.samples as u64; + if utile_bytes > tile_capacity { + return Err(UapiTranslateError::TilebufferSize); + } + let blocks_per_utile = utile_bytes.div_ceil(2048); + + if command.vdm_base == 0 || command.vdm_base & 3 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.vdm_base, 4, GpuAccess::Read)?; + + if command.scissor_base == 0 || command.scissor_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.scissor_base, 8, GpuAccess::Read)?; + if command.depth_bias_base != 0 { + if command.depth_bias_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range(address_space, command.depth_bias_base, 8, GpuAccess::Read)?; + } + if command.occlusion_query_base != 0 { + if command.occlusion_query_base & 7 != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.occlusion_query_base, + 8, + GpuAccess::Write, + )?; + } + validate_zls(address_space, command.depth, command.layers, g18_subpage)?; + validate_zls(address_space, command.stencil, command.layers, g18_subpage)?; + validate_sampler( + address_space, + command.sampler_heap, + command.sampler_count as u32, + )?; + if command.vertex_helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + || command.fragment_helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + // `isp_bgobjvals` is the *whole* ISP_BGOBJVALS register value, not just + // the stencil clear: the UAPI documents "the bottom 8-bits contain the + // stencil buffer clear value", and every Mesa backend unconditionally + // seeds the upper bits with 0x300 before OR-ing the clear in + // (hk_cmd_draw.c `render->cr.isp_bgobjvals = 0x300;`, + // agx_pipe.c `c->isp_bgobjvals = 0x300;`, d12_queue.c `0x300 | stencil`). + // The legacy queue path passes the field straight through to the firmware + // (queue/render.rs `let load_bgobjvals = cmdbuf.isp_bgobjvals as u64`), so + // rejecting anything above 0xff refused every real render pass while the + // in-tree render selftest -- which leaves the field zero -- was accepted. + // Accept the register value; `build_*_descriptor` re-asserts the 0x300 + // bits with `stencil_clear_value | 0x300`, which is idempotent for the + // value userspace sends and still supplies 0x300 when it sends zero. + + let background = queue.full_program(address_space, command.background.usc)?; + let end_of_tile = queue.full_program(address_space, command.end_of_tile.usc)?; + let partial_background = queue.full_program(address_space, command.partial_background.usc)?; + let partial_end_of_tile = queue.full_program(address_space, command.partial_end_of_tile.usc)?; + validate_attachments(address_space, &vertex_attachments)?; + validate_attachments(address_space, &fragment_attachments)?; + + let utile_config = ((command.utile_width as u64 / 16) << 12) + | ((command.utile_height as u64 / 16) << 14) + | samples_log2; + let mut tile_config = 0x280; + if command.layers > 1 { + tile_config |= 1; + } + if command.flags & RENDER_PROCESS_EMPTY_TILES != 0 { + tile_config |= 0x1_0000; + } + + let mut parameters = internal.parameters; + parameters.width = command.width as u32; + parameters.height = command.height as u32; + parameters.encoder = command.vdm_base; + parameters.layers = command.layers; + parameters.utile_width = command.utile_width; + parameters.utile_height = command.utile_height; + parameters.utile_config = utile_config; + parameters.multisample_control = command.multisample_control; + parameters.ppp_control = command.ppp_control as u64; + parameters.tib_blocks = blocks_per_utile; + parameters.tile_config = tile_config; + parameters.depth_dimensions = command.depth_dimensions as u64; + parameters.depth_buffer = command.depth.base; + parameters.depth_aux_buffer = command.depth.compression_base; + parameters.depth_stride = command.depth.stride as u64; + parameters.depth_aux_stride = command.depth.compression_stride as u64; + parameters.stencil_buffer = command.stencil.base; + parameters.stencil_aux_buffer = command.stencil.compression_base; + parameters.stencil_stride = command.stencil.stride as u64; + parameters.stencil_aux_stride = command.stencil.compression_stride as u64; + parameters.depth_flags = command.zls_control; + // The firmware's internal partial-render phases preserve attached Z/S + // surfaces even when the render-pass boundary is store-only. Mesa already + // computes this distinction for its partial BG/EOT programs, but the + // current 256-byte render UAPI carries only the boundary ZLS word. Derive + // the exact uncompressed enable bits here until the append-only partial + // ZLS UAPI field is plumbed; tiling/compression/format bits remain those + // packed into the original word. + parameters.partial_depth_flags = command.zls_control + | if command.depth.base != 0 { (1 << 15) | (1 << 19) } else { 0 } + | if command.stencil.base != 0 { (1 << 14) | (1 << 18) } else { 0 }; + parameters.occlusion_query_base = command.occlusion_query_base; + parameters.scissor_array = command.scissor_base; + parameters.depth_bias_array = command.depth_bias_base; + parameters.sampler_array = command.sampler_heap; + parameters.sampler_count = command.sampler_count as u32; + parameters.process_empty_tiles = command.flags & RENDER_PROCESS_EMPTY_TILES != 0; + parameters.merge_upper_x_bits = command.merge_upper_x; + parameters.merge_upper_y_bits = command.merge_upper_y; + parameters.depth_clear_value_bits = command.depth_clear; + parameters.stencil_clear_value = command.stencil_clear; + parameters.load_pipeline_bind = g17p_bg_resource_spec( + command.background.resource_spec, + resource_spec_mask, + internal.parameters.bg_resource_prefix, + ); + parameters.load_pipeline = background; + parameters.store_pipeline_bind = command.end_of_tile.resource_spec & resource_spec_mask; + parameters.store_pipeline = end_of_tile; + // These high 17 bits are BG shader pass control, not an address class. + // Native derives main and partial masks from their respective shader + // outputs; they need not match. The default diagnostic prefix is zero, + // preserving both full RT0 output masks and no-output masks from userspace. + parameters.partial_load_pipeline_bind = g17p_bg_resource_spec( + command.partial_background.resource_spec, + resource_spec_mask, + internal.parameters.bg_resource_prefix, + ); + parameters.partial_load_pipeline = partial_background; + parameters.partial_store_pipeline_bind = + command.partial_end_of_tile.resource_spec & resource_spec_mask; + parameters.partial_store_pipeline = partial_end_of_tile; + + Ok(TranslatedRenderCommand { + render_order: submission::G17PRenderOrder::default(), + render_barrier: UAPI_BARRIER_NONE, + compute_barrier: UAPI_BARRIER_NONE, + compute_wait_through: None, + flags: command.flags, + vdm_base: command.vdm_base, + parameters, + vertex_attachments, + fragment_attachments, + vertex_timestamps: command.vertex_timestamps, + fragment_timestamps: command.fragment_timestamps, + }) +} + +fn g17p_bg_resource_spec(resource_spec: u64, present_mask: u64, diagnostic_prefix: u64) -> u64 { + diagnostic_prefix | (resource_spec & present_mask) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedComputeCommand { + /// Monotonic logical index, independent of the hardware timestamp. + pub(crate) compute_sequence: u64, + /// Explicit same-engine prefix; only host-published timestamps need host waits. + pub(crate) compute_wait_through: Option, + pub(crate) render_wait_through: Option, + pub(crate) render_wait_last: Option, + pub(crate) usc_exec_base: u64, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) control_stream_terminator: u64, + pub(crate) sampler_heap: u64, + pub(crate) sampler_count: u32, + pub(crate) attachments: UapiAttachmentList, + pub(crate) g17p_add3_buffers: Option<[u64; 3]>, + pub(crate) timestamps: UapiTimestamps, + pub(crate) scs_layout: u64, + /// G18 only: the command carried `DRM_ASAHI_COMPUTE_USES_TEXTURE_STATE`. + /// [`translate_compute_command`] refuses that bit like any other unknown + /// one and leaves this false; the G18 adapter admits the bit, strips it + /// before translation and sets this afterwards. + pub(crate) g18_texture_state: bool, + /// Validated native descriptor inputs; resource/launch integration is + /// separate, so this field does not by itself authorize scratch use. + pub(crate) g18_resource_descriptor: Option<[u32; 23]>, +} + +pub(crate) fn translate_compute_command( + command: UapiComputeCommand, + attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, +) -> Result { + queue.validate()?; + if command.flags & !COMPUTE_G17P_ADD3_PROOF != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + if command.g18_resources != [0; 16] { + return Err(UapiTranslateError::FlagsUnsupported); + } + if command.control_stream_base & 3 != 0 + || command.control_stream_end & 3 != 0 + || command.control_stream_end <= command.control_stream_base + { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.control_stream_base, + command.control_stream_end - command.control_stream_base, + GpuAccess::Read, + )?; + validate_sampler(address_space, command.sampler_heap, command.sampler_count)?; + if command.helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + let g17p_add3_buffers = if command.flags & COMPUTE_G17P_ADD3_PROOF != 0 { + let entries = attachments.as_slice(); + if entries.len() != 3 + || entries + .iter() + .any(|entry| entry.size < G17P_ADD3_BUFFER_BYTES) + { + return Err(UapiTranslateError::Add3ProofAttachments); + } + require_range( + address_space, + entries[0].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Read, + )?; + require_range( + address_space, + entries[1].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Read, + )?; + require_range( + address_space, + entries[2].address, + G17P_ADD3_BUFFER_BYTES, + GpuAccess::Write, + )?; + Some([entries[0].address, entries[1].address, entries[2].address]) + } else { + validate_attachments(address_space, &attachments)?; + None + }; + Ok(TranslatedComputeCommand { + compute_sequence: 0, + compute_wait_through: None, + render_wait_through: None, + render_wait_last: None, + usc_exec_base: queue.base, + control_stream_base: command.control_stream_base, + control_stream_end: command.control_stream_end, + control_stream_terminator: command.control_stream_end - 4, + sampler_heap: command.sampler_heap, + sampler_count: command.sampler_count, + attachments, + g17p_add3_buffers, + timestamps: command.timestamps, + scs_layout: command.scs_layout, + g18_texture_state: false, + g18_resource_descriptor: None, + }) +} + +impl TranslatedComputeCommand { + +} diff --git a/drivers/gpu/drm/asahi/m5/completion.rs b/drivers/gpu/drm/asahi/m5/completion.rs new file mode 100644 index 0000000000000..b6b1e748df482 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/completion.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Completion ownership for the bounded M5 TA/fragment pair. +//! +//! 25G76 GFX 0x17544 compares scheduler.completed with *scheduler.submitted; +//! 0x1763c unlinks the completed scheduler. The submitted total is host-owned, +//! not a firmware-decremented pending count. 0x17a68/0x17ae0 then write the +//! first stamp pointer and 0x17b98 publishes the corresponding event mask. + +// The 32-bit completion value reserves its low byte. Stop before it wraps; +// storage-ring wrap is independent of this monotonically increasing stamp. +pub(crate) const MAX_GRAPHICS_GENERATION: u32 = 0x00ff_ffff; +pub(crate) const GRAPHICS_POINTER_COUNT: u32 = 0x500; + +pub(crate) const fn graphics_pointer_head(generation: u32) -> u32 { + (1 + generation * 3) % GRAPHICS_POINTER_COUNT +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct M5GraphicsCompletion { + pub(crate) submitted: u32, + pub(crate) completed: u32, + pub(crate) linked: u32, + pub(crate) list: [u64; 2], + pub(crate) stamps: [u32; 2], + pub(crate) signalled: u128, +} + +/// The same native scheduler retirement contract for one compute record. +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct M5ComputeCompletion { + pub(crate) submitted: u32, + pub(crate) completed: u32, + pub(crate) linked: u32, + pub(crate) list: [u64; 2], + pub(crate) stamp: u32, + pub(crate) signalled: u128, +} + +impl M5ComputeCompletion { + pub(crate) fn complete_at(self, list_address: u64, queue_id: u8, stamp: u32) -> bool { + list_address != 0 && queue_id < 128 && stamp != 0 && stamp & 0xff == 0 + && self.submitted == 1 && self.completed == 1 && self.linked == 0 + && self.list == [0, list_address] && self.stamp == stamp + && self.signalled & (1u128 << queue_id) != 0 + } +} + +impl M5GraphicsCompletion { + + pub(crate) fn pair_complete_at(self, list_address: u64, stamp: u32) -> bool { + list_address != 0 + && stamp != 0 && stamp & 0xff == 0 + && self.submitted == 2 + && self.completed == self.submitted + && self.linked == 0 + && self.list == [0, list_address] + && self.stamps == [stamp; 2] + && self.signalled & 0xc00 == 0xc00 + } +} + diff --git a/drivers/gpu/drm/asahi/m5/compute.rs b/drivers/gpu/drm/asahi/m5/compute.rs new file mode 100644 index 0000000000000..71e1384b070bc --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/compute.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8142/G17G C0 J813 compute register program for the 25G76 native profile. +//! +//! The native SKSM payload has selector 3 and its register array begins at +//! +0x40. The 40-write program below is separately captured on M5; it is not +//! the G17P program with a new chip ID. Queue registration, descriptor-owned +//! pointers, shader/CDM construction and completion still belong to the M5 +//! runtime. Encoding these registers does not admit a Linux GPU job. + +pub(crate) const M5_COMPUTE_REGISTER_COUNT: usize = 40; +pub(crate) const M5_COMPUTE_REGISTER_BYTES: usize = 12 * M5_COMPUTE_REGISTER_COUNT; +/// Measured descriptor prefix, not a claim about the host allocation size. +pub(crate) const M5_COMPUTE_DESCRIPTOR_BYTES: usize = 0x1000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5ComputeError { + AddressOverflow, + BufferTooSmall, + CdmTerminatorBeforeStream, +} + +/// Retained objects and per-command state for the measured simple CL path. +/// Samplers, user timestamps, indirect dispatch and extra spill modes are not +/// part of this profile. Pointer ownership belongs to the runtime caller. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ComputeDescriptor { + pub(crate) registers: M5ComputeParameters, + pub(crate) submit_sequence: u64, + /// Host descriptor +0x188, copied to native command +0x10. + pub(crate) scheduler_record: u64, + pub(crate) descriptor_low_alias: u64, + pub(crate) cdm_terminator: u64, + pub(crate) dispatch_stamps: [u64; 2], + pub(crate) event_machine_stamp: u32, + pub(crate) queue_id: u32, + pub(crate) queue_ordinal: u32, + pub(crate) has_event: bool, + pub(crate) submission_index: u32, + pub(crate) status: [u64; 2], + pub(crate) shared_control: u64, + pub(crate) shared_control_enabled: bool, + /// Sixteen bytes copied by the native host from the event machine's + /// +0x70 record. Their bit meanings are not all qualified; keep opaque. + pub(crate) event_metadata: [u8; 16], + pub(crate) zero_page: u64, + /// Host descriptor +0x148; not a fixed target constant. + pub(crate) owner_id: u8, + /// Full host descriptor +0x140. Register context packing uses only its + /// low byte; the descriptor copy at +0xf70 must retain all 32 bits. + pub(crate) descriptor_work_ordinal: u32, +} + +/// Encode all 4096 bytes of the native simple-compute descriptor prefix. +/// The register arrays and their aliases are generated from owned inputs. +/// This function performs no allocation, cache operation or publication. +pub(crate) fn encode_m5_compute_descriptor( + p: M5ComputeDescriptor, + raw: &mut [u8], +) -> Result<(), M5ComputeError> { + if raw.len() < M5_COMPUTE_DESCRIPTOR_BYTES { + return Err(M5ComputeError::BufferTooSmall); + } + if p.cdm_terminator < p.registers.cdm_base { + return Err(M5ComputeError::CdmTerminatorBeforeStream); + } + let registers = build_m5_compute_registers(p.registers)?; + let primary_alias = add(p.descriptor_low_alias, 0x40)?; + let secondary_alias = add(p.descriptor_low_alias, 0x760)?; + let secondary = [ + (0x10099u32, add(p.registers.operand_state_base, 5)?), + (0x10091, add(p.registers.operand_state_base, 0xf000)?), + (0x0a5c1, add(p.registers.operand_state_base, 0xf405)?), + (0x0a5c9, add(p.registers.operand_state_base, 0x1e400)?), + ]; + let raw = &mut raw[..M5_COMPUTE_DESCRIPTOR_BYTES]; + raw.fill(0); + for (offset, value) in [ + (0x00, 3u32), (0x0c, u32::from(p.registers.context_id)), + (0x748, 0x01e0_0028), (0xe68, 0x0030_0004), + (0xf20, (p.registers.dispatch_identity >> 32) as u32), + (0xf28, u32::MAX), (0xf50, p.event_machine_stamp), + (0xf54, p.queue_id), (0xf58, p.queue_ordinal), + (0xf5c, u32::from(p.has_event)), (0xf60, p.submission_index), + (0xf70, p.descriptor_work_ordinal), + ] { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [ + (4, p.submit_sequence), (0x10, p.scheduler_record), + (0x740, primary_alias), (0xe60, secondary_alias), + (0xed8, p.registers.preempt_base), (0xee0, p.cdm_terminator), + (0xf08, 0x0000_0001_5402_4201), + (0xf40, p.dispatch_stamps[0]), (0xf48, p.dispatch_stamps[1]), + (0xf68, p.registers.dispatch_identity & 0xffff_ffff), + (0xf7c, p.status[0]), (0xf84, p.status[1]), + (0xfb2, p.shared_control), (0xfcb, p.zero_page), + ] { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + for (index, count) in [37u16, 39, 39, 40].into_iter().enumerate() { + raw[0x18 + index * 2..0x1a + index * 2].copy_from_slice(&count.to_le_bytes()); + } + for (index, register) in registers.into_iter().enumerate() { + let offset = 0x40 + index * 12; + raw[offset..offset + 4].copy_from_slice(®ister.number.to_le_bytes()); + raw[offset + 4..offset + 12].copy_from_slice(®ister.value.to_le_bytes()); + } + for (index, (number, value)) in secondary.into_iter().enumerate() { + let offset = 0x760 + index * 12; + raw[offset..offset + 4].copy_from_slice(&number.to_le_bytes()); + raw[offset + 4..offset + 12].copy_from_slice(&value.to_le_bytes()); + } + raw[0xfba] = u8::from(p.shared_control_enabled); + raw[0xfbb..0xfcb].copy_from_slice(&p.event_metadata); + raw[0xfd3] = p.owner_id; + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ComputeRegister { + pub(crate) number: u32, + pub(crate) value: u64, +} + +impl M5ComputeRegister { + const fn new(number: u32, value: u64) -> Self { + Self { number, value } + } +} + +/// Command-owned addresses and identities; the runtime must retain their +/// backing and validate its mappings before publishing the complete command. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ComputeParameters { + pub(crate) preempt_base: u64, + pub(crate) cdm_base: u64, + pub(crate) dispatch_identity: u64, + pub(crate) context_id: u8, + pub(crate) work_ordinal: u8, + pub(crate) robustness: u64, + pub(crate) operand_state_base: u64, + /// Owned UMA pool slot, copied from native host descriptor +0x788 by + /// 25G76 generateRegisterList at 0x89e38dc..0x89e391c. This must match + /// shared-control +8 and the populated hardware UMA descriptor slot. + pub(crate) private_pool_hw_id: u64, + /// Native command descriptor +0x530, copied to register 0xd4a1. + pub(crate) descriptor_control_530: u64, + /// Descriptor +0x538; zero selects the native 0x60 default. The native + /// encoder shifts the selected value left 32 bits, discarding high bits. + pub(crate) descriptor_control_538: u64, + /// Native descriptor +0x60a/+0x60b bit 0. Their broader semantics are + /// still under investigation; neither is a fixed M5 device property. + pub(crate) descriptor_flag_60a: bool, + pub(crate) descriptor_flag_60b: bool, +} + +fn add(base: u64, offset: u64) -> Result { + base.checked_add(offset) + .ok_or(M5ComputeError::AddressOverflow) +} + +/// Encode the measured J813 profile, including its M5 control words and +/// enlarged operand-state offsets. Other G17G configurations are not implied. +pub(crate) fn build_m5_compute_registers( + p: M5ComputeParameters, +) -> Result<[M5ComputeRegister; M5_COMPUTE_REGISTER_COUNT], M5ComputeError> { + let context = ((p.context_id as u64) << 8) | p.work_ordinal as u64; + // 25G76 generateRegisterList +0x5d4..0x670. The final 0xa599 write + // restores 0x60 irrespective of the command-specific initial value. + let control_538 = if p.descriptor_control_538 == 0 { + 0x60 + } else { + p.descriptor_control_538 + }; + // 25G76 +0x644..0x66c; additional device-dependent bits used by other + // profiles are outside this measured J813 configuration. + let mode = if p.descriptor_flag_60a { + 0x14c0_8a60 + } else { + 0x10c0_8860 + } | if p.descriptor_flag_60b { 0x80 } else { 0 }; + Ok([ + M5ComputeRegister::new(0x1a510, p.preempt_base), + M5ComputeRegister::new(0x1a420, p.cdm_base), + M5ComputeRegister::new(0x1a4d0, add(p.preempt_base, 0x1480)?), + M5ComputeRegister::new(0x1a4d8, add(p.preempt_base, 0x1488)?), + M5ComputeRegister::new(0x1a4e0, add(p.preempt_base, 0x1490)?), + M5ComputeRegister::new(0x1a4e8, add(p.preempt_base, 0x1498)?), + M5ComputeRegister::new(0x1a440, 0x0000_0001_5402_4201), + M5ComputeRegister::new(0x0d4a1, p.descriptor_control_530), + M5ComputeRegister::new(0x0d411, 0x0000_0002_0000_0001), + M5ComputeRegister::new(0x0a599, (control_538 << 32) | 0x0040_0020), + M5ComputeRegister::new(0x1a458, mode), + M5ComputeRegister::new(0x101d9, 0x1c), + M5ComputeRegister::new(0x1a089, 0), + M5ComputeRegister::new(0x1a091, 0), + M5ComputeRegister::new(0x1a059, 0), + M5ComputeRegister::new(0x1a061, 0), + M5ComputeRegister::new(0x1a0b9, 0), + M5ComputeRegister::new(0x1a0c1, 0), + M5ComputeRegister::new(0x101d1, 0), + M5ComputeRegister::new(0x0d479, 0), + M5ComputeRegister::new(0x1a0e9, 8), + M5ComputeRegister::new(0x1a141, 0), + M5ComputeRegister::new(0x107a1, 0xff_0000), + M5ComputeRegister::new(0x1a540, p.dispatch_identity), + M5ComputeRegister::new(0x014a9, p.dispatch_identity), + M5ComputeRegister::new(0x0a351, p.dispatch_identity), + M5ComputeRegister::new(0x0a731, 0), + M5ComputeRegister::new(0x10201, context), + M5ComputeRegister::new(0x10428, context), + M5ComputeRegister::new(0x14028, p.private_pool_hw_id), + M5ComputeRegister::new(0x14070, p.robustness | 1), + M5ComputeRegister::new(0x10229, add(p.operand_state_base, 0x1e800)?), + M5ComputeRegister::new(0x140a8, add(p.operand_state_base, 0x1f000)?), + M5ComputeRegister::new(0x10099, add(p.operand_state_base, 0x0f405)?), + M5ComputeRegister::new(0x10091, add(p.operand_state_base, 0x1e400)?), + M5ComputeRegister::new(0x0a5c1, add(p.operand_state_base, 0x00005)?), + M5ComputeRegister::new(0x0a5c9, add(p.operand_state_base, 0x0f000)?), + M5ComputeRegister::new(0x0a731, 0), + M5ComputeRegister::new(0x1a440, 0x0000_0001_5402_4209), + M5ComputeRegister::new(0x0a599, 0x0000_0060_0040_0020), + ]) +} + +/// Write only the register array. No descriptor header or publication state +/// is manufactured here, and an invalid request leaves the destination alone. +pub(crate) fn encode_m5_compute_registers( + p: M5ComputeParameters, + bytes: &mut [u8], +) -> Result<(), M5ComputeError> { + if bytes.len() < M5_COMPUTE_REGISTER_BYTES { + return Err(M5ComputeError::BufferTooSmall); + } + let registers = build_m5_compute_registers(p)?; + for (index, register) in registers.iter().enumerate() { + let offset = index * 12; + bytes[offset..offset + 4].copy_from_slice(®ister.number.to_le_bytes()); + bytes[offset + 4..offset + 12].copy_from_slice(®ister.value.to_le_bytes()); + } + Ok(()) +} + diff --git a/drivers/gpu/drm/asahi/m5/drm.rs b/drivers/gpu/drm/asahi/m5/drm.rs new file mode 100644 index 0000000000000..a2a7b9d2679ac --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/drm.rs @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Experimental M5 DRM adapter using the retained M5 firmware session. +//! First-job admission is deliberately separate from general Mesa support. + +use core::{any::Any, ops::Range}; +use kernel::{bindings, c_str, device, macros::vtable, new_mutex, prelude::*, static_lock_class, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + sync::{Arc, Mutex}, uapi, xarray}; +use crate::{alloc, driver::AsahiRegData, file, m5::live_boot::M5Runtime, + m5::resources::M5FirmwareMemory, m5::command, gem, gpu, hw, m5::vm_layout, mmu, queue}; + +pub(crate) type SharedM5Runtime = Arc>; +const USER_END: u64 = (1 << 42) - 2 * mmu::UAT_PGSZ as u64; + +/// Bytes of CDM control stream the witnessed M5 compute profile covers. The +/// captured stream (compute_program::STREAM) is twelve words: one dispatch +/// and the terminator at +44. Nothing longer has native evidence, and a +/// longer one is what a geometry-shader prepass submits. +const M5_COMPUTE_STREAM_BYTES: u64 = 48; +static QUEUE_NAME: &CStr = c_str!("m5_queue"); +static QUEUE_CLASS_KEY: Pin<&'static kernel::sync::LockClassKey> = static_lock_class!(); + +#[cfg(CONFIG_APPLE_CLPC)] +extern "C" { + fn apple_clpc_gpu_get(dev: *mut core::ffi::c_void, out: *mut *mut core::ffi::c_void) + -> core::ffi::c_int; + fn apple_clpc_gpu_set_active(gpu: *mut core::ffi::c_void, active: bool); + fn apple_clpc_gpu_put(gpu: *mut core::ffi::c_void); +} + +/// The CLPC's GPU input (`apple,clpc` on the GPU node). While the PMP runs, +/// its vote leaves the DRAM controller and the fabric at their lowest states; +/// the CLPC holds them at the top of its maps while M5 work is in flight, +/// and releases them its hold time after the last of it retires. Without a +/// CLPC the GPU runs as before, only slower under the PMP. +#[pin_data(PinnedDrop)] +pub(crate) struct M5ClpcInput { + gpu: *mut core::ffi::c_void, + /// Submissions in flight; the input is active while it is not zero. + #[pin] + in_flight: Mutex, +} + +// SAFETY: the CLPC serializes its input under its own lock, and the counter +// is only changed under `in_flight`. +unsafe impl Send for M5ClpcInput {} +// SAFETY: as above; shared use never changes the reference itself. +unsafe impl Sync for M5ClpcInput {} + +impl M5ClpcInput { + /// Takes the input before the firmware session starts, so that a CLPC + /// which has not probed yet defers the GPU without stranding a session. + pub(crate) fn new(dev: &device::Device) -> Result> { + #[allow(unused_mut)] + let mut gpu: *mut core::ffi::c_void = core::ptr::null_mut(); + #[cfg(CONFIG_APPLE_CLPC)] + { + // SAFETY: a live device and a writable result; C returns a + // reference, or NULL when the node names no CLPC. + let ret = unsafe { apple_clpc_gpu_get(dev.as_raw().cast(), &mut gpu) }; + if ret == EPROBE_DEFER.to_errno() { + return Err(EPROBE_DEFER); + } + if ret < 0 { + dev_warn!(dev, "no CLPC input ({}): GPU work gets no memory floor\n", ret); + gpu = core::ptr::null_mut(); + } + } + #[cfg(not(CONFIG_APPLE_CLPC))] + let _ = dev; + Arc::pin_init(pin_init!(Self { gpu, in_flight <- new_mutex!(0) }), GFP_KERNEL) + } + + fn set_active(&self, _active: bool) { + #[cfg(CONFIG_APPLE_CLPC)] + // SAFETY: NULL is accepted; a non-NULL reference is held until drop. + unsafe { apple_clpc_gpu_set_active(self.gpu, _active) }; + } + + /// Marks one submission in flight until the returned guard drops. + fn busy(&self) -> M5ClpcBusy<'_> { + let mut count = self.in_flight.lock(); + *count += 1; + if *count == 1 { + self.set_active(true); + } + M5ClpcBusy(self) + } +} + +#[pinned_drop] +impl PinnedDrop for M5ClpcInput { + fn drop(self: Pin<&mut Self>) { + #[cfg(CONFIG_APPLE_CLPC)] + // SAFETY: releases exactly the reference taken in new(), clearing the + // input first; NULL is accepted. + unsafe { apple_clpc_gpu_put(self.gpu) }; + } +} + +struct M5ClpcBusy<'a>(&'a M5ClpcInput); + +impl Drop for M5ClpcBusy<'_> { + fn drop(&mut self) { + let mut count = self.0.in_flight.lock(); + *count -= 1; + if *count == 0 { + self.0.set_active(false); + } + } +} + +pub(crate) struct M5DrmGpu { + // A strong reference prevents open DRM files from outliving the session. + runtime: SharedM5Runtime, + id: crate::regs::G17Id, + ids: gpu::SequenceIDs, + clpc: Arc, +} + +impl M5DrmGpu { + pub(crate) fn new(runtime: SharedM5Runtime, id: crate::regs::G17Id, + clpc: Arc) -> Self { + Self { runtime, id, ids: Default::default(), clpc } + } +} + +impl gpu::Gpu for M5DrmGpu { + fn stats_snapshot(&self) -> Option> { None } + fn as_any(&self) -> &dyn Any { self } + fn arc_as_any(self: Arc) -> Arc { self as Arc } + fn manager(&self) -> Option<&dyn gpu::GpuManager> { None } + fn init(&self) -> Result { + if self.is_crashed() { Err(ENODEV) } else { Ok(()) } + } + fn ids(&self) -> &gpu::SequenceIDs { &self.ids } + fn is_crashed(&self) -> bool { self.runtime.lock().is_crashed() } + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.gpu_generation = hw::GpuGen::G17 as u32; + params.gpu_variant = hw::GpuVariant::G as u32; + params.gpu_revision = self.id.gpu_rev as u32; + params.chip_id = 0x8142; + params.num_dies = self.id.num_dies; + params.num_clusters_total = self.id.num_clusters; + params.num_cores_per_cluster = self.id.num_cores; + for (i, mask) in self.id.core_masks.iter().enumerate() { + *params.core_masks.get_mut(i).ok_or(EIO)? = (*mask).into(); + } + // Dynamic M5 clock policy is not implemented. + params.max_frequency_khz = 0; + params.usc_generation = 3; + params.gpu_hal_generation = 300; + // One hardware command per submission. + params.max_commands_per_submission = 1; + Ok(()) + } + fn supports_vm_status(&self) -> bool { true } + /// VM_BIND defers unmaps while accepted work runs, and a submission installs its fence + /// in every output sync, also on failure. + fn tracked_submission(&self) -> bool { true } + fn user_range(&self) -> Range { mmu::UAT_PGSZ as u64..USER_END } + fn base_clock_hz(&self) -> u32 { 24_000_000 } + fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } + fn new_vm(&self, kernel_range: Range) -> Result { + self.runtime.lock().new_user_vm(self.ids.vm.next(), kernel_range) + } + fn new_queue(&self, vm: mmu::Vm, + _ualloc: Arc>, + _ualloc_priv: Arc>, + priority: u32, usc_exec_base: u64) -> Result> { + if priority > 3 { return Err(EINVAL); } + let window = command::QueueUscWindow { + base: usc_exec_base, user_start: vm_layout::DRIVER_BASE, user_end: USER_END, + }; + window.validate().map_err(|_| EINVAL)?; + let queue = M5Queue { + runtime: self.runtime.clone(), vm, window, + fences: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, + clpc: self.clpc.clone(), + }; + Ok(KBox::new(queue, GFP_KERNEL)?) + } + fn map_timestamp_buffer(&self, _bo: gem::ObjectRef, _range: Range) + -> Result { Err(ENOTSUPP) } +} + +struct M5SubmissionFence; +#[vtable] +impl FenceOps for M5SubmissionFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("m5-submit") } +} + +/// Keep the original sync vector: creating a second output list would add an +/// allocation failure after userspace has already supplied its wait targets. +struct OutputPublication { + fence: UserFence, + syncs: KVec, + input_count: usize, + result: Result, +} + +impl Drop for OutputPublication { + fn drop(&mut self) { + if let Err(error) = self.result { self.fence.set_error(error); } + let fence = Fence::from_fence(&self.fence); + for mut sync in self.syncs.drain(self.input_count..) { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj.add_point(chain, &fence, sync.timeline_value); + } else { sync.syncobj.replace_fence(Some(&fence)); } + } + self.fence.signal(); + } +} + +struct UserAddressSpace<'a> { vm: &'a mmu::Vm } +impl command::GpuAddressSpace for UserAddressSpace<'_> { + fn covers(&self, address: u64, size: u64, access: command::GpuAccess) -> bool { + let (read, write) = match access { + command::GpuAccess::Read => (true, false), + command::GpuAccess::Write => (false, true), + command::GpuAccess::ReadWrite => (true, true), + }; + vm_layout::user_range_contains(&self.vm.kernel_range(), address, size, USER_END) + && self.vm.covers_range(address, size, read, write) + } +} + +struct M5Queue { + runtime: SharedM5Runtime, + vm: mmu::Vm, + window: command::QueueUscWindow, + fences: FenceContexts, + clpc: Arc, +} + +impl queue::Queue for M5Queue { + fn submit(&mut self, _reg_data: &AsahiRegData<'_>, _submission_id: u64, + syncs: KVec, + in_sync_count: usize, cmdbuf_raw: &[u8], + _objects: Pin<&xarray::XArray>>) -> Result { + if in_sync_count > syncs.len() { return Err(EINVAL); } + let fence = self.fences.new_fence(0, M5SubmissionFence)?.into(); + let mut publication = OutputPublication { + fence, syncs, input_count: in_sync_count, result: Err(ECANCELED), + }; + let result = (|| { + if self.vm.status().is_some_and(|status| status.get() != 0) { return Err(EIO); } + // Wait outside the runtime lock; another client may own the job + // whose fence satisfies this submission's dependency. + for sync in &publication.syncs[..in_sync_count] { + if let Some(fence) = &sync.fence { + // SAFETY: both calls borrow the strongly held input fence. + let waited = unsafe { bindings::dma_fence_wait_timeout( + fence.raw(), true, kernel::task::MAX_SCHEDULE_TIMEOUT) }; + if waited < 0 { return Err(Error::from_errno(waited as i32)); } + if waited == 0 { return Err(ETIMEDOUT); } + let status = unsafe { bindings::dma_fence_get_status(fence.raw()) }; + if status < 0 { return Err(Error::from_errno(status)); } + } + } + let _vm_job = self.vm.retain_job()?; + let address_space = UserAddressSpace { vm: &self.vm }; + let mut parser = command::UapiCommandParser::new(cmdbuf_raw); + let command = parser.next_hardware().map_err(|_| EINVAL)?.ok_or(EINVAL)?; + if parser.next_hardware().map_err(|_| EINVAL)?.is_some() { + pr_err!("M5 submit admission rejected: more than one hardware command\n"); + return Err(ENOTSUPP); + } + parser.finish().map_err(|_| EINVAL)?; + if let command::ParsedHardwareCommand::Compute { payload, attachments, .. } = command { + let timestamps = [payload.timestamps.start, payload.timestamps.end]; + if payload.flags != 0 || payload.sampler_heap != 0 || payload.sampler_count != 0 + || timestamps.iter().any(|t| t.handle != 0 || t.offset != 0) { + pr_err!("M5 compute admission rejected flags={:#x} sampler_heap={:#x} samplers={} timestamps={}\n", + payload.flags, payload.sampler_heap, payload.sampler_count, + timestamps.iter().any(|t| t.handle != 0 || t.offset != 0)); + return Err(ENOTSUPP); + } + // The witnessed M5 compute profile is one dispatch: the + // captured control stream is twelve words, a single dispatch + // followed by the terminator at +44, and the descriptor + // carries exactly one dispatch identity, one submission index + // and one pair of dispatch stamps. A longer stream -- every + // geometry-shader and transform-feedback prepass emits four or + // five dispatches -- is accepted by firmware, linked into the + // job list and never retired, so the submitting ioctl times + // out after forty seconds and quarantines the VM. Refuse it + // here instead: the client drops the batch and the session + // survives, exactly as it does for an out-of-profile render. + let stream_bytes = payload.control_stream_end + .saturating_sub(payload.control_stream_base); + if stream_bytes > M5_COMPUTE_STREAM_BYTES { + pr_err!("M5 compute admission rejected stream_bytes={} (profile admits one dispatch, {})\n", + stream_bytes, M5_COMPUTE_STREAM_BYTES); + return Err(ENOTSUPP); + } + let translated = command::translate_compute_command(payload, attachments, + self.window, &address_space).map_err(|_| EINVAL)?; + let _busy = self.clpc.busy(); + return self.runtime.lock().submit_application_compute(&self.vm, &translated); + } + let command::ParsedHardwareCommand::Render { + payload, vertex_attachments, fragment_attachments, .. + } = command else { + pr_err!("M5 submit admission rejected: neither a render nor a compute command\n"); + return Err(ENOTSUPP); + }; + admit_reference_render(&payload).map_err(|error| { + pr_err!("M5 render admission rejected size={}x{} layers={} utiles={}x{} samples={}\n", + payload.width, payload.height, payload.layers, + payload.utile_width, payload.utile_height, payload.samples); + error + })?; + // An attached stencil plane is admitted only while it is inert: + // its buffers are range-checked with the depth ones and its + // registers are written from the same parameter block, but a + // render that loads or stores it, or that runs the stencil unit, + // is outside the witnessed profile. + if payload.zls_control & 0x44000 != 0 { + pr_err!("M5 render admission rejected stencil transfer control={:#x}\n", + payload.zls_control); + return Err(ENOTSUPP); + } + command::validate_m5_reference_depth(&payload, &address_space) + .map_err(|error| { + pr_err!("M5 depth admission rejected {:?}: size={}x{} dimensions={:#x} control={:#x} stride={:#x} meta_stride={:#x} base={:#x} meta={:#x} clear={:#x}\n", + error, payload.width, payload.height, payload.depth_dimensions, + payload.zls_control, payload.depth.stride, payload.depth.compression_stride, + payload.depth.base, payload.depth.compression_base, payload.depth_clear); + ENOTSUPP + })?; + let (parameters, mut controls) = M5FirmwareMemory::reference_render_profile(); + let mut translated = command::translate_render_command(payload, + vertex_attachments, fragment_attachments, self.window, &address_space, + command::RenderInternalState { parameters }).map_err(|_| EINVAL)?; + // Native single-layer M5 ZLS does not encode an array stride. + // The full data/metadata ranges were validated above. + translated.parameters.depth_stride = 0; + translated.parameters.depth_aux_stride = 0; + // Native 25G76 indexed draws use 15021=0x10c000. The reference + // triangle's low bit causes fragment recovery with larger + // indexed overdraw, even after the tiling stage completes. + // Bit18 selects the narrow depth plane: the same controlled + // render against a Depth16Unorm attachment publishes 0x14c001 + // where the 32-bit one publishes 0x10c001, and that bit is the + // only difference in the whole register program beyond the ZLS + // format field and the integer clear value. + let z16 = payload.zls_control & (3 << 25) == (2 << 25); + translated.parameters.aux_fb_flags = + if z16 { 0x14c000 } else { 0x10c000 }; + // M5's second store-program pair refers to the same EOT program + // in the admitted native profile, with caller-owned program VAs. + controls.store_pipeline_2 = translated.parameters.store_pipeline; + controls.store_pipeline_2_bind = translated.parameters.store_pipeline_bind; + controls.partial_store_pipeline_2 = translated.parameters.partial_store_pipeline; + controls.partial_store_pipeline_2_bind = translated.parameters.partial_store_pipeline_bind; + let _busy = self.clpc.busy(); + self.runtime.lock().submit_application(&self.vm, translated.parameters, + controls, 1, [0x7d4000, 0x934000])?; + Ok(()) + })(); + // The runtime records and quarantines a failed published VM before + // this output fence is installed/signalled by the publication guard. + publication.result = result; + result + } +} + +/// Until the M5 controls are derived for other render modes, admit only the +/// reference render-state envelope. This is an experimental first-job ABI +/// path, not an assertion that arbitrary Vulkan programs are supported. +fn admit_reference_render(command: &command::UapiRenderCommand) -> Result { + let timestamps = [command.vertex_timestamps.start, command.vertex_timestamps.end, + command.fragment_timestamps.start, command.fragment_timestamps.end]; + if !(1..=command::M5_MAX_RENDER_DIMENSION).contains(&u32::from(command.width)) + || !(1..=command::M5_MAX_RENDER_DIMENSION).contains(&u32::from(command.height)) + || command.layers != 1 + || command.utile_width != 32 || command.utile_height != 32 || command.samples != 1 + || command.sampler_heap != 0 || command.sampler_count != 0 + || command.occlusion_query_base != 0 + || timestamps.iter().any(|timestamp| timestamp.handle != 0 || timestamp.offset != 0) { + return Err(ENOTSUPP); + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/m5/fw_msg.rs b/drivers/gpu/drm/asahi/m5/fw_msg.rs new file mode 100644 index 0000000000000..368ed4cc36f34 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/fw_msg.rs @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +const INITDATA_TAG: u64 = 0x0081_0000_0000_0000; + +const INITDATA_ADDRESS_MASK: u64 = 0x0000_0fff_ffff_ffff; + +const INITDATA_ADDRESS_PREFIX: u64 = 0xffff_f000_0000_0000; + +/// `AGXArmFirmware::powerTransitionEnd(AGFIFirmwareRole, bool)` closes the GPU +/// power-on transition by sending this note to EACH firmware role. +/// +/// KEXT_PROVEN, 25G83 @0xfffffe00088936a4. Both send sites are identical: +/// +/// ldr x0, [x19, #6616] / [x19, #6672] ; the two role objects +/// ldr x8, [obj + 0x8a8] ; sendAsyncNoteToFirmware +/// mov x1, #0x89000000000000 ; tag 0x89, payload 0 +/// mov w2, #0x1 ; gated=true +/// blraa x8, x16 +/// +/// MEASURED on hardware (m1n1 hv capture 20260904-001852Z): it is the LAST +/// EP-0x21 message of bring-up, sent once per role right after the initdata +/// handshake completes and 11.4 seconds before the first work doorbell: +/// +/// 57.95ms role0 -> ep=0x21 0x0089000000000000 +/// 59.31ms role1 -> ep=0x21 0x0089000000000000 +/// ... +/// 11425.12ms role0 -> ep=0x21 0x0083000000000000 <- first doorbell +/// +/// This driver has never sent it. Its EP-0x21 vocabulary was 0x81 initdata, +/// 0x83 doorbell, 0x84 device control and 0x86 recovery inspection only. +const POWER_TRANSITION_END_TAG: u64 = 0x0089_0000_0000_0000; + +/// Encode the role's power-transition-end note. The payload is zero in every +/// captured instance and the kext materialises the bare tag with a single +/// `movz x1, #0x89, lsl #48`. +pub(crate) const fn encode_power_transition_end() -> u64 { + POWER_TRANSITION_END_TAG +} + +/// Refusals made by the evidence-backed codec. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum WireError { + AddressOutOfRange, +} + +/// Encode the G17 init-data handoff exactly as the recovered host does. +/// +/// The host instruction truncates to 44 bits. Linux instead rejects an +/// address unless it is already in low-44-bit or firmware-canonical form. +pub(crate) const fn encode_initdata_doorbell(firmware_va: u64) -> Result { + let high = firmware_va & !INITDATA_ADDRESS_MASK; + if high != 0 && high != INITDATA_ADDRESS_PREFIX { + return Err(WireError::AddressOutOfRange); + } + + Ok(INITDATA_TAG | (firmware_va & INITDATA_ADDRESS_MASK)) +} diff --git a/drivers/gpu/drm/asahi/m5/initdata.rs b/drivers/gpu/drm/asahi/m5/initdata.rs new file mode 100644 index 0000000000000..f2ad6afea017f --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/initdata.rs @@ -0,0 +1,1630 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +use core::result::Result; + +/// Root size for the primary (`gfx-asc`) instance. +pub(crate) const ROOT_SIZE_PRIMARY: usize = 0x0b8; + +/// Root size for the secondary (`gfx1-asc`) instance; the extra 0x10 bytes are +/// the two secondary-only pointers at `+0xb8`/`+0xc0`. +pub(crate) const ROOT_SIZE_SECONDARY: usize = 0x0c8; + +pub(crate) const MAIN_CONFIG_SIZE: usize = 0x600; + +pub(crate) const REGION_A_SIZE: usize = 0x4000; + +pub(crate) const REGION_C_SIZE: usize = 0x1000; + +pub(crate) const STATUS_BLOCK_SIZE: usize = 0x80; + +pub(crate) const HW_DATA_BUNDLE_MIN_ALLOC: usize = 0x10000; + +/// The native placement additionally hosts the twelve work rings and both main +/// configuration objects inside the same cluster, which then spans this much +/// (see [`WORK_RING_BUNDLE_OFFSETS`], [`NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET`]). +pub(crate) const NATIVE_SHARED_CLUSTER_SIZE: usize = 0x28000; + +/// The same tuple as the little-endian u64 the firmware compares in one go. +pub(crate) const INITDATA_VERSION_WORD: u64 = 0x0c8a_a322_0396_04c0; + +/// Target/build-specific root encoding. This is a serialization choice, not +/// permission to start firmware or reuse the other target's hardware tables. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RootProfile { + J700G17P, + /// J813 Mac17,3 / 25G76, RTKit-3255.160.4.release. Both roots were captured + /// before INBOX1 publication. Independently confirmed by the loaded GFX + /// version comparison at +0x1fa5c and GFX1 at +0x21140. + J813G17G25G76, + /// J873 Mac18,5 / T8152 / G18G / 26A428, RTKit_release-3514.0.15 + /// (armfw_g18g-b000 on GFX, armfw1_g18g-b010 on GFX1). No capture of a + /// native root exists; the encoding is source-constructed from the 26A428 + /// kext and firmware (see the "J873 / G18G" section below). Selecting it + /// changes nothing for the other two profiles. + J873G18G26A428, +} + +impl RootProfile { + pub(crate) const fn version_word(self) -> u64 { + match self { + Self::J700G17P => INITDATA_VERSION_WORD, + Self::J813G17G25G76 => 0x0c8a_a322_039e_04c0, + Self::J873G18G26A428 => G18G_26A428_INITDATA_VERSION_WORD, + } + } + + /// Bytes of the root object for `role`. J700/J813 keep the historical + /// 0xb8/0xc8 split. G18G sizes both roots to 0xc8 because both firmware + /// images copy 0xc8 bytes from each root (FW_PROVEN b000 0x21664-0x216a8). + pub(crate) const fn root_size(self, role: InstanceRole) -> usize { + match self { + Self::J873G18G26A428 => G18G_26A428_ROOT_SIZE, + Self::J700G17P | Self::J813G17G25G76 => root_size(role), + } + } + + /// Opcode of the one opening device-control record staged at main + /// `+0x4c0`. G18G renumbered the opening handler: 0x16 is now an + /// unrelated handler and 0x15 reaches the opening (FW_PROVEN jump table + /// 0xfffffc0000006c60, handler b000 0x3ed0). The secondary keeps 0x2a. + pub(crate) const fn control_opening_opcode(self, role: InstanceRole) -> u32 { + match (self, role) { + (Self::J873G18G26A428, InstanceRole::Primary) => { + G18G_26A428_CONTROL_OPENING_OPCODE_PRIMARY + } + (Self::J873G18G26A428, InstanceRole::Secondary) => { + G18G_26A428_CONTROL_OPENING_OPCODE_SECONDARY + } + (_, InstanceRole::Primary) => CONTROL_OPENING_OPCODE_PRIMARY, + (_, InstanceRole::Secondary) => CONTROL_OPENING_OPCODE_SECONDARY, + } + } + + /// Bundle offsets of the five views published at main `+0x254`. + pub(crate) const fn bundle_view_offsets(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => G18G_26A428_BUNDLE_VIEW_OFFSETS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_OFFSETS, + } + } + + /// Extents each view must fit inside the bundle allocation. + pub(crate) const fn bundle_view_extents(self) -> [usize; 5] { + match self { + Self::J873G18G26A428 => G18G_26A428_BUNDLE_VIEW_EXTENTS, + Self::J700G17P | Self::J813G17G25G76 => BUNDLE_VIEW_EXTENTS, + } + } + + /// Bundle offset named twice by main `+0x08`/`+0x10` (the empty BRN + /// table and the 0x88 block, which the allocator places together). + pub(crate) const fn repeated_region_bundle_offset(self) -> usize { + match self { + Self::J873G18G26A428 => G18G_26A428_SHARED_LAYOUT.block_88, + Self::J700G17P | Self::J813G17G25G76 => MAIN_REPEATED_REGION_BUNDLE_OFFSET, + } + } + +} + +const INITDATA_ADDR_MASK: u64 = 0x0000_0fff_ffff_ffff; + +const INITDATA_ADDR_PREFIX: u64 = 0xffff_f000_0000_0000; + +const ROOT_VERSION: usize = 0x00; + +const ROOT_REGION_A: usize = 0x08; + +const ROOT_MAIN_CONFIG: usize = 0x18; + +const ROOT_REGION_C: usize = 0x20; + +const ROOT_INSTANCE_KIND: usize = 0x28; + +const ROOT_CONSTANT_2C: usize = 0x2c; + +const ROOT_PAGE_SIZE: usize = 0x30; + +const ROOT_PAGE_BITS: usize = 0x32; + +const ROOT_LEVEL_COUNT: usize = 0x33; + +const ROOT_LEVELS: usize = 0x34; + +/// Root `+0xa8`: Status A pointer. (The M5-derived model misused this slot as +/// a role pointer.) +const ROOT_STATUS_A: usize = 0xa8; + +/// Root `+0xb0`: Status B pointer. Real for the primary, zero for the +/// secondary. (The M5-derived model misused it as a cross-role pointer.) +const ROOT_STATUS_B: usize = 0xb0; + +const ROOT_SECONDARY_EXTRA_0: usize = 0xb8; + +const ROOT_SECONDARY_EXTRA_1: usize = 0xc0; + +pub(crate) const UAT_PAGE_SIZE: u16 = 0x4000; + +pub(crate) const UAT_PAGE_BITS: u8 = 14; + +pub(crate) const UAT_LEVEL_COUNT: usize = 3; + +/// `(index_shift, entry_count)` for the three levels below the root, +/// confirmed against a live table walk. +pub(crate) const UAT_LEVELS: [(u8, u16); UAT_LEVEL_COUNT] = [(36, 64), (25, 2048), (14, 2048)]; + +pub(crate) const UAT_LEVEL_DESC_SIZE: usize = 0x20; + +/// Level descriptor `+0x10`: physical mask, identical in all three. +pub(crate) const UAT_LEVEL_PHYS_MASK: u64 = 0x0000_03ff_ffff_c000; + +const MAIN_HW_DATA: usize = 0x00; + +/// The same repeated-region address appears at `+0x08` and `+0x10`; it is the +/// bundle base plus [`MAIN_REPEATED_REGION_BUNDLE_OFFSET`], not the bundle +/// base itself. It reads zero at handoff (firmware writes into it), so the +/// host only has to provide it mapped. +const MAIN_REPEATED_0: usize = 0x08; + +const MAIN_REPEATED_1: usize = 0x10; + +pub(crate) const MAIN_REPEATED_REGION_BUNDLE_OFFSET: usize = 0xc500; + +const MAIN_CHANNEL_TABLE: usize = 0x20; + +/// Main `+0x254`: five unaligned u64 views into the hardware-data bundle. +const MAIN_BUNDLE_VIEWS: usize = 0x254; + +/// Main `+0x2d0`: six qwords of region views. +const MAIN_REGION_VIEWS: usize = 0x2d0; + +/// Main `+0x300`: u32 4, secondary only. +const MAIN_SECONDARY_KIND: usize = 0x300; + +const MAIN_SECONDARY_KIND_VALUE: u32 = 4; + +/// Main `+0x3e0`: u32 0xff, primary only. +const MAIN_PRIMARY_MASK: usize = 0x3e0; + +const MAIN_PRIMARY_MASK_VALUE: u32 = 0xff; + +const MAIN_PRIMARY_SCALAR_32C: usize = 0x32c; + +const MAIN_PRIMARY_SCALAR_344: usize = 0x344; + +/// Main `+0x471`: unaligned u64, secondary only, pointing *into* bundle view 2 +/// at [`SECONDARY_EXTRA_VIEW_OFFSET`] (not a fresh allocation — the M5-derived +/// model had this at `+0x481` backed by an invented 0x12688-byte buffer; both +/// were wrong). +const MAIN_SECONDARY_EXTRA_PTR: usize = 0x471; + +pub(crate) const SECONDARY_EXTRA_VIEW_INDEX: usize = 2; + +pub(crate) const SECONDARY_EXTRA_VIEW_OFFSET: usize = 0xe40; + +/// Main `+0x4c0` is both a scalar and a place: the leading u32 of the +/// device-control ring, which is embedded in the main configuration object at +/// exactly this offset on both instances. Writing the opening opcode here *is* +/// staging the opening device-control record (records are 0x40 bytes; fields +/// past the opcode are zero). The control channel's producer counter must then +/// be set to the role-specific value returned by +/// [`control_opening_producer`] before the descriptor is handed over. +const MAIN_CONTROL_RING: usize = 0x4c0; + +pub(crate) const CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x16; + +pub(crate) const CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; + +/// Bundle offsets of the five views named at main `+0x254`. They are views of +/// the common bundle, not independent objects: view 0 overlaps the tail of +/// the hardware-data object, and views 3 and 4 overlap each other by all but +/// 0x80 bytes. +pub(crate) const BUNDLE_VIEW_OFFSETS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; + +/// Logical extents of the five views. +pub(crate) const BUNDLE_VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; + +/// The context-0 mappings of the PB descriptor and UMA page-pool descriptor +/// tables published at main `+0x2d8` and `+0x2e8`, respectively. The matching +/// firmware-high mappings are published at `+0x2e0` and `+0x2f0`. +pub(crate) const REGION_VIEW_LOW_ADDRS: [u64; 2] = [0x70_0183_8000, 0x70_0184_0000]; + +pub(crate) const CHANNEL_TABLE_ENTRIES: usize = 17; + +pub(crate) const CHANNEL_ENTRY_SIZE: usize = 0x20; + +/// Channel entry layout: three *separate* 32-bit state-counter addresses at +/// `+0x00`/`+0x08`/`+0x10` (the counters themselves sit at 0x10 spacing in +/// their state block — do not collapse them into one state object), then the +/// ring base at `+0x18`. +pub(crate) const CHANNEL_ENTRY_STATE_COUNT: usize = 3; + +pub(crate) const CHANNEL_STATE_SPACING: usize = 0x10; + +/// Entries 0..11 are ordered TA_0, 3D_0, CL_0, TA_1, ... CL_3. +pub(crate) const WORK_CHANNEL_COUNT: usize = 12; + +/// Entry 12 is device control. It continues the compact control-state grid +/// ([`WORK_STATE_GRID_OFFSETS`] has 13 entries, one more than there are work +/// channels), and its ring is embedded at main `+0x4c0`. The M5-derived model +/// refused this index, claiming a "cursor pair at +0x1a0/+0x1a8"; that was a +/// misread — those offsets are exactly this entry's state-counter addresses 0 +/// and 1. +pub(crate) const CONTROL_CHANNEL_INDEX: usize = 12; + +/// Entries 13 and 14 are report channels with split state objects: their +/// state addresses alias off status A rather than the compact grid, and the +/// firmware peer of each host-owned counter is at `+0x20` within its object. +pub(crate) const REPORT_CHANNEL_INDICES: [usize; 2] = [13, 14]; + +/// Entry 15 is partial: only its first state address, no ring. Entry 16 is +/// empty. Filling them like the rest declares channels that do not exist. +pub(crate) const PARTIAL_CHANNEL_INDEX: usize = 15; + +/// Offsets of the channel state blocks (channels 0..=12) inside the compact +/// state grid. Native state is *not* in channel-table order: within each +/// workload group the work queues sit at descending offsets, and the control +/// channel's block follows at `+0x300`. Preserving these address identities is +/// part of the descriptor ABI. +pub(crate) const WORK_STATE_GRID_OFFSETS: [usize; 13] = [ + 0x0c0, 0x1c0, 0x2c0, // TA_0, 3D_0, CL_0 + 0x080, 0x180, 0x280, // TA_1, 3D_1, CL_1 + 0x040, 0x140, 0x240, // TA_2, 3D_2, CL_2 + 0x000, 0x100, 0x200, // TA_3, 3D_3, CL_3 + 0x300, // device control +]; + +/// The primary status-B object runs from there to status A: `0xeb00` bytes. +pub(crate) const PRIMARY_STATUS_B_OBJECT_SIZE: usize = 0xeb00; + +/// State-counter addresses of report channels 13/14, as offsets from status A. +pub(crate) const REPORT_STATE_STATUS_A_OFFSETS: [[usize; 3]; 2] = + [[0x00040, 0x002c0, 0x00080], [0x00240, 0xa6ac0, 0x00280]]; + +/// Ring bases of report channels 13/14, as offsets from status A. +pub(crate) const REPORT_RING_STATUS_A_OFFSETS: [usize; 2] = [0x04ac0, 0xafac0]; + +/// Entry 15's sole state address, as an offset from status A. +pub(crate) const PARTIAL_STATE_STATUS_A_OFFSET: usize = 0x2d2c0; + +/// Native bundle offsets of the twelve work rings (0x1800 each), in +/// channel-table order. These extend past [`HW_DATA_BUNDLE_MIN_ALLOC`]; a host +/// using this placement sizes the cluster [`NATIVE_SHARED_CLUSTER_SIZE`]. +pub(crate) const WORK_RING_BUNDLE_OFFSETS: [usize; 12] = [ + 0x10dc0, 0x16dc0, 0x1cdc0, 0x0f5c0, 0x155c0, 0x1b5c0, 0x0ddc0, 0x13dc0, 0x19dc0, 0x0c5c0, + 0x125c0, 0x185c0, +]; + +/// Native bundle offsets of the two main configuration objects. +pub(crate) const NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET: usize = 0x1e5c0; + +pub(crate) const NATIVE_SECONDARY_MAIN_BUNDLE_OFFSET: usize = 0x22a80; + +pub(crate) const HW_REGISTER_MAP_ENTRY_SIZE: usize = 0x28; + +pub(crate) const HW_REGISTER_MAP_SLOTS: usize = 53; + +/// Two GPU mappings of one present completion-queue backing. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17PKsmCompletionAliases { + pub(crate) low: u64, + pub(crate) high: u64, +} + +pub(crate) const M5_25G76_COMPLETION_QUEUE_COUNT: usize = 4; + +pub(crate) const M5_25G76_COMPLETION_BACKING_SIZE: usize = 0x800 * 0x40; + +/// M5 publishes all four descriptors at hardware-data +0x2614, with ordinals +/// 0,1,2,3. Each names low/high mappings of its own retained 128-KiB backing. +/// The two native pre-handoff captures agree on every descriptor byte. +/// Construction only: this does not allocate, map, publish, or poll a queue. +pub(crate) fn encode_m5_25g76_completion_table( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_COMPLETION_QUEUE_COUNT * 0x20 { + return Err(InitdataError::BufferSize); + } + let extent = M5_25G76_COMPLETION_BACKING_SIZE as u64; + // Validate every range before mutating the output. Different queues must + // not alias each other in either address space. + for (i, alias) in aliases.iter().enumerate() { + for address in [alias.low, alias.high] { + required_address(address)?; + let end = address.checked_add(extent).ok_or(InitdataError::AddressOutOfRange)?; + required_address(end - 1)?; + } + for prior in &aliases[..i] { + if (alias.low < prior.low + extent && prior.low < alias.low + extent) + || (alias.high < prior.high + extent && prior.high < alias.high + extent) + { + return Err(InitdataError::PairAliasedObject); + } + } + } + out.fill(0); + for (ordinal, alias) in aliases.iter().enumerate() { + let offset = ordinal * 0x20; + put_u64(out, offset, alias.low); + put_u64(out, offset + 8, alias.high); + put_u32(out, offset + 0x10, 0x800); + put_u32(out, offset + 0x14, 0x40); + put_u32(out, offset + 0x18, ordinal as u32); + } + Ok(()) +} + +/// Status block `+0x04` holds u32 1 pre-init. `+0x10`/`+0x14` are *post-ack +/// firmware state* and must be zero at handoff. +pub(crate) const STATUS_BLOCK_ONE_OFFSET: usize = 0x04; + +pub(crate) const PRIMARY_STATUS_B_FWCTL_STATE: usize = 0x48e0; + +pub(crate) const PRIMARY_STATUS_B_FWCTL_RING: usize = 0x48e8; + +pub(crate) const PRIMARY_STATUS_B_CONFIG_HEADER: usize = 0xe434; + +pub(crate) const PRIMARY_STATUS_B_CONFIG_OFFSET: usize = 0xe440; + +/// The secondary root sits exactly this far above the primary root; the +/// secondary's initialisation message carries the primary's address plus this +/// delta in the same field, so the pair is not placed freely. +pub(crate) const SECONDARY_ROOT_DELTA: u64 = 0x8000; + +/// Physical firmware instance encoded at root `+0x28`. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum InstanceRole { + /// `gfx-asc`: scheduler, device control, and all work channels. + Primary = 0, + /// `gfx1-asc`: control-only peer. Linux starts it after primary RTKit is ready. + Secondary = 1, +} + +pub(crate) const fn root_size(role: InstanceRole) -> usize { + match role { + InstanceRole::Primary => ROOT_SIZE_PRIMARY, + InstanceRole::Secondary => ROOT_SIZE_SECONDARY, + } +} + +/// GPU virtual addresses of the objects both instances share. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedAddresses { + /// Shared 16 KiB region A (root `+0x08`). + pub(crate) region_a: u64, + /// Shared region C (root `+0x20`), 0x1000 bytes. + pub(crate) region_c: u64, + /// Base of the contiguous hardware-data bundle. The hardware-data object + /// sits at the base; the five main-config views, the repeated region, and + /// the secondary's `+0x471` pointer are all derived offsets into it. + pub(crate) hw_data_bundle: u64, + /// Size of that one allocation; must be at least + /// [`HW_DATA_BUNDLE_MIN_ALLOC`]. + pub(crate) hw_data_bundle_alloc: usize, +} + +/// GPU virtual addresses of one instance's private objects. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct InstanceAddresses { + /// This instance's root; its address rides the endpoint-0x20 doorbell. + pub(crate) root: u64, + pub(crate) main_config: u64, + pub(crate) status_a: u64, + /// Required for the primary; must be zero for the secondary (whose root + /// still has the field, written zero). + pub(crate) status_b: u64, + /// Root `+0xb8`/`+0xc0`. Required for the secondary; the primary root is + /// only 0xb8 bytes and cannot even hold them, so they must be zero there. + pub(crate) secondary_extras: [u64; 2], +} + +/// One channel-table entry: three state-counter addresses, then the ring base. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct ChannelEntry { + pub(crate) states: [u64; 3], + pub(crate) ring: u64, +} + +impl ChannelEntry { + pub(crate) const EMPTY: ChannelEntry = ChannelEntry { + states: [0; 3], + ring: 0, + }; +} + +/// The primary's global region views at main `+0x2d0`: a globally retained, +/// initially zero HWDSID accounting table (256 pairs of u32), followed by +/// the high halves of two low/high alias pairs (the low halves +/// are the fixed context-0 addresses in +/// [`REGION_VIEW_LOW_ADDRS`]). Each pair must map one physical page from both +/// address spaces — an invariant of the UAT mapping layer that cannot be +/// checked from the addresses alone. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct PrimaryRegionViews { + pub(crate) sentinel_high: u64, + pub(crate) alias_high: [u64; 2], +} + +/// M5 25G76 descriptor fields +0x10 and +0x14 are independent. In slot 39 +/// native publishes 0x10000 and 0x8000 respectively. The older RegisterWindow +/// encoder duplicates one size and must not serialize this target's table. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RegisterWindow { + pub(crate) slot: usize, + pub(crate) phys: u64, + pub(crate) device_va: u64, + pub(crate) size_10: u32, + pub(crate) size_14: u32, + pub(crate) unk_18: u64, + pub(crate) flag: u32, +} + +impl M5RegisterWindow { + /// Actual M5 PTE walks cover size_10 rounded to 16 KiB, including the + /// four pages of slot 39. size_14 is independently serialized firmware + /// metadata and must not truncate that mapping to two pages. + pub(crate) fn mapped_pages(&self) -> Result<(u64, u64, u64), InitdataError> { + let mask = UAT_PAGE_SIZE as u64 - 1; + if self.size_10 == 0 || self.phys & mask != self.device_va & mask { + return Err(InitdataError::RegisterWindow); + } + let physical = self.phys & !mask; + let virtual_address = self.device_va & !mask; + let end = self.device_va.checked_add(self.size_10 as u64) + .and_then(|value| value.checked_add(mask)) + .ok_or(InitdataError::AddressOutOfRange)? & !mask; + let span = end.checked_sub(virtual_address).ok_or(InitdataError::AddressOutOfRange)?; + physical.checked_add(span).ok_or(InitdataError::AddressOutOfRange)?; + Ok((physical, virtual_address, span)) + } +} + +/// Source-constructed register descriptors from both identical pre-handoff +/// J813/25G76 captures. These are firmware virtual mappings, not a request to +/// access or power the devices. No runtime currently consumes this profile. +pub(crate) const M5_25G76_REGISTER_WINDOWS: [M5RegisterWindow; 22] = { + const fn w(slot: usize, phys: u64, va: u64, size_10: u32, size_14: u32, + unk_18: u64, flag: u32) -> M5RegisterWindow { + M5RegisterWindow { slot, phys, device_va: 0xffff_fc21_8000_0000 + va, + size_10, size_14, unk_18, flag } + } + [ + w(0, 0x381014000, 0x000000, 0x04000, 0x04000, 0, 2), + w(3, 0x220104000, 0x008000, 0x18000, 0x18000, 0, 2), + w(8, 0x3a1350000, 0x028000, 0x04000, 0x04000, 0, 2), + w(9, 0x3803d0000, 0x030000, 0x04000, 0x04000, 0, 2), + w(10, 0x3803c0000, 0x038000, 0x04000, 0x04000, 0, 0), + w(12, 0x42165c000, 0x040000, 0x04000, 0x04000, 0, 2), + w(14, 0x380280000, 0x048000, 0x08000, 0x08000, 0, 0), + w(15, 0x38c840000, 0x058000, 0x24000, 0x24000, 0, 2), + w(17, 0x580000000, 0x080000, 0x21400, 0x21400, 0, 2), + w(22, 0x581000000, 0x0a8000, 0x08000, 0x08000, 0, 2), + w(24, 0x580ec0000, 0x0b8000, 0x10000, 0x10000, 0, 2), + w(26, 0x580d04000, 0x0d0000, 0x08000, 0x08000, 0xd04000, 2), + w(27, 0x580d0d000, 0x0e1000, 0x01000, 0x01000, 0xd0d000, 2), + w(28, 0x580d50000, 0x0e8000, 0x10000, 0x10000, 0xd50000, 2), + w(29, 0x580d10000, 0x100000, 0x04000, 0x04000, 0xd10000, 2), + w(31, 0x580d40000, 0x108000, 0x04000, 0x04000, 0xd40000, 2), + w(32, 0x580d60000, 0x110000, 0x08000, 0x08000, 0xd60000, 2), + w(35, 0x580e00000, 0x120000, 0x04000, 0x04000, 0xe00000, 2), + w(39, 0x580e08000, 0x128000, 0x10000, 0x08000, 0, 2), + w(41, 0x580e33800, 0x143800, 0x04000, 0x04000, 0, 2), + w(42, 0x580e35000, 0x151000, 0x04000, 0x04000, 0, 2), + w(50, 0x580014400, 0x160400, 0x00400, 0x00400, 0x14400, 0), + ] +}; + +pub(crate) fn encode_m5_25g76_register_map(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != HW_REGISTER_MAP_SLOTS * HW_REGISTER_MAP_ENTRY_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + for window in M5_25G76_REGISTER_WINDOWS { + let base = window.slot * HW_REGISTER_MAP_ENTRY_SIZE; + put_u64(out, base, window.phys); + put_u64(out, base + 8, window.device_va); + put_u32(out, base + 0x10, window.size_10); + put_u32(out, base + 0x14, window.size_14); + put_u64(out, base + 0x18, window.unk_18); + put_u32(out, base + 0x20, window.flag); + } + for slot in [2, 5, 6, 7, 30, 33, 34, 37, 38, 40, 43, 46, 47, 48, 49, 52] { + put_u32(out, slot * HW_REGISTER_MAP_ENTRY_SIZE + 0x20, 2); + } + Ok(()) +} + +pub(crate) const M5_25G76_HW_DATA_SIZE: usize = 0x2708; + +pub(crate) const M5_25G76_SHARED_PREFIX_SIZE: usize = 0xc500; + +pub(crate) const M5_25G76_AON_PTD_SIZE: usize = 0xe4; + +/// J813/25G76 AON PTD backing, captured twice before each role's handoff and +/// unchanged after a passing Metal workload. The three recorded words are +/// scoped to this platform's telemetry description; they are not generic +/// M5 constants. The allocation and GPU binding must remain caller-owned. +pub(crate) fn encode_m5_25g76_aon_ptd(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != M5_25G76_AON_PTD_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, 0, 0x6730_5452); + put_u32(out, 4, 0x001d_0001); + put_u32(out, 8, 0x0000_1401); + Ok(()) +} + +const M5_25G76_STATUS_CONFIG_WORDS: &[(usize, u32)] = &[ + (0x010, 0x0000001f), (0x014, 0x3e0a3d71), (0x018, 0x401ae148), + (0x01c, 0x00000028), (0x020, 0x00000001), (0x0c8, 0x00002af8), + (0x0cc, 0x00001f40), (0x0d8, 0x41980000), (0x0dc, 0x3d0f5c28), + (0x0f0, 0x000026ac), (0x0f4, 0x000000c8), (0x108, 0x42c80000), + (0x10c, 0x43c80000), (0x110, 0x000000c8), (0x128, 0x00000001), + (0x12c, 0x00005eb0), (0x130, 0x3e800000), (0x134, 0x3f99999a), + (0x158, 0x0000000e), (0x1c0, 0x00000001), (0x1c4, 0x00000001), + (0x1c8, 0x00000002), (0x1cc, 0x00000001), (0x1d0, 0x00000001), + (0x1d4, 0x00000001), (0x1d8, 0x00000001), (0x588, 0x00000001), + (0x58c, 0x0000000a), (0x590, 0x00000dac), (0x594, 0x000003e8), + (0x598, 0x00000bb8), (0x59c, 0x000002bc), (0x5cc, 0x00000001), + (0x5d0, 0x00000004), (0x5dc, 0x00001f40), (0x5e0, 0x000000c8), + (0x5e4, 0x00000fa0), (0x5e8, 0x0000012c), (0x5ec, 0x000007d0), + (0x5f0, 0x0000012c), (0x5f4, 0x000003e8), (0x5f8, 0x0000012c), + (0x600, 0x00000001), (0x604, 0x41700000), (0x608, 0x40a00000), + (0x60c, 0x00000020), (0x610, 0x000004b0), (0x620, 0x0000044c), +]; + +/// Build the primary status object before either firmware role is started. +/// Secondary root+0xb8 aliases status+0xe440. GFX1 copies that pointer to +/// firmware global 0x176248 at text+0x211d4 and later writes config+0x670, +/// +0x678 and +0x680 at text+0x1af94..0x1afc4. Those are firmware-owned +/// runtime state, not platform constants, and must start at zero even when +/// a primary pre-handoff capture (taken after secondary startup) has values. +pub(crate) fn encode_m5_25g76_primary_status_b( + fwctl_state: u64, + fwctl_ring: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != PRIMARY_STATUS_B_OBJECT_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_status_block(&mut out[..STATUS_BLOCK_SIZE])?; + put_u64(out, PRIMARY_STATUS_B_FWCTL_STATE, fwctl_state); + put_u64(out, PRIMARY_STATUS_B_FWCTL_RING, fwctl_ring); + put_u32(out, PRIMARY_STATUS_B_CONFIG_HEADER, 1); + for &(offset, value) in M5_25G76_STATUS_CONFIG_WORDS { + put_u32(out, PRIMARY_STATUS_B_CONFIG_OFFSET + offset, value); + } + Ok(()) +} + +/// Shared private arena observed before primary initialization on J813. +/// The status and secondary-grid pointers name overlapping views of this +/// one backing. This extent is a measured mapped arena, not a claim about +/// the sizes of the native host's individual suballocations. +pub(crate) const M5_25G76_PRIVATE_CLUSTER_SIZE: usize = 0x178000; + +pub(crate) fn encode_m5_25g76_private_cluster( + fwctl_state: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_PRIVATE_CLUSTER_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(fwctl_state)?; + let fwctl_ring = fwctl_state.checked_add(0x40).ok_or(InitdataError::AddressOutOfRange)?; + required_address(fwctl_ring)?; + out.fill(0); + encode_m5_25g76_primary_status_b(fwctl_state, fwctl_ring, &mut out[0x340..0xee40])?; + for offset in [0xee40, 0xc3100] { + encode_status_block(&mut out[offset..offset + STATUS_BLOCK_SIZE])?; + } + encode_secondary_root_extra_1_for_profile(RootProfile::J813G17G25G76, + &mut out[0x177400..0x177480])?; + // The main-config encoders carry one measured opening record (0x16 / + // 0x2a). Publish exactly one producer entry for each control channel. + // All work-channel and report counters remain zero. + put_u32(out, 0x320, 1); + put_u32(out, 0x1773e0, 1); + Ok(()) +} + +/// Caller-owned bindings in the power/performance policy allocation. +/// The host stores mach_absolute_time() at policy+0x651c. Supply the live +/// platform counter in its native timebase, never a captured timestamp. +#[derive(Clone, Copy)] +pub(crate) struct M5PolicyBindings { + pub state: u64, + pub aon_ptd: u64, + pub boot_timestamp: u64, +} + +/// Pre-handoff J813/25G76 scalars, identical across two cold boots after +/// removing owned pointers and the native boot timestamp. The repeated +/// 512-entry u16 table is encoded separately. Calibration fields whose +/// meanings remain unknown are scoped to this measured ten-core target. +const M5_25G76_POLICY_WORDS: &[(usize, u32)] = &[ + (0x3d98, 0xffffffff), (0x3db0, 0xffffffff), (0x5244, 0x00030d40), + (0x5248, 0x00030d40), (0x5250, 0x00000004), (0x5254, 0x3f800000), + (0x526c, 0x00000001), (0x5270, 0x00000001), (0x5280, 0x00000064), + (0x5284, 0x00000001), (0x5288, 0x000004b0), (0x5290, 0x00000064), + (0x52a0, 0x00000004), (0x52a4, 0x3f800000), (0x52ac, 0x00000001), + (0x52b0, 0x00000001), (0x52c0, 0x00000064), (0x52c4, 0x00000001), + (0x52c8, 0x000004b0), (0x52d0, 0x00000064), (0x5300, 0x3f828f5c), + (0x5304, 0x3f828f5c), (0x5308, 0x3f828f5c), (0x530c, 0x3f828f5c), + (0x5310, 0x3f828f5c), (0x5314, 0x3f828f5c), (0x5318, 0x3f828f5c), + (0x531c, 0x3f828f5c), (0x5320, 0x3f828f5c), (0x5324, 0x3f828f5c), + (0x5328, 0x3f828f5c), (0x532c, 0x3f828f5c), (0x5330, 0x3f828f5c), + (0x5bf8, 0x00000271), (0x5c04, 0x3f7f2e9f), (0x5c0c, 0x3b516154), + (0x5c14, 0x3ca59586), (0x5c20, 0x46bd6000), (0x5c24, 0x40a90fe4), + (0x5c2c, 0xbd4ac5b4), (0x5c30, 0x00000028), (0x5c34, 0x000004b0), + (0x5c38, 0x000004b0), (0x5c44, 0x00005eb0), (0x5c78, 0x000004b0), + (0x5cb0, 0x00000008), (0x5cbc, 0x3f000000), (0x5cc4, 0x3f000000), + (0x5ccc, 0x3f99999a), (0x5cd8, 0x47800000), (0x5cdc, 0x3e800000), + (0x5ce8, 0x00000028), (0x5cec, 0x000004b0), (0x5cfc, 0x00005eb0), + (0x5d08, 0x00000010), (0x5d10, 0x0005dc00), (0x5d68, 0x0000004a), + (0x5d70, 0x0000005a), (0x5d74, 0x00000032), (0x5d78, 0x00000006), + (0x5d80, 0x00000006), (0x5d84, 0x00000001), (0x5d8c, 0x3f4ccccd), + (0x5d90, 0x3f7eb852), (0x5d94, 0x3e4ccccd), (0x5d98, 0x3ba3d70a), + (0x5d9c, 0x3f144b3c), (0x5da0, 0x3f144b3c), (0x5da8, 0x42be0000), + (0x5dac, 0x4029eb85), (0x5db0, 0x4029eb85), (0x5db4, 0x4139435e), + (0x5db8, 0x00000064), (0x5dbc, 0x000004b0), (0x5dc0, 0x00000064), + (0x5dcc, 0x0000004a), (0x5dd4, 0x00000001), (0x5e00, 0x00000064), + (0x5e20, 0x00000401), (0x5e24, 0x00006400), (0x5e2c, 0x001f4000), + (0x5e30, 0x0000c800), (0x5e34, 0x000fa000), (0x5e38, 0x00012c00), + (0x5e3c, 0x0007d000), (0x5e40, 0x00012c00), (0x5e44, 0x0003e800), + (0x5e48, 0x00012c00), (0x5e4c, 0x00000100), (0x5ea4, 0x00000001), + (0x5ed0, 0x00000001), (0x74d8, 0x47800000), (0x74dc, 0x42200000), + (0x74e0, 0x44960000), (0x74e4, 0x0000062a), (0x74f8, 0x00000028), + (0x7500, 0x42c80000), (0x7504, 0x000004b0), (0x750c, 0x3f4ccccd), + (0x7510, 0x3e4ccccd), (0x7550, 0xdcdcfbfa), (0x7558, 0x00001f40), + (0x7560, 0x00000004), (0x7574, 0x3f800000), (0x757c, 0x3d0f5c28), + (0x7588, 0x47800000), (0x758c, 0x41980000), (0x7598, 0x00000028), + (0x759c, 0x000004b0), (0x75a0, 0x000004b0), (0x75ac, 0x00002af8), + (0x75e0, 0x000004b0), (0x77f8, 0x00000002), (0x77fc, 0x42000006), + (0x7a10, 0xb8520000), (0x7a14, 0x00003f7e), (0x7a18, 0xd70a0000), + (0x7a1c, 0x00003ba3), (0x7a24, 0x000043c8), (0x7a30, 0x00004780), + (0x7a34, 0x000042c8), (0x7a3c, 0x0384bac8), (0x7a40, 0x03e80000), + (0x7a44, 0x03e80000), (0x7a4c, 0x80000000), (0x7a50, 0x26ac453b), + (0x7a84, 0x03e80000), (0x7aa0, 0x00c80000), (0x7b58, 0x000004b0), + (0x7b6c, 0x00000001), (0x7ba0, 0x00000101), (0x7ba4, 0x00000200), + (0x7bbc, 0x00023f00), (0x7bc0, 0x00000100), (0x7bc4, 0x00000100), + (0x7bdc, 0x00024000), (0x7be0, 0x00000100), (0x7be4, 0x00000100), + (0x7bfc, 0x00024100), (0x7c48, 0x00000001), (0x7c50, 0x00005eb0), + (0x7c54, 0x00005eb0), (0x7c58, 0x00005eb0), (0x7c68, 0x3f800000), + (0x7c6c, 0x00000004), (0x7c70, 0x00000008), (0x7c74, 0x0002ee00), + (0x7d44, 0x0000007c), (0x7d50, 0x3f77bdef), (0x7d58, 0x3d042108), + (0x7d60, 0x3e0a3d71), (0x7d6c, 0x47800000), (0x7d70, 0x401ae148), + (0x7d7c, 0x00000028), (0x7d80, 0x000004b0), (0x7d84, 0x000004b0), + (0x7d8c, 0x46bd6000), (0x7d90, 0x00005eb0), (0x7d9c, 0x000000f8), + (0x7da4, 0x005ad200), (0x7dc4, 0x000004b0), (0x7e04, 0x00000001), + (0x7ed4, 0x00000032), (0x7ed8, 0x00000001), (0x7ee0, 0x3f638e39), + (0x7ee4, 0x3f2aaaab), (0x7ee8, 0x3de38e39), (0x7eec, 0x3eaaaaab), + (0x7ef0, 0xbecccccd), (0x7ef4, 0xbf4ccccd), (0x7efc, 0x47800000), + (0x7f00, 0xc0a00000), (0x7f04, 0xc0a00000), (0x7f0c, 0x00000064), + (0x7f10, 0x000004b0), (0x7f14, 0x00000064), (0x7f1c, 0x45fa0000), + (0x7f20, 0x00000578), (0x7f28, 0x0000000e), (0x7f2c, 0x00000048), + (0x7f30, 0x00000018), (0x7f34, 0x001a5e00), (0x7f3c, 0x0008ca00), + (0x7f48, 0x47800000), (0x7f70, 0x00001f40), (0x7f74, 0x00000004), + (0x7f78, 0x000009c4), (0x7f7c, 0x00000258), (0x7f80, 0x00000002), + (0x7f84, 0x00000004), (0x7f88, 0x00000032), (0x7fb8, 0x47800000), + (0x7fc8, 0x00000028), (0x7fcc, 0x000004b0), (0x8098, 0x00000001), + (0x80ac, 0x0000000a), (0x80b8, 0x00000001), (0x8174, 0xac000000), + (0x8178, 0xe800000d), (0x817c, 0xb8000003), (0x8180, 0xb000000b), + (0x8184, 0xb0000004), (0x8188, 0xbc000004), (0x818c, 0x20000002), + (0x8198, 0x00000001), (0x81b4, 0x00000004), (0x81c0, 0x3f780000), + (0x81c8, 0x3d000000), (0x81d0, 0x40a00000), (0x81dc, 0x47800000), + (0x81e0, 0x41700000), (0x81ec, 0x0000044c), (0x81f0, 0x000004b0), + (0x81f4, 0x000004b0), (0x8200, 0x000004b0), (0x8234, 0x000004b0), + (0x827c, 0x00000064), (0x8290, 0xdcdcfbfa), (0x8298, 0x0000007d), + (0x829c, 0x00000001), (0x8ee0, 0xdcdcfbfb), (0x8ee8, 0x00000001), + (0xaeec, 0x41c80000), (0xaef0, 0x41c80000), (0xb6a8, 0x67305452), + (0xb6e8, 0x00000001), (0xb728, 0x5243464c), (0xb72c, 0x4758464c), + (0xb738, 0x47584c50), (0xb748, 0x00000001), (0xb7a8, 0x00000001), + (0xb7b4, 0x42b73333), (0xb7f4, 0x449b8000), (0xb834, 0x42bc0000), + (0xb8e0, 0x0000186a), +]; + +/// Initialize the six allocations following M5 hardware-data. Preserve the +/// hardware block and subsequent channel resources. No firmware is started. +pub(crate) fn encode_m5_25g76_bundle_policy( + bindings: M5PolicyBindings, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() < M5_25G76_SHARED_PREFIX_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(bindings.state)?; + required_address(bindings.aon_ptd)?; + let state_end = bindings.state.checked_add(STATUS_BLOCK_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + let aon_end = bindings.aon_ptd.checked_add(M5_25G76_AON_PTD_SIZE as u64) + .ok_or(InitdataError::AddressOutOfRange)?; + required_address(state_end - 1)?; + required_address(aon_end - 1)?; + if bindings.state < aon_end && bindings.aon_ptd < state_end { + return Err(InitdataError::PairAliasedObject); + } + out[0x2740..M5_25G76_SHARED_PREFIX_SIZE].fill(0); + for &(offset, value) in M5_25G76_POLICY_WORDS { + put_u32(out, offset, value); + } + for index in 0..512 { + put_u16(out, 0x6ed7 + index * 2, 0x7fff); + } + put_u64(out, 0x8ed8, bindings.state); + put_u64(out, 0xb6a0, bindings.aon_ptd); + put_u64(out, 0xb75c, bindings.boot_timestamp); + Ok(()) +} + +/// Recorded platform words outside the explicit register, performance and +/// owned-pointer fields. Meanings not yet derived are deliberately kept as +/// target-scoped measured values. Never insert runtime completion counters +/// or captured allocation addresses here. Reference hardware-data SHA-256: +/// 6f9ac0c4664f898a0ddb4e5df9ad160a23c1b52d3fd750750779cbdebe446be9. +const M5_25G76_HW_PLATFORM_WORDS: &[(usize, u32)] = &[ + (0x0004, 0x0000006f), (0x0008, 0xffc00000), (0x0014, 0x00000010), + (0x001c, 0x00000010), (0x0020, 0xffff8000), (0x0024, 0x000002ff), + (0x0028, 0x81400000), (0x002c, 0xfffffc21), (0x00e0, 0x00002008), + (0x00e8, 0x20080000), (0x00f4, 0x00002008), (0x00f8, 0x000024cb), + (0x00fc, 0xc98a2cfa), (0x0100, 0xf4f624cb), (0x0104, 0x1877e917), + (0x0108, 0x38d924cb), (0x010c, 0xbdab0000), (0x02d8, 0xfff82000), + (0x02dc, 0xd32d2cdb), (0x02e0, 0xf5002000), (0x02e4, 0x21dae926), + (0x02e8, 0x38b62000), (0x02ec, 0xc7420008), (0x03e0, 0x00007fe0), + (0x03e8, 0x7fe00000), (0x03f4, 0x00007fe0), (0x03f8, 0x00008000), + (0x0400, 0x80000000), (0x040c, 0x00008000), (0x05d8, 0x4b232645), + (0x05dc, 0x00000e98), (0x05e0, 0xd5a2ea5f), (0x05e4, 0x40003fff), + (0x05e8, 0xca5e4000), (0x05ec, 0x4000f5a2), (0x0e90, 0x00008142), + (0x0e94, 0x00000002), (0x0ea0, 0x00000001), (0x0ea4, 0x00000001), + (0x0eb8, 0x00000001), (0x0ec0, 0x00000001), (0x0ec8, 0x00000001), + (0x0ed0, 0x00005dc0), (0x0ed4, 0x00000001), (0x0ed8, 0x00000008), + (0x0ee0, 0x00000001), (0x0ee4, 0x00000001), (0x0ee8, 0x00000001), + (0x0f04, 0x0000001f), (0x0f24, 0x07800004), (0x0f28, 0x00000006), + (0x0f34, 0x00000001), (0x0f38, 0x00000001), (0x0f4c, 0x00000031), + (0x0f6c, 0x00000001), (0x0f88, 0x0000000a), (0x0f8c, 0x00000001), + (0x0fac, 0x00000001), (0x0fb8, 0x00000020), (0x0fbc, 0x00000005), + (0x0fc0, 0x0000000a), (0x0fc4, 0x0000000c), (0x2544, 0x00000004), + (0x2548, 0x00000006), (0x254c, 0x00000003), (0x2550, 0x00000007), + (0x2554, 0x00000007), (0x2568, 0x0000000a), (0x2570, 0x00000001), + (0x258c, 0x00180000), (0x2594, 0x00000001), (0x2598, 0x00000001), + (0x259c, 0x00000001), (0x25a4, 0x00000001), (0x25a8, 0x00000001), + (0x25ac, 0x00000001), (0x25b4, 0xffffffff), (0x25b8, 0xffffffff), + (0x25bc, 0xffffffff), (0x25c0, 0xffffffff), (0x25c4, 0xffffffff), + (0x25c8, 0xffffffff), (0x25cc, 0xffffffff), (0x25d0, 0xffffffff), + (0x25d4, 0xffffffff), (0x25d8, 0xffffffff), (0x25dc, 0xffffffff), + (0x25e0, 0xffffffff), (0x25f4, 0x00000001), (0x2600, 0x00000001), + (0x2608, 0x00000001), (0x2610, 0x00000100), (0x26b4, 0x00000001), + (0x26b8, 0x00000001), (0x26e0, 0x000003e8), (0x26e8, 0x00000001), +]; + +/// Reconstruct the M5 reference hardware-data block with caller-owned queue +/// and QoS bindings. Performance values are measured on the ten-core J813 +/// reference, not derived from ADT and not a generic M5 power configuration. +/// Both pre-handoff firmware roles supplied identical reference bytes. +/// This encoder grants no runtime admission and performs no MMIO accesses. +pub(crate) fn encode_m5_25g76_hw_data( + aliases: [G17PKsmCompletionAliases; M5_25G76_COMPLETION_QUEUE_COUNT], + qos_high: u64, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != M5_25G76_HW_DATA_SIZE { + return Err(InitdataError::BufferSize); + } + required_address(qos_high)?; + let qos_end = qos_high.checked_add(0xe40).ok_or(InitdataError::AddressOutOfRange)?; + required_address(qos_end - 1)?; + let mut completion = [0; 0x80]; + encode_m5_25g76_completion_table(aliases, &mut completion)?; + for alias in aliases { + if qos_high < alias.high + M5_25G76_COMPLETION_BACKING_SIZE as u64 + && alias.high < qos_end { + return Err(InitdataError::PairAliasedObject); + } + } + + out.fill(0); + for &(offset, value) in M5_25G76_HW_PLATFORM_WORDS { + put_u32(out, offset, value); + } + encode_m5_25g76_register_map(&mut out[0x640..0xe88])?; + out[0x2614..0x2694].copy_from_slice(&completion); + put_u64(out, 0x26ac, qos_high); + + // Native includes index zero plus twelve operating points. Each voltage + // occupies all sixteen lanes of its 0x40-byte state row. + const FREQUENCY: [u32; 13] = [0, 338, 486, 636, 796, 888, 988, 1084, 1182, 1278, 1374, 1470, 1578]; + const CORE_VOLTAGE: [u32; 13] = [125, 615, 630, 660, 705, 735, 755, 800, 830, 865, 905, 945, 1010]; + const MEMORY_VOLTAGE: [u32; 13] = [770, 770, 770, 770, 770, 770, 770, 800, 830, 865, 905, 945, 1010]; + const RELATIVE_A: [u32; 13] = [0, 17, 23, 30, 42, 50, 58, 71, 82, 85, 89, 93, 100]; + const RELATIVE_B: [u32; 13] = [0, 0, 11, 24, 36, 44, 52, 60, 68, 75, 83, 91, 100]; + for index in 0..13 { + put_u32(out, 0xfc8 + index * 4, FREQUENCY[index]); + put_u32(out, 0x1848 + index * 4, 0x3f828f5c); + put_u32(out, 0x18c8 + index * 4, RELATIVE_A[index]); + put_u32(out, 0x1908 + index * 4, RELATIVE_B[index]); + for lane in 0..16 { + put_u32(out, 0x1008 + index * 0x40 + lane * 4, CORE_VOLTAGE[index]); + put_u32(out, 0x1408 + index * 0x40 + lane * 4, MEMORY_VOLTAGE[index]); + } + } + for index in 0..16 { + put_u32(out, 0x19c8 + index * 4, index as u32); + } + Ok(()) +} + +/// Fail-closed validation errors for the grounded layout. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum InitdataError { + BufferSize, + AddressOutOfRange, + NullRequiredAddress, + RoleSpecificAddress, + BundleAllocationTooSmall, + ChannelTableShape, + RegisterWindow, + PairAliasedObject, +} + +fn address_representable(address: u64) -> bool { + let high = address & !INITDATA_ADDR_MASK; + high == 0 || high == INITDATA_ADDR_PREFIX +} + +fn required_address(address: u64) -> Result<(), InitdataError> { + if address == 0 { + Err(InitdataError::NullRequiredAddress) + } else if !address_representable(address) { + Err(InitdataError::AddressOutOfRange) + } else { + Ok(()) + } +} + +fn put_u16(bytes: &mut [u8], offset: usize, value: u16) { + bytes[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn write_runs(bytes: &mut [u8], runs: &[(usize, &[u8])]) { + for (offset, data) in runs { + bytes[*offset..*offset + data.len()].copy_from_slice(data); + } +} + +fn validate_shared(shared: &SharedAddresses) -> Result<(), InitdataError> { + required_address(shared.region_a)?; + required_address(shared.region_c)?; + required_address(shared.hw_data_bundle)?; + if shared.hw_data_bundle_alloc < HW_DATA_BUNDLE_MIN_ALLOC { + return Err(InitdataError::BundleAllocationTooSmall); + } + Ok(()) +} + +fn validate_instance(role: InstanceRole, inst: &InstanceAddresses) -> Result<(), InitdataError> { + required_address(inst.root)?; + required_address(inst.main_config)?; + required_address(inst.status_a)?; + match role { + InstanceRole::Primary => { + // The primary root has no +0xb8/+0xc0 fields at all; carrying + // secondary extras there is a role-confusion bug at the caller. + if inst.secondary_extras != [0, 0] { + return Err(InitdataError::RoleSpecificAddress); + } + required_address(inst.status_b)?; + } + InstanceRole::Secondary => { + // Every hardware-booting host hands the secondary a zero status-B; + // a nonzero value there is ungrounded, so fail closed on it. + if inst.status_b != 0 { + return Err(InitdataError::RoleSpecificAddress); + } + for extra in inst.secondary_extras { + required_address(extra)?; + } + } + } + Ok(()) +} + +/// Encode one 0x20-byte UAT level descriptor. Every field is derivable from +/// the level's index shift and entry count; the index mask +/// `(entries - 1) << shift` reproduces all three captured masks exactly. +fn encode_uat_level_descriptor(bytes: &mut [u8], offset: usize, shift: u8, entries: u16) { + bytes[offset] = 8; + bytes[offset + 1] = UAT_PAGE_BITS; + bytes[offset + 2] = UAT_PAGE_BITS; + bytes[offset + 3] = shift; + put_u16(bytes, offset + 0x04, entries); + put_u16(bytes, offset + 0x06, UAT_PAGE_SIZE); + put_u64(bytes, offset + 0x08, 1); + put_u64(bytes, offset + 0x10, UAT_LEVEL_PHYS_MASK); + put_u64(bytes, offset + 0x18, (entries as u64 - 1) << shift); +} + +/// Encode the common root layout with an explicitly selected firmware build. +/// The existing runtime continues to call the J700-only wrapper above. +pub(crate) fn encode_root_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + out: &mut [u8], +) -> Result<(), InitdataError> { + // Profile-sized: identical to root_size(role) for J700/J813. + if out.len() != profile.root_size(role) { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + + out.fill(0); + put_u64(out, ROOT_VERSION, profile.version_word()); + put_u64(out, ROOT_REGION_A, shared.region_a); + // ROOT_ZERO_10 deliberately left zero. + put_u64(out, ROOT_MAIN_CONFIG, inst.main_config); + put_u64(out, ROOT_REGION_C, shared.region_c); + put_u32(out, ROOT_INSTANCE_KIND, role as u32); + put_u32(out, ROOT_CONSTANT_2C, 1); + put_u16(out, ROOT_PAGE_SIZE, UAT_PAGE_SIZE); + out[ROOT_PAGE_BITS] = UAT_PAGE_BITS; + out[ROOT_LEVEL_COUNT] = UAT_LEVEL_COUNT as u8; + for (index, (shift, entries)) in UAT_LEVELS.iter().enumerate() { + encode_uat_level_descriptor( + out, + ROOT_LEVELS + index * UAT_LEVEL_DESC_SIZE, + *shift, + *entries, + ); + } + put_u64(out, ROOT_STATUS_A, inst.status_a); + put_u64(out, ROOT_STATUS_B, inst.status_b); + if role == InstanceRole::Secondary { + put_u64(out, ROOT_SECONDARY_EXTRA_0, inst.secondary_extras[0]); + put_u64(out, ROOT_SECONDARY_EXTRA_1, inst.secondary_extras[1]); + } + Ok(()) +} + +/// Derive the native channel table for one instance. +/// +/// `state_grid` is this instance's compact channel-state block (0x40-stride, +/// permuted per [`WORK_STATE_GRID_OFFSETS`]); `status_a` anchors the trailing +/// report/partial channels; the ring for entry 12 is `main_config + 0x4c0`; +/// work rings for the primary sit in the bundle at +/// [`WORK_RING_BUNDLE_OFFSETS`] (which requires the +/// [`NATIVE_SHARED_CLUSTER_SIZE`] cluster). The secondary leaves entries +/// 0..11 empty: it is a control-only peer with no work channels. +pub(crate) fn derive_channel_table( + role: InstanceRole, + state_grid: u64, + status_a: u64, + main_config: u64, + bundle: u64, +) -> [ChannelEntry; CHANNEL_TABLE_ENTRIES] { + let mut table = [ChannelEntry::EMPTY; CHANNEL_TABLE_ENTRIES]; + let grid_states = |index: usize| { + let base = state_grid + WORK_STATE_GRID_OFFSETS[index] as u64; + [ + base, + base + CHANNEL_STATE_SPACING as u64, + base + 2 * CHANNEL_STATE_SPACING as u64, + ] + }; + if role == InstanceRole::Primary { + for index in 0..WORK_CHANNEL_COUNT { + table[index] = ChannelEntry { + states: grid_states(index), + ring: bundle + WORK_RING_BUNDLE_OFFSETS[index] as u64, + }; + } + } + table[CONTROL_CHANNEL_INDEX] = ChannelEntry { + states: grid_states(CONTROL_CHANNEL_INDEX), + ring: main_config + MAIN_CONTROL_RING as u64, + }; + for (slot, index) in REPORT_CHANNEL_INDICES.iter().enumerate() { + let mut states = [0u64; 3]; + for (word, offset) in states.iter_mut().zip(REPORT_STATE_STATUS_A_OFFSETS[slot]) { + *word = status_a + offset as u64; + } + table[*index] = ChannelEntry { + states, + ring: status_a + REPORT_RING_STATUS_A_OFFSETS[slot] as u64, + }; + } + table[PARTIAL_CHANNEL_INDEX] = ChannelEntry { + states: [status_a + PARTIAL_STATE_STATUS_A_OFFSET as u64, 0, 0], + ring: 0, + }; + table +} + +fn validate_populated_entry(entry: &ChannelEntry) -> Result<(), InitdataError> { + // A populated channel needs all three state-counter addresses — a null + // producer (state 2) on a populated entry makes firmware fault on address + // zero — and a real ring. + for state in entry.states { + if state == 0 { + return Err(InitdataError::ChannelTableShape); + } + if !address_representable(state) { + return Err(InitdataError::AddressOutOfRange); + } + } + required_address(entry.ring) +} + +fn validate_channel_table( + role: InstanceRole, + main_config: u64, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], +) -> Result<(), InitdataError> { + for (index, entry) in channels.iter().enumerate() { + match index { + 0..=11 => match role { + InstanceRole::Primary => validate_populated_entry(entry)?, + // The secondary has no work channels at all. + InstanceRole::Secondary => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + }, + 12 => { + validate_populated_entry(entry)?; + // Device control is embedded in the main object, at exactly + // main + 0x4c0 on both instances. + if entry.ring != main_config + MAIN_CONTROL_RING as u64 { + return Err(InitdataError::ChannelTableShape); + } + } + 13 | 14 => validate_populated_entry(entry)?, + 15 => { + // Partial: a first state address and nothing else. + required_address(entry.states[0])?; + if entry.states[1] != 0 || entry.states[2] != 0 || entry.ring != 0 { + return Err(InitdataError::ChannelTableShape); + } + } + _ => { + if *entry != ChannelEntry::EMPTY { + return Err(InitdataError::ChannelTableShape); + } + } + } + } + Ok(()) +} + +/// The two complete M5/25G76 pre-handoff main configurations match the common +/// layout except primary +0x32c/+0x344 and secondary +0x300. Keep these +/// measured scalars build-specific; their meanings are not inferred here. +pub(crate) fn encode_main_config_for_profile( + profile: RootProfile, + role: InstanceRole, + shared: &SharedAddresses, + inst: &InstanceAddresses, + channels: &[ChannelEntry; CHANNEL_TABLE_ENTRIES], + region_views: Option<&PrimaryRegionViews>, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != MAIN_CONFIG_SIZE { + return Err(InitdataError::BufferSize); + } + validate_shared(shared)?; + validate_instance(role, inst)?; + validate_channel_table(role, inst.main_config, channels)?; + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + required_address(views.sentinel_high)?; + for high in views.alias_high { + required_address(high)?; + } + } + (InstanceRole::Secondary, None) => {} + _ => return Err(InitdataError::RoleSpecificAddress), + } + // Every view must fit inside the one bundle allocation. (The last two end + // exactly at 0xf000, inside the 0x10000 minimum.) The view table is the + // profile's: J700/J813 keep BUNDLE_VIEW_OFFSETS/EXTENTS unchanged. + let view_offsets = profile.bundle_view_offsets(); + for (offset, extent) in view_offsets.iter().zip(profile.bundle_view_extents()) { + if offset + extent > shared.hw_data_bundle_alloc { + return Err(InitdataError::BundleAllocationTooSmall); + } + } + + out.fill(0); + // Firmware log enable. The emitter at A000 0xfffffc0000021fa0 reads this + // word and returns immediately when it is zero; every buffer the facility + // needs is already allocated and pointed at by the addresses below. The + // records are plain ASCII. Shipping macOS provisions the whole facility + // and then throws the messages away (drainFirmwareLogRing is `bti c; ret`), + // which is why the captured value is zero. It stays zero here. + let bundle = shared.hw_data_bundle; + put_u64(out, MAIN_HW_DATA, bundle); + let repeated = bundle + profile.repeated_region_bundle_offset() as u64; + put_u64(out, MAIN_REPEATED_0, repeated); + put_u64(out, MAIN_REPEATED_1, repeated); + + for (index, entry) in channels.iter().enumerate() { + let base = MAIN_CHANNEL_TABLE + index * CHANNEL_ENTRY_SIZE; + for (word, state) in entry.states.iter().enumerate() { + put_u64(out, base + word * 8, *state); + } + put_u64(out, base + CHANNEL_ENTRY_STATE_COUNT * 8, entry.ring); + } + + match (role, region_views) { + (InstanceRole::Primary, Some(views)) => { + // The five bare pointers are views into the one bundle + // allocation, at fixed offsets. Unaligned stores at +0x254. + for (index, offset) in view_offsets.iter().enumerate() { + put_u64(out, MAIN_BUNDLE_VIEWS + index * 8, bundle + *offset as u64); + } + // High-only sentinel followed by two low/high region-view pairs. + put_u64(out, MAIN_REGION_VIEWS, views.sentinel_high); + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x10, views.alias_high[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u64(out, MAIN_REGION_VIEWS + 0x20, views.alias_high[1]); + if profile == RootProfile::J700G17P { + put_u32(out, MAIN_PRIMARY_SCALAR_32C, 1); + put_u32(out, MAIN_PRIMARY_SCALAR_344, 0xabcd_abcd); + } + // KEXT_PROVEN for G18G too (0xfffffe0008c31780: main+0x3e0=0xff). + put_u32(out, MAIN_PRIMARY_MASK, MAIN_PRIMARY_MASK_VALUE); + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + (InstanceRole::Secondary, _) => { + // The secondary keeps the two low context-0 values but carries no + // view or region addresses; the values are therefore not + // properties of any region the host allocates. + put_u64(out, MAIN_REGION_VIEWS + 0x08, REGION_VIEW_LOW_ADDRS[0]); + put_u64(out, MAIN_REGION_VIEWS + 0x18, REGION_VIEW_LOW_ADDRS[1]); + put_u32(out, MAIN_SECONDARY_KIND, match profile { + RootProfile::J700G17P => MAIN_SECONDARY_KIND_VALUE, + RootProfile::J813G17G25G76 => 0xc, + // INFERRED: 26A428 stores accel+0x10a74 here (25G76: + // accel+0xed30, measured 0xc on J813). Value not read on J873. + RootProfile::J873G18G26A428 => G18G_26A428_MAIN_SECONDARY_KIND, + }); + if profile == RootProfile::J873G18G26A428 { + // KEXT_PROVEN (26A428 initFirmwareSharedData, g18g_key.c:4464): + // role 1 main+0x481 = FW VA of the perf/policy allocation. + // 25G76 stored it at +0x471 and also wrote an accelerator + // pair at +0x479/+0x481, which 26A428 no longer writes. + put_u64(out, G18G_26A428_MAIN_SECONDARY_POLICY_PTR, + bundle + G18G_26A428_SHARED_LAYOUT.policy as u64); + } else { + // One additional unaligned pointer *into* primary view 2, landing + // five bytes before that view's first populated run. + let extra = bundle + + BUNDLE_VIEW_OFFSETS[SECONDARY_EXTRA_VIEW_INDEX] as u64 + + SECONDARY_EXTRA_VIEW_OFFSET as u64; + put_u64(out, MAIN_SECONDARY_EXTRA_PTR, extra); + } + put_u32(out, MAIN_CONTROL_RING, profile.control_opening_opcode(role)); + } + _ => unreachable!(), + } + Ok(()) +} + +/// Encode target-specific pre-handoff platform policy. M5's native policy is +/// kept separate from the experimental T8140 watchdog and idle overrides. +pub(crate) fn encode_region_c_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + if profile == RootProfile::J873G18G26A428 { + // Derived from the J813 bytes below through the proven +4 shift. + return encode_g18g_26a428_region_c(out); + } + out.fill(0); + write_runs(out, REGION_C_RUNS); + if profile == RootProfile::J813G17G25G76 { + // Scalars measured before each ASC received its root on two cold + // J813 / 25G76 boots. These are a scoped reference configuration, + // not generic M5 calibration or a power-management qualification. + // The final pair moved by 0x20; leaving the T8140 pair in place would + // initialize unrelated M5 fields. + for (offset, value) in [ + (0x084, 0x0000_5eb0), + (0x088, 0x0000_04b0), + (0x08c, 0x0000_04b0), + (0x098, 0x0000_04b0), + (0x7c4, 0x0000_04e2), + (0x7e0, 0x3f33_3333), + (0x9a8, 0x0000_0333), + (0xe38, 0), + (0xe3c, 0), + (0xe58, 0x0000_0300), + (0xe5c, 0x0000_0100), + ] { + put_u32(out, offset, value); + } + return Ok(()); + } + // regionC+0x9cc is the KSM progress-watchdog threshold for data master 2 + // (compute); the firmware copies it to 0x1778d0 and its handler starts + // `cbz w20 -> ret`, so 0 disables the watchdog. We ship 0x32 (50) here and + // leave the other data master's slot at 0 -- i.e. we arm the watchdog on + // exactly the queue we submit to. When work cannot progress for any other + // reason this turns a stall into a self-sustaining recovery loop. + for (offset, value) in [ + (0x7c, 1), + (0x80, 1), + (0x9c, 0x3000_001d), + (0xa0, 0x3100_0002), + ] { + put_u32(out, offset, value); + } + Ok(()) +} + +/// Encode one 0x80-byte status block in its pre-init state: u32 1 at `+0x04`, +/// everything else zero. `+0x10`/`+0x14` are post-acknowledgement firmware +/// state and must be zero at handoff. +pub(crate) fn encode_status_block(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + put_u32(out, STATUS_BLOCK_ONE_OFFSET, 1); + Ok(()) +} + +pub(crate) fn encode_secondary_root_extra_1_for_profile( + profile: RootProfile, + out: &mut [u8], +) -> Result<(), InitdataError> { + if out.len() != STATUS_BLOCK_SIZE { + return Err(InitdataError::BufferSize); + } + out.fill(0); + write_runs(out, SECONDARY_ROOT_EXTRA_1_RUNS); + // G18G: KEXT_PROVEN identical store geometry for this 0x60 object + // (26A428 vs 25G76 R_60 stores, all at the same offsets); the J813 values + // are carried unchanged (INFERRED, listed in G18G_26A428_UNPROVEN). + if matches!(profile, RootProfile::J813G17G25G76 | RootProfile::J873G18G26A428) { + put_u32(out, 0x10, 2); + } + Ok(()) +} + +/// Recorded T8140 region-C platform tuning constants (values known, meanings +/// not; generated from the hardware-validated `g17p_initdata.py` +/// `REGION_C_CONSTANTS`). 12 sparse runs, 180 bytes total. The final run +/// initializes the byte preceding the host power-assert counter and the +/// counter itself to their native cold value of zero. +pub(crate) const REGION_C_RUNS: &[(usize, &[u8])] = &[ + (0x0024, &[0xb8, 0x0b, 0x00, 0x00]), + ( + 0x0030, + &[ + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x0054, + &[ + 0xff, 0xff, 0x28, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, + ], + ), + (0x0078, &[0x01, 0x00, 0x00, 0x00]), + ( + 0x0084, + &[ + 0x4e, 0x25, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, + ], + ), + (0x0098, &[0xe8, 0x03, 0x00, 0x00]), + ( + 0x07c4, + &[ + 0xd0, 0x07, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0xcd, 0xcc, 0x4c, 0x3f, 0xcd, 0xcc, + 0x4c, 0x3e, 0x66, 0x66, 0x66, 0x3f, 0xcd, 0xcc, 0xcc, 0x3d, 0x00, 0x00, 0x80, 0x3e, + 0x9a, 0x99, 0x19, 0x3f, 0x66, 0x66, 0x66, 0x3f, 0x06, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, + ], + ), + ( + 0x0998, + &[ + 0x28, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, + ], + ), + ( + 0x09b8, + &[ + 0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + ], + ), + (0x0e1c, &[0x00, 0x01, 0x00, 0x00]), + (0x0e38, &[0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00]), + (0x0e50, &[0x00, 0x00, 0x00, 0x00]), +]; + +pub(crate) const SECONDARY_ROOT_EXTRA_1_RUNS: &[(usize, &[u8])] = &[ + (0x14, &[0x01]), + (0x2c, &[0x01]), + (0x30, &[0x01]), + (0x40, &[0x01]), + (0x4c, &[0x01]), + (0x50, &[0x6a, 0x18]), +]; + +/// Root `+0x00`: u16 {0x04c0, 0x83a0, 0x4339, 0x0c8b}. FW_PROVEN: b000 +/// 0x2164c-0x21660 and b010 0x294f0-0x29504 compare root+0 with this word and +/// abort with "Host and FW Interface is mismatched" otherwise. KEXT_PROVEN: +/// AGXArmFirmware::initFirmwareData (0xfffffe0008c31f6c) writes it to both +/// roots. +pub(crate) const G18G_26A428_INITDATA_VERSION_WORD: u64 = 0x0c8b_4339_83a0_04c0; + +/// Both roots are 0xc8 bytes. FW_PROVEN: each image copies 0xc8 bytes of its +/// root (b000 0x21664-0x216a8) and requires root+0x2c != 0 (b000 0x216ac). +/// Root field offsets are unchanged from 25G76 (KEXT_PROVEN g18g_key.c:3748). +pub(crate) const G18G_26A428_ROOT_SIZE: usize = 0xc8; + +/// `AGXFirmware::allocFirmwareData` allocation-size table, 24 u64. +/// KEXT_PROVEN: 25G76 __const 0xfffffe0007139820. Index names are the +/// `KX_*` constants below. +pub(crate) const KEXT_ALLOCATION_SIZES_25G76: [usize; 24] = [ + 0x2708, 0x0c18, 0x1048, 0x0e10, 0x6a30, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe440, 0x06a8, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e78, 0x00e4, +]; + +/// KEXT_PROVEN: 26A428 __const 0xfffffe000714f0d8. Only entries 0 (hw_data), +/// 4 (perf/policy), 12 (status B), 13 (status config) and 22 (region C) +/// differ from 25G76. +pub(crate) const KEXT_ALLOCATION_SIZES_26A428: [usize; 24] = [ + 0x2848, 0x0c18, 0x1048, 0x0e10, 0x7800, 0x0068, 0x0800, 0x0088, 0x1800, 0x04c0, 0x4000, 0x0030, + 0xe730, 0x06d0, 0x0020, 0x01b0, 0x4800, 0x28800, 0x79800, 0x9000, 0x0060, 0x0008, 0x0e80, 0x00e4, +]; + +pub(crate) const KX_HW_DATA: usize = 0; + +pub(crate) const KX_BLOCK_C18: usize = 1; + +pub(crate) const KX_BLOCK_1048: usize = 2; + +pub(crate) const KX_BLOCK_E10: usize = 3; + +pub(crate) const KX_POLICY: usize = 4; + +pub(crate) const KX_BLOCK_68: usize = 5; + +pub(crate) const KX_BLOCK_800: usize = 6; + +pub(crate) const KX_BLOCK_88: usize = 7; + +pub(crate) const KX_DM_RING: usize = 8; + +pub(crate) const KX_MAIN_CONFIG: usize = 9; + +pub(crate) const KX_CONTROL_RING: usize = 10; + +pub(crate) const KX_REGION_C: usize = 22; + +/// `getSizeOfFWBRNTable` returns 0 in both kexts (KEXT_PROVEN +/// 0xfffffe00089ee74c / 0xfffffe0008c507e4). +pub(crate) const KEXT_BRN_TABLE_SIZE: usize = 0; + +/// Both shared allocations are rounded to one 16 KiB page (allocateSharedData). +pub(crate) const KEXT_ALLOCATION_PAGE: usize = 0x4000; + +pub(crate) const SHARED_ALLOCATION_ENTRIES: usize = 25; + +const fn round_up_usize(value: usize, alignment: usize) -> usize { + (value + alignment - 1) & !(alignment - 1) +} + +/// The allocateSharedData packing rule (KEXT_PROVEN 0xfffffe0008c038d8): +/// each next entry starts at `(previous offset + previous size + 0x3f) & !0x3f`. +pub(crate) const fn next_packed_allocation(offset: usize, size: usize) -> usize { + round_up_usize(offset + size, 0x40) +} + +/// Offsets of `sizes` packed by [`next_packed_allocation`], the first at 0. +pub(crate) const fn packed_allocation_offsets(sizes: &[usize; N]) -> [usize; N] { + let mut out = [0usize; N]; + let mut index = 1; + while index < N { + out[index] = next_packed_allocation(out[index - 1], sizes[index - 1]); + index += 1; + } + out +} + +/// The 25 entries of shared allocation #1 in allocation order: hw_data, the +/// 0xc18/0x1048/0xe10 blocks, perf/policy, the 0x68/0x800 blocks, the BRN +/// table, the 0x88 block, twelve data-master rings, then main config and +/// device-control ring for role 0 and for role 1 (KEXT_PROVEN descriptor +/// build, 25G76 g17g_key.c:1224-1310 / 26A428 g18g_key.c:1236-1320). +pub(crate) const fn shared_allocation_sizes(table: &[usize; 24]) -> [usize; SHARED_ALLOCATION_ENTRIES] { + let mut sizes = [0usize; SHARED_ALLOCATION_ENTRIES]; + sizes[0] = table[KX_HW_DATA]; + sizes[1] = table[KX_BLOCK_C18]; + sizes[2] = table[KX_BLOCK_1048]; + sizes[3] = table[KX_BLOCK_E10]; + sizes[4] = table[KX_POLICY]; + sizes[5] = table[KX_BLOCK_68]; + sizes[6] = table[KX_BLOCK_800]; + sizes[7] = KEXT_BRN_TABLE_SIZE; + sizes[8] = table[KX_BLOCK_88]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + sizes[9 + ring] = table[KX_DM_RING]; + ring += 1; + } + sizes[21] = table[KX_MAIN_CONFIG]; + sizes[22] = table[KX_CONTROL_RING]; + sizes[23] = table[KX_MAIN_CONFIG]; + sizes[24] = table[KX_CONTROL_RING]; + sizes +} + +/// Offsets (from the start of the shared allocation #1 backing) of every +/// object in it. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct SharedClusterLayout { + pub(crate) hw_data: usize, + pub(crate) block_c18: usize, + pub(crate) block_1048: usize, + pub(crate) block_e10: usize, + pub(crate) policy: usize, + pub(crate) block_68: usize, + pub(crate) block_800: usize, + pub(crate) brn_table: usize, + pub(crate) block_88: usize, + /// Data-master rings in allocation order (see [`WORK_CHANNEL_DM_INDEX`]). + pub(crate) dm_rings: [usize; WORK_CHANNEL_COUNT], + pub(crate) main_config: [usize; 2], + pub(crate) control_ring: [usize; 2], + pub(crate) end: usize, + pub(crate) total: usize, +} + +pub(crate) const fn shared_cluster_layout(table: &[usize; 24]) -> SharedClusterLayout { + let sizes = shared_allocation_sizes(table); + let offsets = packed_allocation_offsets(&sizes); + let mut dm_rings = [0usize; WORK_CHANNEL_COUNT]; + let mut ring = 0; + while ring < WORK_CHANNEL_COUNT { + dm_rings[ring] = offsets[9 + ring]; + ring += 1; + } + let end = offsets[24] + sizes[24]; + SharedClusterLayout { + hw_data: offsets[0], + block_c18: offsets[1], + block_1048: offsets[2], + block_e10: offsets[3], + policy: offsets[4], + block_68: offsets[5], + block_800: offsets[6], + brn_table: offsets[7], + block_88: offsets[8], + dm_rings, + main_config: [offsets[21], offsets[23]], + control_ring: [offsets[22], offsets[24]], + end, + total: round_up_usize(end, KEXT_ALLOCATION_PAGE), + } +} + +/// The 26A428 layouts: hw_data 0x2848 at 0, blocks at 0x2880/0x34c0/0x4540, +/// perf/policy 0x7800 at 0x5380, 0x68/0x800 at 0xcb80/0xcc00, BRN and 0x88 at +/// 0xd400, rings from 0xd4c0, main r0 0x1f4c0, control r0 0x1f980, main r1 +/// 0x23980, control r1 0x23e40; status B 0x340, status config 0xea80, status +/// A 0xf180/0xc3440, tail 0x177700/0x177740/0x1777c0. Totals 0x28000 and +/// 0x178000, as in 25G76. +pub(crate) const G18G_26A428_SHARED_LAYOUT: SharedClusterLayout = + shared_cluster_layout(&KEXT_ALLOCATION_SIZES_26A428); + +/// Main `+0x254` views (KEXT_PROVEN 26A428 initFirmwareSharedData: the FW VAs +/// of the 0xc18, 0x1048, 0xe10, 0x68 and 0x800 blocks, this+0x310..0x338). +pub(crate) const G18G_26A428_BUNDLE_VIEW_OFFSETS: [usize; 5] = [ + G18G_26A428_SHARED_LAYOUT.block_c18, + G18G_26A428_SHARED_LAYOUT.block_1048, + G18G_26A428_SHARED_LAYOUT.block_e10, + G18G_26A428_SHARED_LAYOUT.block_68, + G18G_26A428_SHARED_LAYOUT.block_800, +]; + +pub(crate) const G18G_26A428_BUNDLE_VIEW_EXTENTS: [usize; 5] = [ + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_C18], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_1048], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_E10], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_68], + KEXT_ALLOCATION_SIZES_26A428[KX_BLOCK_800], +]; + +/// Role-1 main `+0x481`: FW VA of the perf/policy allocation (KEXT_PROVEN). +pub(crate) const G18G_26A428_MAIN_SECONDARY_POLICY_PTR: usize = 0x481; + +/// Role-1 main `+0x300`. INFERRED: the J813 value (26A428 source accel+0x10a74). +pub(crate) const G18G_26A428_MAIN_SECONDARY_KIND: u32 = 0xc; + +/// J873 ADT gpu-num-perf-states. ADT_PROVEN. +pub(crate) const J873_GPU_NUM_PERF_STATES: u32 = 13; + +/// Region-C ADT tunables. KEXT_PROVEN: +0x7c8 = accel+0x106b0 +/// (gpu-idleoff-standby-timer; J813 +0x7c4 carried its ADT 1250) and +0x7e4 = +/// accel+0x106cc (gpu-idleoff-min_conf); ADT_PROVEN values 1500 and 0.8f. +/// +0x9c = this+0xf14 * 100 (25G76 +0x98 = this+0xf40 * 100 = 1200 on J813): +/// INFERRED to be the perf-state count again. +pub(crate) const G18G_26A428_REGION_C_ADT_WORDS: [(usize, u32); 3] = [ + (0x7c8, 1500), + (0x7e4, 0x3f4c_cccd), + (0x09c, J873_GPU_NUM_PERF_STATES * 100), +]; + +/// Stable region-C inputs from the same eight initialization snapshots. +/// +0x38 is the new 26A428 feature bit, no longer a guessed zero. +pub(crate) const G18G_26A428_REGION_C_CAPTURED_WORDS: [(usize, u32); 7] = [ + (0x0038, 1), (0x0088, 0x00008f16), (0x008c, 0x00000514), + (0x0090, 0x00000514), (0x09b0, 0x00000200), + (0x09dc, 0x00000032), (0x0e0c, 0x000000fa), +]; + +/// Encode the G18G region C: the J813 bytes, the 25G76 fields from +0x38 on +/// moved up by 4 (FW_PROVEN), the J873 ADT tunables and stable native J873 +/// inputs. The new +0x38 feature bit (accel+0x6b9 bit 2) is captured as one. +pub(crate) fn encode_g18g_26a428_region_c(out: &mut [u8]) -> Result<(), InitdataError> { + if out.len() != REGION_C_SIZE { + return Err(InitdataError::BufferSize); + } + encode_region_c_for_profile(RootProfile::J813G17G25G76, out)?; + // The J813 object never reaches its 0xe78 end; shifting in place keeps the + // tail zero and needs no second 4 KiB buffer on the kernel stack. + let shifted = KEXT_ALLOCATION_SIZES_25G76[KX_REGION_C]; + if out[shifted..].iter().any(|byte| *byte != 0) { + return Err(InitdataError::BufferSize); + } + out.copy_within(0x38..shifted, 0x3c); + out[0x38..0x3c].fill(0); + for (offset, value) in G18G_26A428_REGION_C_ADT_WORDS { + put_u32(out, offset, value); + } + for (offset, value) in G18G_26A428_REGION_C_CAPTURED_WORDS { + put_u32(out, offset, value); + } + // Idle power-off policy word (Region C +0x30). The firmware loads + // it on every cold wake, which undoes the IdlePowerOff(0) sent once at boot. + // 0 keeps the scheduler idle event from powering the cores down between + // jobs. Measured on the M6: 8.6 percent less time per op. + put_u32(out, 0x30, 0); + Ok(()) +} + +/// FW_PROVEN (jump table 0xfffffc0000006c60, 56 entries from opcode 4): the +/// opening/region-C reload handler (b000 0x3ed0) is reached by 0x15, 0x2a +/// and 0x30. +pub(crate) const G18G_26A428_CONTROL_OPENING_OPCODE_PRIMARY: u32 = 0x15; + +pub(crate) const G18G_26A428_CONTROL_OPENING_OPCODE_SECONDARY: u32 = 0x2a; diff --git a/drivers/gpu/drm/asahi/m5/live_boot.rs b/drivers/gpu/drm/asahi/m5/live_boot.rs new file mode 100644 index 0000000000000..89bf0b08c7124 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/live_boot.rs @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M5-specific initialization and retained application submission session. +//! DRM admission remains in the device adapter; no G17P manager is selected. + +use super::provider::{validate_provider, ExpectedProvider, G17PAscwrapV6Role, + G17PMailboxProvider, G17PNoRtkitBuffer}; +use crate::{driver::AsahiDrmDevice, m5::resources::M5FirmwareMemory, + m5::fw_msg, mmu, regs}; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use crate::m5::completion::{graphics_pointer_head, MAX_GRAPHICS_GENERATION}; + +/// Application jobs are polled for retirement. A 1 ms step put up to a +/// millisecond of idle GPU after every job, several per video frame. +/// (The first render job keeps 1 ms steps; see submit_application.) +const APPLICATION_POLL_US: i64 = 50; +const APPLICATION_POLLS_PER_MS: u32 = 1000 / APPLICATION_POLL_US as u32; +use kernel::{device::Core, macros::vtable, platform, prelude::*, + soc::apple::rtkit, sync::{aref::ARef, Arc}, time::{delay::fsleep, Delta}}; + +struct M5RoleState { + dev: ARef, + role: usize, + acked: AtomicBool, + crashed: AtomicBool, + events: AtomicU32, +} + +struct M5RtkitOps; + +#[vtable] +impl rtkit::Operations for M5RtkitOps { + type Data = Arc; + type Buffer = G17PNoRtkitBuffer; + + fn recv_message_early( + data: ::Borrowed<'_>, + endpoint: u8, + message: u64, + ) -> bool { + // Both native M5 roles reply with exactly 0x0009000000000000. + if endpoint == 0x20 && message == 0x0009_0000_0000_0000 { + data.acked.store(true, Ordering::Release); + true + } else { + false + } + } + + fn recv_message( + data: ::Borrowed<'_>, + endpoint: u8, + message: u64, + ) { + let ordinal = data.events.fetch_add(1, Ordering::Relaxed); + if ordinal < 8 { + dev_dbg!(data.dev.as_ref(), "M5 message role={} ep={:#x} payload={:#x}\n", + data.role, endpoint, message); + } + } + + fn crashed( + data: ::Borrowed<'_>, + _crashlog: Option<&[u8]>, + ) { + data.crashed.store(true, Ordering::Release); + dev_err!(data.dev.as_ref(), "M5 firmware crashed role={}\n", data.role); + } +} + +struct M5Role { + provider: G17PMailboxProvider, + rtkit: rtkit::RtKit, + state: Arc, + started: bool, +} + +struct M5Session { + roles: [M5Role; 2], + memory: Option>, + uat: Arc, + /// The GPU mappings of both firmware crash log buffers. + _crash_views: [mmu::KernelMapping; 2], + registers: regs::G17Resources, + dev: ARef, +} + +impl M5Session { + fn stop_cpus(&mut self) -> Result { + let mut failure = None; + // Reverse the measured M5 GFX1/GFX start order. + for index in [0, 1] { + if self.roles[index].started { + match self.roles[index].provider.stop_cpu() { + Ok(()) => self.roles[index].started = false, + Err(error) => failure = Some(error), + } + } + } + failure.map_or(Ok(()), Err) + } + + fn initialize(&mut self) -> Result { + self.registers.m5_match_pmgr_boot_control()?; + for index in [1, 0] { + let role = &mut self.roles[index]; + role.provider.start_cpu()?; + role.started = true; + Pin::new(&mut role.rtkit).boot()?; + dev_dbg!(self.dev.as_ref(), "M5_INIT role={} rtkit-ready\n", index); + } + // These helpers operate only on the proven shared dual-root layout; + // the Neo-specific contexts, metrics and manager remain unselected. + let roots = self.uat.adopt_firmware_root_entries()?; + let timestamp: u64; + // SAFETY: Read-only architectural counter, available on this target. + unsafe { core::arch::asm!("mrs {value}, cntvct_el0", value = out(reg) timestamp, + options(nomem, nostack, preserves_flags)); } + self.memory = Some(M5FirmwareMemory::new_with_uat( + &self.dev, self.uat.clone(), timestamp)?); + self.uat.confirm_firmware_root_entries(roots)?; + self.uat.mirror_secondary_root()?; + dev_dbg!(self.dev.as_ref(), "M5_INIT retained memory graph mapped\n"); + self.registers.enable_g17_gating()?; + + // M5's own trace publishes and ACKs GFX1 first (133/137), then GFX + // (160/164). The Neo's opposite ordering is not used here. + for index in [1, 0] { + let address = self.memory.as_ref().ok_or(EIO)?.root(index == 1); + let message = fw_msg::encode_initdata_doorbell(address).map_err(|_| EINVAL)?; + let role = &mut self.roles[index]; + Pin::new(&mut role.rtkit).start_endpoint(0x20)?; + Pin::new(&mut role.rtkit).start_endpoint(0x21)?; + Pin::new(&mut role.rtkit).send_message(0x20, message)?; + dev_dbg!(self.dev.as_ref(), "M5_INIT role={} published root={:#x}\n", index, address); + for _ in 0..200 { + if self.memory.as_mut().ok_or(EIO)?.drain_reports(&self.dev)? { + return Err(EIO); + } + if role.state.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if role.state.acked.load(Ordering::Acquire) { + break; + } + fsleep(Delta::from_millis(10)); + } + if !role.state.acked.load(Ordering::Acquire) { + return Err(ETIMEDOUT); + } + dev_dbg!(self.dev.as_ref(), "M5_INIT role={} initdata-ack\n", index); + } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + self.retire_openings() + } + + fn install_graphics_queues(&mut self) -> Result { + let memory = self.memory.as_mut().ok_or(EIO)?; + dev_dbg!(self.dev.as_ref(), "M5_GRAPHICS install before counters={:x?} registration={:x?}\n", + memory.graphics_status()?, memory.graphics_installation_status()?); + memory.publish_graphics_install()?; + Pin::new(&mut self.roles[0].rtkit).send_message(0x21, 0x0083_0000_0000_000a)?; + for poll in 0..1000 { + if memory.drain_reports(&self.dev)? { return Err(EIO); } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + let (outer, inner, _) = memory.graphics_status()?; + if outer == [2; 3] && inner == [[1; 3]; 2] + && memory.graphics_queues_installed()? { + dev_dbg!(self.dev.as_ref(), "M5_GRAPHICS installed TA/fragment queues; registration={:x?}; no draws published\n", + memory.graphics_installation_status()?); + return Ok(()); + } + if [0, 100].contains(&poll) { + dev_dbg!(self.dev.as_ref(), "M5_GRAPHICS install poll={} counters={:x?} registration={:x?}\n", + poll, memory.graphics_status()?, memory.graphics_installation_status()?); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.dev.as_ref(), "M5_GRAPHICS install timeout counters={:x?} registration={:x?}\n", + memory.graphics_status()?, memory.graphics_installation_status()?); + Err(ETIMEDOUT) + } + + fn retire_idle_policy(&mut self) -> Result { + let memory = self.memory.as_mut().ok_or(EIO)?; + // Retire the idle-policy request. + memory.publish_idle_wake()?; + Pin::new(&mut self.roles[0].rtkit).send_message(0x21, 0x0084_0000_0000_0011)?; + for _ in 0..1000 { + if memory.drain_reports(&self.dev)? { return Err(EIO); } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + dev_err!(self.dev.as_ref(), "M5: firmware crashed retiring the idle policy\n"); + return Err(EIO); + } + if memory.control_counters()? == [[2; 3], [1; 3]] { + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.dev.as_ref(), "M5: idle policy not retired, counters={:?}\n", + memory.control_counters()?); + Err(ETIMEDOUT) + } + + fn retire_openings(&mut self) -> Result { + let mut counters = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + dev_dbg!(self.dev.as_ref(), "M5_OPEN before power-transition-end {:?}\n", counters); + // M5's own cold trace sends one 0x89 to GFX, then GFX1 (events + // 167/171), immediately after the two initdata ACKs. No other control + // record or work item is published in this bounded experiment. + for index in [0, 1] { + Pin::new(&mut self.roles[index].rtkit) + .send_message(0x21, fw_msg::encode_power_transition_end())?; + } + for _ in 0..1000 { + if self.memory.as_mut().ok_or(EIO)?.drain_reports(&self.dev)? { + return Err(EIO); + } + if self.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + counters = self.memory.as_mut().ok_or(EIO)?.control_counters()?; + if counters == [[1; 3]; 2] { + dev_dbg!(self.dev.as_ref(), "M5_OPEN both native control openings retired {:?}\n", counters); + return Ok(()); + } + fsleep(Delta::from_millis(1)); + } + dev_err!(self.dev.as_ref(), "M5_OPEN control openings did not retire {:?}\n", counters); + Err(ETIMEDOUT) + } + +} + +/// Sizes of the GFX and GFX1 `__DATA` segments the 25G76 firmware runs with. +const DATA_SIZES: [u64; 2] = [0x134000, 0x130000]; +/// The firmware's crash log buffer, at this offset of each `__DATA` segment. +const CRASH_LOG_OFFSET: u64 = 0xa0000; +const CRASH_LOG_SIZE: usize = 0x8000; + +/// Creates the UAT both firmware instances share, as the A18 Pro's: each instance publishes +/// its own kernel root, the secondary one 0x40000 into `pagetables`. Each instance asks for +/// its crash log at the GPU address of `__DATA` + 0xa0000 truncated to 40 bits, which is +/// mapped here, before either instance starts. +fn new_uat(dev: &AsahiDrmDevice) -> Result<(Arc, [mmu::KernelMapping; 2])> { + let node = dev.as_ref().as_ref().of_node().ok_or(EINVAL)?; + let mut crash_logs = [0u64; 2]; + for (index, name) in [c"gfx-data", c"gfx1-data"].into_iter().enumerate() { + let region = node.reserved_mem_region_to_resource_byname(name)?; + if region.size() != DATA_SIZES[index] || region.start() & (mmu::UAT_PGSZ as u64 - 1) != 0 + { + return Err(EINVAL); + } + crash_logs[index] = region.start() + CRASH_LOG_OFFSET; + } + let uat = Arc::new( + mmu::Uat::new( + dev.into(), + mmu::UatGeometry { ias: 42, oas: 42 }, + false, + mmu::UatFirmware::PublishedRoots { secondary_root_offset: Some(0x40000) }, + mmu::UatQuirks::DEFAULT, + )?, + GFP_KERNEL, + )?; + let map = |pa: u64| { + uat.kernel_lower_vm().map_io( + dev.as_ref().as_ref(), + pa & ((1u64 << 40) - 1), + pa.try_into()?, + CRASH_LOG_SIZE, + mmu::PROT_FW_SHARED_RW, + ) + }; + let views = [map(crash_logs[0])?, map(crash_logs[1])?]; + Ok((uat, views)) +} + +pub(crate) struct M5Runtime { + session: Option>, + application: bool, + faulted: bool, + application_generation: u32, + application_compute_generation: u32, +} + +impl M5Runtime { + fn prepare( + pdev: &platform::Device>, + dev: ARef, + registers: regs::G17Resources, + ) -> Result { + let gfx = G17PMailboxProvider::get(pdev.as_ref(), 0)?; + let gfx1 = G17PMailboxProvider::get(pdev.as_ref(), 1)?; + for (provider, expected) in [ + (&gfx, ExpectedProvider { role: G17PAscwrapV6Role::Gfx, + wrapper: (0x582600000, 0x88000), iop_vbar: (0x582050000, 8) }), + (&gfx1, ExpectedProvider { role: G17PAscwrapV6Role::Gfx1, + wrapper: (0x582e00000, 0x88000), iop_vbar: (0x582850000, 8) }), + ] { + validate_provider(provider, expected)?; + provider.require_safe_cpu_lifecycle()?; + } + let (uat, crash_views) = new_uat(&dev)?; + let make_role = |index, provider| -> Result { + let state = Arc::new(M5RoleState { dev: dev.clone(), role: index, + acked: AtomicBool::new(false), crashed: AtomicBool::new(false), + events: AtomicU32::new(0) }, GFP_KERNEL)?; + let rtkit = rtkit::RtKit::::new_with_firmware_crashlog( + dev.as_ref().as_ref(), None, index, state.clone())?; + Ok(M5Role { provider, rtkit, state, started: false }) + }; + let session = KBox::new(M5Session { + roles: [make_role(0, gfx)?, make_role(1, gfx1)?], + memory: None, uat, _crash_views: crash_views, registers, dev, + }, GFP_KERNEL)?; + // From this point every path returns the retained platform owner, + // even when firmware rejects the graph. Failed probe unwinding must + // never release UAT or transports beneath a running firmware CPU. + // Keep callback code resident even if a failed stop retains owners. + // This explicit RAM-boot experiment is released by reboot only. + unsafe { kernel::bindings::__module_get((*kernel::module::this_module::()).as_ptr()); } + Ok(Self { session: Some(session), application: false, faulted: false, + application_generation: 0, application_compute_generation: 0 }) + } + + /// Boot without a native program or job graph and leave both processors + /// running for explicit application submissions. This does not register + /// a render node or claim that Mesa command translation is complete. + pub(crate) fn start_application( + pdev: &platform::Device>, dev: ARef, registers: regs::G17Resources, + ) -> Result { + // Periodic power/submission traces need continuous draining while + // the session idles. Recovery bit 9 emits only on a fault, which the + // synchronous submit loop drains before retaining a failed graph. + if crate::module_parameters::m5_fw_trace.value() & !0x200 != 0 { + return Err(EINVAL); + } + let mut owner = Self::prepare(pdev, dev, registers)?; + owner.application = true; + let result = (|| { + let session = owner.session.as_mut().ok_or(EIO)?; + session.initialize()?; + session.retire_idle_policy()?; + Ok(()) + })(); + if let Err(error) = result { + owner.faulted = true; + return Err(error); + } + dev_info!(pdev.as_ref(), "M5: GPU firmware 25G76 running\n"); + Ok(owner) + } + + pub(crate) fn new_user_vm(&self, id: u64, kernel_range: core::ops::Range) -> Result { + if !self.application || self.is_crashed() { return Err(ENODEV); } + crate::m5::vm_layout::compact_range(&kernel_range).ok_or(EINVAL)?; + self.session.as_ref().ok_or(EIO)?.uat.new_vm(id, kernel_range)?.with_status() + } + + pub(crate) fn is_crashed(&self) -> bool { + self.faulted || self.session.as_ref().is_none_or(|session| + session.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire))) + } + + /// The adapter serializes this call and validates the translated command + /// plus every user mapping before entry. Return success only after paired + /// queue retirement, scheduler unlink, stamps and firmware notifications. + /// A failed published job retains and quarantines its VM until reboot. + #[allow(clippy::too_many_arguments)] + pub(crate) fn submit_application( + &mut self, vm: &mmu::Vm, parameters: crate::m5::render_desc::G17pRenderParameters, + controls: crate::m5::render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], + ) -> Result { + if !self.application || self.is_crashed() { return Err(ENODEV); } + // Reuse retired ring storage while keeping completion stamps unique. + if self.application_generation >= MAX_GRAPHICS_GENERATION { return Err(EBUSY); } + let generation = self.application_generation + 1; + let session = self.session.as_deref_mut().ok_or(EIO)?; + if generation == 1 { + session.memory.as_mut().ok_or(EIO)?.initialize_application_graphics( + &session.dev, vm, parameters, controls, clusters, usc_required)?; + } + // From installation onward, even a failed mailbox send can have made + // memory visible to firmware. Never unwind or recycle the job graph. + self.application_generation = generation; + let result = (|| { + if generation == 1 { session.install_graphics_queues()?; } + let memory = session.memory.as_mut().ok_or(EIO)?; + if generation == 1 { + memory.publish_graphics_job()?; + } else { + memory.publish_next_application_job(&session.dev, vm, parameters, controls, + clusters, usc_required, generation)?; + } + Pin::new(&mut session.roles[0].rtkit).send_message(0x21, 0x0083_0000_0000_000a)?; + for _ in 0..10000 * APPLICATION_POLLS_PER_MS { + if memory.drain_reports(&session.dev)? + || session.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + let (outer, inner, _) = memory.graphics_status()?; + let completion = memory.graphics_completion()?; + if outer == [memory.application_outer_head(); 3] + && inner == [[graphics_pointer_head(generation); 3]; 2] + && M5FirmwareMemory::graphics_pair_complete(completion, generation) { + dev_dbg!(session.dev.as_ref(), + "M5_APPLICATION generation={} completed; both queues retired\n", + generation); + return Ok(completion); + } + // The first job also installs the queues; polling it this + // often met a firmware recovery on its fragment queue. + fsleep(Delta::from_micros(if generation == 1 { 1000 } else { APPLICATION_POLL_US })); + } + Err(ETIMEDOUT) + })(); + if let Err(error) = result { + if let Some(memory) = session.memory.as_mut() { + for _ in 0..4 { let _ = memory.drain_reports(&session.dev); } + dev_err!(session.dev.as_ref(), + "M5_APPLICATION failed graph counters={:x?} completion={:x?}\n", + memory.graphics_status(), memory.graphics_completion()); + } + self.faulted = true; + if let Some(status) = vm.status() { status.record(error); } + if let Some(memory) = &session.memory { memory.quarantine_graphics(); } + // A successful stop request alone does not prove a halt. Keep + // the full session even after this best-effort stop succeeds. + let stopped = session.stop_cpus(); + dev_err!(session.dev.as_ref(), + "M5_APPLICATION submission failed {:?}; stop={:?}; VM and firmware owners retained\n", + error, stopped); + } + result + } + + /// Execute a compute command in the already retained application VM. + /// Installation, dispatch and retirement are separate firmware transitions. + /// The first submission installs queue 12 and dispatches generation 1; + /// every later one reuses that owner and dispatches the next generation, + /// as submit_application() does for the graphics pair. + pub(crate) fn submit_application_compute(&mut self, vm: &mmu::Vm, + command: &crate::m5::command::TranslatedComputeCommand) -> Result { + if !self.application || self.is_crashed() || self.application_generation == 0 { + return Err(ENOTSUPP); + } + if self.application_compute_generation >= MAX_GRAPHICS_GENERATION { + return Err(EBUSY); + } + let generation = self.application_compute_generation + 1; + let session = self.session.as_deref_mut().ok_or(EIO)?; + if generation == 1 { + session.memory.as_mut().ok_or(EIO)?.initialize_application_compute( + &session.dev, vm, command, self.application_generation)?; + } else { + session.memory.as_mut().ok_or(EIO)?.prepare_next_application_compute( + &session.dev, vm, command, generation)?; + } + // The owner is retained from installation onward. Even a failed mailbox + // send can follow firmware seeing the new producer, so never unwind. + self.application_compute_generation = generation; + let result = (|| { + let memory = session.memory.as_mut().ok_or(EIO)?; + // Installation is published once; the dispatch record every time. + let stages: &[bool] = if generation == 1 { &[false, true] } else { &[true] }; + for &dispatch in stages { + memory.publish_application_compute(dispatch, generation)?; + Pin::new(&mut session.roles[0].rtkit) + .send_message(0x21, 0x0083_0000_0000_000a)?; + let mut retired = false; + for _ in 0..if dispatch { 10000 } else { 1000 } * APPLICATION_POLLS_PER_MS { + if memory.drain_reports(&session.dev)? + || session.roles.iter().any(|role| role.state.crashed.load(Ordering::Acquire)) { + return Err(EIO); + } + let (outer, _, _) = memory.graphics_status()?; + if outer == [memory.application_outer_head(); 3] + && if dispatch { memory.application_compute_complete(generation)? } + else { memory.application_compute_installed()? } { + retired = true; + break; + } + fsleep(Delta::from_micros(APPLICATION_POLL_US)); + } + if !retired { return Err(ETIMEDOUT); } + dev_dbg!(session.dev.as_ref(), + "M5_APPLICATION_COMPUTE {} generation={} queue=12 outer={} status={:x?}\n", + if dispatch { "completed" } else { "installed" }, generation, + memory.application_outer_head(), memory.application_compute_status()); + } + Ok(()) + })(); + if let Err(error) = result { + self.faulted = true; + if let Some(status) = vm.status() { status.record(error); } + if let Some(memory) = session.memory.as_mut() { + for _ in 0..4 { let _ = memory.drain_reports(&session.dev); } + memory.quarantine_graphics(); + dev_err!(session.dev.as_ref(), + "M5_APPLICATION_COMPUTE failed {:?}; outer/graphics={:x?} compute={:x?}\n", + error, memory.graphics_status(), memory.application_compute_status()); + } + let stopped = session.stop_cpus(); + dev_err!(session.dev.as_ref(), + "M5_APPLICATION_COMPUTE stop={:?}; VM and owners retained until reboot\n", stopped); + } + result + } +} + +impl Drop for M5Runtime { + fn drop(&mut self) { + if let Some(mut session) = self.session.take() { + if let Some(memory) = &session.memory { memory.quarantine_graphics(); } + let stopped = session.stop_cpus(); + dev_dbg!(session.dev.as_ref(), + "M5 session release requested; stop={:?}; halt unqualified, owners retained until reboot\n", + stopped); + core::mem::forget(session); + } + } +} diff --git a/drivers/gpu/drm/asahi/m5/mapped.rs b/drivers/gpu/drm/asahi/m5/mapped.rs new file mode 100644 index 0000000000000..2ccc37b82531b --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/mapped.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Kernel-owned, write-combined GEM objects mapped at one GPU address. + +use core::ops::Range; +use kernel::prelude::*; + +use crate::{driver::AsahiDrmDevice, gem, mmu}; + +/// A zeroed, write-combined kernel object and its owning GPU mapping. +pub(crate) struct MappedObject { + _mapping: mmu::KernelMapping, + object: gem::ObjectRef, + iova: u64, + /// Size of the object in bytes, as requested. + pub(crate) logical_size: usize, + /// The VM of the owning mapping. + home: u64, +} + +impl MappedObject { + fn new_object(dev: &AsahiDrmDevice, size: usize) -> Result { + let mut object = gem::new_kernel_object_wc(dev, size)?; + let len = object.size(); + let vmap = object.vmap()?; + // SAFETY: the vmap covers the whole page-rounded allocation. + unsafe { core::ptr::write_bytes(vmap.addr.cast::(), 0, len) }; + Ok(object) + } + + fn from_mapping( + object: gem::ObjectRef, vm: &mmu::Vm, mapping: mmu::KernelMapping, size: usize, + ) -> Self { + let iova = mapping.iova(); + Self { _mapping: mapping, object, iova, logical_size: size, home: vm.id() } + } + + /// Maps a new object of `size` bytes at `address`. + pub(crate) fn new_wc_at( + dev: &AsahiDrmDevice, vm: &mmu::Vm, address: u64, size: usize, prot: mmu::Prot, + ) -> Result { + let mut object = Self::new_object(dev, size)?; + let mapping = object.map_at(vm, address, prot, false)?; + Ok(Self::from_mapping(object, vm, mapping, size)) + } + + /// Maps a new object of `size` bytes at a free address in `range`. + pub(crate) fn new_wc_in_range( + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: Range, size: usize, alignment: u64, + prot: mmu::Prot, + ) -> Result { + let mut object = Self::new_object(dev, size)?; + let mapping = object.map_into_range(vm, range, alignment, prot, true)?; + Ok(Self::from_mapping(object, vm, mapping, size)) + } + + /// As [`Self::new_wc_in_range`], followed by `guard_pages` unmapped pages. + pub(crate) fn new_wc_in_range_with_guards( + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: Range, size: usize, alignment: u64, + prot: mmu::Prot, guard_pages: usize, + ) -> Result { + let object = Self::new_object(dev, size)?; + let guard = guard_pages.checked_mul(mmu::UAT_PGSZ).ok_or(EOVERFLOW)?; + let mapping = vm.map_in_range_with_guard_size( + &object.gem, 0..object.size(), alignment, range, prot, guard, + )?; + Ok(Self::from_mapping(object, vm, mapping, size)) + } + + /// Maps this object again at `address` of `vm`. + pub(crate) fn map_alias_at( + &mut self, vm: &mmu::Vm, address: u64, prot: mmu::Prot, + ) -> Result { + self.object.map_at(vm, address, prot, false) + } + + /// Maps this object at its own address in another client VM, followed by `guard_pages` + /// unmapped pages. In the VM of the owning mapping, where the object already is at that + /// address, there is nothing to map. + pub(crate) fn map_alias_at_with_guards( + &mut self, vm: &mmu::Vm, address: u64, prot: mmu::Prot, guard_pages: usize, + ) -> Result> { + if vm.id() == self.home { + return if address == self.iova { Ok(None) } else { Err(EINVAL) }; + } + let guard = guard_pages.checked_mul(mmu::UAT_PGSZ).ok_or(EOVERFLOW)?; + vm.map_at_with_guard_size(address, self.object.size(), &self.object.gem, prot, guard) + .map(Some) + } + + /// GPU address of the object. + pub(crate) fn iova(&self) -> u64 { + self.iova + } + + /// Runs `f` on the CPU view of the object's `size` bytes. + pub(crate) fn with_bytes_mut(&mut self, f: impl FnOnce(&mut [u8]) -> Result) -> Result { + let size = self.logical_size; + let vmap = self.object.vmap()?; + // SAFETY: the vmap covers the whole allocation, which is at least `size` bytes, and + // `&mut self` excludes other CPU views made through this object. + let bytes = unsafe { core::slice::from_raw_parts_mut(vmap.addr.cast::(), size) }; + f(bytes) + } + + fn offset(&self, address: u64, width: usize) -> Result { + let offset: usize = address.checked_sub(self.iova).ok_or(EINVAL)?.try_into()?; + let end = offset.checked_add(width).ok_or(EINVAL)?; + if end > self.logical_size { + return Err(EINVAL); + } + Ok(offset) + } + + /// Reads the little-endian word at GPU address `address` (any alignment). + pub(crate) fn read_u32(&mut self, address: u64) -> Result { + let offset = self.offset(address, 4)?; + self.with_bytes_mut(|bytes| { + Ok(u32::from_le_bytes(bytes[offset..offset + 4].try_into().map_err(|_| EINVAL)?)) + }) + } + + /// Reads the little-endian double word at GPU address `address` (any alignment). + pub(crate) fn read_u64(&mut self, address: u64) -> Result { + let offset = self.offset(address, 8)?; + self.with_bytes_mut(|bytes| { + Ok(u64::from_le_bytes(bytes[offset..offset + 8].try_into().map_err(|_| EINVAL)?)) + }) + } + + /// Writes the naturally aligned little-endian word at GPU address `address` with one store. + pub(crate) fn write_u32(&mut self, address: u64, value: u32) -> Result { + let offset = self.offset(address, 4)?; + if offset % 4 != 0 { + return Err(EINVAL); + } + self.with_bytes_mut(|bytes| { + // SAFETY: checked bounds and natural alignment. + unsafe { + core::ptr::write_volatile(bytes.as_mut_ptr().add(offset).cast::(), value.to_le()) + }; + Ok(()) + }) + } +} diff --git a/drivers/gpu/drm/asahi/m5/mod.rs b/drivers/gpu/drm/asahi/m5/mod.rs new file mode 100644 index 0000000000000..96beaac4391f4 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/mod.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple M5 (T8142, G17G) GPU support for the 25G76 firmware (RTKit-3255.160.4). +//! +//! The M5 has two GPU coprocessors, GFX and GFX1, as the A18 Pro does. The boot loader (aura) +//! loads their firmware, checks both loaded TEXT images against the 25G76 build and says so +//! in `/chosen/aurora,m5-gpu-firmware-profile`. The firmware interface is HAL300 and differs +//! from the 26A428 and 25G83 builds the shared G17 code runs, so this backend builds its own +//! initialization graph ([`initdata`], [`resources`]) and talks to the firmware over its own +//! channels ([`channel`], [`sksm`], [`completion`]). +//! +//! # Plugging into the shared driver +//! +//! - `ProbeConfig::M5` (`apple,agx-t8142`) calls [`probe`], which starts both coprocessors +//! ([`live_boot`]) and returns a [`Gpu`] implementing [`crate::gpu::Gpu`]. +//! - The UAT is the shared [`crate::mmu::Uat`] with published roots, as on the A18 Pro, plus +//! [`crate::mmu::M5ExecutionContext`] for application contexts. User VMs carry a status and +//! a job lifetime, and the backend uses the tracked submission model of `file.rs`. +//! - The UAPI is the shared one. The backend reports `VM_STATUS` (with its mirror) and +//! `SOFT_FAULTS`, and one hardware command per submission. +//! +//! # Submission +//! +//! One render pair (TA and fragment queues) and one compute queue serve every client: the +//! queues are installed by the first render and handed from client VM to client VM, with +//! the driver's pools aliased at the same addresses in each. A submission is published, +//! then polled to retirement before the ioctl returns, so there is one job in flight. +//! Renders and compute are admitted only within the state the firmware programs have been +//! verified with ([`render`], [`compute`]); anything else is refused with `ENOTSUPP` +//! before it reaches the firmware. +//! +//! # Failures +//! +//! There is no recovery yet. A job that fails or does not retire marks its VM, stops both +//! coprocessors and retains every firmware-visible object until reboot; the device is lost +//! for all clients. The module stays loaded for the same reason. + +mod channel; +mod command; +mod completion; +mod compute; +mod drm; +mod fw_msg; +mod initdata; +mod live_boot; +mod mapped; +mod provider; +mod render; +mod render_desc; +mod render_target; +mod resources; +mod sksm; +mod submission; +mod vm_layout; + +use kernel::{device::Core, of, platform, prelude::*, sync::Arc}; + +use crate::{driver::AsahiDrmDevice, regs}; + +pub(crate) use drm::M5DrmGpu as Gpu; + +/// The firmware profile aura publishes in `/chosen/aurora,m5-gpu-firmware-profile` after it +/// checked both loaded TEXT images against the exact 25G76 build. +const FIRMWARE_PROFILE: &[u8] = b"M5-RTKit-3255.160.4.release\0"; + +/// Starts both firmware instances and returns the GPU, ready for DRM registration. +pub(crate) fn probe( + pdev: &platform::Device>, + dev: &AsahiDrmDevice, + res: regs::G17Resources, +) -> Result> { + let chosen = of::chosen().ok_or(ENODEV)?; + let profile: KVec = chosen.get_property(c"aurora,m5-gpu-firmware-profile")?; + if profile.as_slice() != FIRMWARE_PROFILE { + dev_err!(pdev.as_ref(), "M5: GPU firmware profile not published or not 25G76\n"); + return Err(ENODEV); + } + let id = res.get_g17_id()?; + let cores: u32 = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); + // M5 GPUs have 10 cores, or 8 with two fused off (MacBook Air). + if id.family != 0xb || id.num_dies != 1 || !matches!(cores, 8 | 10) { + dev_err!(pdev.as_ref(), "M5: unexpected GPU family {:#x}, {} dies, {} cores\n", + id.family, id.num_dies, cores); + return Err(ENODEV); + } + // Before the session: a CLPC that has not probed yet defers the GPU. + let clpc = drm::M5ClpcInput::new(pdev.as_ref())?; + let runtime = live_boot::M5Runtime::start_application(pdev, dev.into(), res)?; + let runtime = Arc::pin_init(kernel::new_mutex!(runtime), GFP_KERNEL)?; + Ok(Arc::new(Gpu::new(runtime, id, clpc), GFP_KERNEL)?) +} diff --git a/drivers/gpu/drm/asahi/m5/provider.rs b/drivers/gpu/drm/asahi/m5/provider.rs new file mode 100644 index 0000000000000..d54a591f85448 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/provider.rs @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The GFX and GFX1 ASCWrap v6 mailbox providers, as seen by the M5 firmware session. + +use core::mem::MaybeUninit; + +use kernel::{ + bindings, + error::{from_err_ptr, to_result}, + io::{IoSysMap, Region}, + prelude::*, + soc::apple::rtkit, +}; + +use crate::mmu; + +const G17P_CRASHLOG_MAX_BYTES: usize = 256 * 1024; + +const G17P_RTKIT_UAT_DVA_MASK: u64 = (1u64 << 40) - 1; + +/// Module-local, read-only view of one existing ASCWrap mailbox provider. +/// +/// The C provider owns the mapping and all lifecycle transitions. This view +/// neither maps an alias nor changes callback ownership; it only borrows the +/// provider's established mailbox-register pointer for two control reads. +pub(super) struct G17PMailboxProvider { + mbox: *mut bindings::apple_mbox, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) enum G17PAscwrapV6Role { + Gfx, + Gfx1, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) struct G17PAscwrapV6Lifecycle { + role: G17PAscwrapV6Role, + wrapper: (u64, u64), + iop_vbar: (u64, u64), + missing: u32, +} + +impl G17PMailboxProvider { + + pub(super) fn get(dev: &kernel::device::Device, index: usize) -> Result { + // SAFETY: `dev` is live and apple_mbox_get() establishes a device link + // to keep the resolved provider alive for this consumer. + let mbox = unsafe { + from_err_ptr(bindings::apple_mbox_get( + dev.as_raw(), + index.try_into()?, + ))? + }; + Ok(Self { mbox }) + } + + pub(super) fn lifecycle(&self) -> Result { + let mut raw = MaybeUninit::::uninit(); + // SAFETY: `self.mbox` is the live provider returned by + // apple_mbox_get() and `raw` points to writable storage. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_lifecycle(self.mbox, raw.as_mut_ptr()) + })?; + // SAFETY: the successful C call initialized every field. + let raw = unsafe { raw.assume_init() }; + let role = match raw.role as u32 { + 1 => G17PAscwrapV6Role::Gfx, + 2 => G17PAscwrapV6Role::Gfx1, + _ => return Err(EINVAL), + }; + Ok(G17PAscwrapV6Lifecycle { + role, + wrapper: (raw.wrapper_start, raw.wrapper_size), + iop_vbar: (raw.iop_vbar_start, raw.iop_vbar_size), + missing: raw.missing, + }) + } + + pub(super) fn require_safe_cpu_lifecycle(&self) -> Result { + // SAFETY: this is a capability check on the live provider pointer. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(self.mbox) + }) + } + + pub(super) fn start_cpu(&self) -> Result { + // SAFETY: lifecycle serialization and MMIO ownership remain in C. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_start_cpu(self.mbox) }) + } + + pub(super) fn stop_cpu(&self) -> Result { + // SAFETY: lifecycle serialization and MMIO ownership remain in C. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }) + } + +} + +unsafe impl Send for G17PMailboxProvider {} + +unsafe impl Sync for G17PMailboxProvider {} + +pub(super) const fn g17p_rtkit_uat_address(dva: u64) -> u64 { + dva & G17P_RTKIT_UAT_DVA_MASK +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(super) enum G17PProcessor { + Gfx = 0, + Gfx1 = 1, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) struct G17PFirmwareCrashBuffer { + dva: u64, + size: usize, +} + +impl G17PProcessor { + + pub(super) const fn firmware_crash_buffer(self) -> G17PFirmwareCrashBuffer { + // Both endpoint-1 buffer requests are stable across the retained + // G17P captures. Message 0x0010810001e1c000 decodes to the primary + // DVA below and size 0x8000; GFX1 advertises the second DVA with the + // same size. + match self { + Self::Gfx => G17PFirmwareCrashBuffer { + dva: 0x100_01e1_c000, + size: 0x8000, + }, + Self::Gfx1 => G17PFirmwareCrashBuffer { + dva: 0x100_01f5_0000, + size: 0x8000, + }, + } + } +} + +pub(super) const fn bounded_crashlog_size(size: usize) -> usize { + if size < G17P_CRASHLOG_MAX_BYTES { + size + } else { + G17P_CRASHLOG_MAX_BYTES + } +} + +const _: () = { + assert!(bounded_crashlog_size(0) == 0); + assert!(bounded_crashlog_size(4096) == 4096); + assert!(bounded_crashlog_size(G17P_CRASHLOG_MAX_BYTES + 1) == G17P_CRASHLOG_MAX_BYTES); + let gfx = G17PProcessor::Gfx.firmware_crash_buffer(); + let gfx1 = G17PProcessor::Gfx1.firmware_crash_buffer(); + assert!(gfx.dva == 0x100_01e1_c000 && gfx1.dva == 0x100_01f5_0000); + assert!(g17p_rtkit_uat_address(gfx.dva) == 0x01e1_c000); + assert!(g17p_rtkit_uat_address(gfx1.dva) == 0x01f5_0000); + assert!(gfx.size == 0x8000 && gfx1.size == 0x8000); + assert!(gfx.dva & (mmu::UAT_PGSZ as u64 - 1) == 0); + assert!(gfx1.dva & (mmu::UAT_PGSZ as u64 - 1) == 0); + assert!(gfx.dva + gfx.size as u64 <= 1u64 << 42); + assert!(gfx1.dva + gfx1.size as u64 <= 1u64 << 42); +}; + +/// Active G17P boot does not ask Rust to back system-endpoint buffers. The +/// generic RTKit core uses its coherent-DMA fallback when these callbacks are +/// absent. +pub(super) struct G17PNoRtkitBuffer; + +impl rtkit::Buffer for G17PNoRtkitBuffer { + fn iova(&self) -> Result { + Err(ENOTSUPP) + } + + fn buf(&mut self) -> Result> { + Err(ENOTSUPP) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(super) struct ExpectedProvider { + pub(super) role: G17PAscwrapV6Role, + pub(super) wrapper: (u64, u64), + pub(super) iop_vbar: (u64, u64), +} + +pub(super) fn validate_provider( + provider: &G17PMailboxProvider, + expected: ExpectedProvider, +) -> Result { + let lifecycle = provider.lifecycle()?; + if lifecycle.role != expected.role + || lifecycle.wrapper != expected.wrapper + || lifecycle.iop_vbar != expected.iop_vbar + { + return Err(EINVAL); + } + + Ok(lifecycle) +} diff --git a/drivers/gpu/drm/asahi/m5/render.rs b/drivers/gpu/drm/asahi/m5/render.rs new file mode 100644 index 0000000000000..22bede0991343 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/render.rs @@ -0,0 +1,508 @@ +// SPDX-License-Identifier: MIT +//! M5/G17G graphics register construction from owned render resources. +//! +//! Common resource lowering is shared with G17P. Register order, tile packing +//! and per-stage controls follow the native 25G76 M5 reference. This codec is +//! not a claim that Linux TA/3D submission or presentation is qualified. + +use crate::m5::render_desc::{ + self, G17pRenderParameters, RegisterWrite, RenderBuildError, RenderDescriptorMetadata, + RenderDescriptorObjects, +}; + +pub(crate) const TA_REGISTER_COUNT: usize = 82; +pub(crate) const FRAGMENT_REGISTER_COUNT: usize = 90; + +/// Encode the TA parameter-manager metrics slot in the compact GART window. +/// The native 25G76 slot at 0x1000080040 produces register 1c910 = 0x80041: +/// the low bit enables the record; the remaining bits address its 64 bytes. +pub(crate) fn parameter_buffer_metrics_record( + scene_metrics: u64, +) -> Result { + let offset = scene_metrics.checked_sub(0x10_0000_0000) + .ok_or(M5RenderBuildError::OwnershipMismatch)?; + if offset >= (1u64 << 32) || offset & 0x3f != 0 { + return Err(M5RenderBuildError::OwnershipMismatch); + } + Ok(offset | 1) +} + +/// Initial M5 PB slot with fresh ring cursors. The owning job chooses the +/// hardware slot separately and retains the page-list mapping. +pub(crate) fn initial_parameter_buffer_descriptor( + page_list: u64, + page_count: u32, +) -> Result<[u32; 4], M5RenderBuildError> { + if page_list == 0 || page_list >= (1u64 << 42) || page_list & 0x7f != 0 + || page_count == 0 || page_count >= (1 << 22) { + return Err(M5RenderBuildError::OwnershipMismatch); + } + // 25G76 halLoadPBDesc, 89de844..89de8dc. The descriptor has four + // dwords; +0's low flags and both cursors start at zero. Native's + // untagged user address is translated into its PB segment encoding. + let segment = ((page_list + 0x70_0000_0000) & 0x70_0000_0000) >> 8; + Ok([((page_list >> 4) as u32) & !7, + (segment as u32 & 0xf000_0000) | page_count, 0, 0]) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5RenderBuildError { + Common(RenderBuildError), + SharedRegisterLayout, + TileCountOutOfRange, + OverlappingTileFlags, + DispatchBitAlreadySet, + InvalidContext, + OwnershipMismatch, + InvalidUscState, +} + +impl From for M5RenderBuildError { + fn from(error: RenderBuildError) -> Self { + Self::Common(error) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderStageControls { + /// Native host descriptor +0x1120 (TA) / +0x7e8 (fragment). + pub(crate) resource_flags: u64, + /// Explicit per-stage d419/d429 input, independent of the work key. + pub(crate) resource_control: u64, + /// Native +0x1128 / +0x7f0; a zero source selects the native 0x60 fallback. + pub(crate) ptc_max: u32, + pub(crate) work_key: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderControls { + pub(crate) ta: M5RenderStageControls, + pub(crate) fragment: M5RenderStageControls, + pub(crate) ta_tile_control: u64, + /// Base value for the two 1c898 writes. Bit zero is supplied by the codec. + pub(crate) ta_dispatch_control: u64, + pub(crate) fragment_raster_control: u64, + /// Non-geometry bits for 16068. Never includes tile counts, utile or layers. + pub(crate) fragment_tile_flags: u64, + /// M5 adds this second pair of store-program bindings at 153b1/153b9. + pub(crate) store_pipeline_2_bind: u64, + pub(crate) store_pipeline_2: u64, + pub(crate) partial_store_pipeline_2_bind: u64, + pub(crate) partial_store_pipeline_2: u64, + pub(crate) partial_tile_flags: u64, +} + +/// Per-submission USC private-memory accounting, separate from user status +/// registers and queue completion stamps. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5RenderUscState { + pub(crate) control: u64, + /// AGXUMADataRec +0x50. Native allocatePagesEnd derives this from pool + /// sizes and 4-KiB allocation units; it is a byte count, not a GPU VA. + pub(crate) required_bytes: u64, + /// Capacity actually backed by this job's retained USC allocation pages. + pub(crate) allocated_bytes: u64, + /// A 64-byte slot in the firmware-visible statistics ring. + pub(crate) statistics: u64, +} + +impl M5RenderUscState { + fn valid(self) -> bool { + let firmware_range = |address: u64, size: u64| { + address >= 0xffff_fc00_0000_0000 + && address & 0x3f == 0 + && address.checked_add(size).is_some() + }; + firmware_range(self.control, 0x80) + && firmware_range(self.statistics, 0x40) + && self.required_bytes != 0 + && self.required_bytes <= self.allocated_bytes + && self.allocated_bytes & 0xfff == 0 + } +} + +/// Fill the native USC fields only after the caller owns the pool and the +/// statistics slot. Other runtime fields still need queue/link publication. +/// Validate both stages before modifying either descriptor. +pub(crate) fn apply_usc_submission_state( + ta: &mut [u8], + fragment: &mut [u8], + ta_state: M5RenderUscState, + fragment_state: M5RenderUscState, +) -> Result<(), M5RenderBuildError> { + if ta.len() < render_desc::TA_DESCRIPTOR_SIZE + || fragment.len() < render_desc::FRAGMENT_DESCRIPTOR_SIZE + { + return Err(RenderBuildError::BufferTooSmall.into()); + } + if !ta_state.valid() || !fragment_state.valid() { + return Err(M5RenderBuildError::InvalidUscState); + } + // 25G76 AGXTAChannelSKSM::submitBuffer: 8a244c4..8a244ec. + // AGX3DChannelSKSM::submitBuffer takes the corresponding value from + // host +0xba0 = AGXUMADataRec +0x50 at 895b70c..895b718. + for (raw, state, control, required, statistics) in [ + (ta, ta_state, 0x934, 0x93d, 0x945), + (fragment, fragment_state, 0x21ce, 0x21d7, 0x21df), + ] { + put_u64(raw, control, state.control); + put_u64(raw, required, state.required_bytes); + put_u64(raw, statistics, state.statistics); + } + Ok(()) +} + +const EMPTY_REGISTER: RegisterWrite = RegisterWrite { + number: 0, + value: 0, +}; + +struct RegisterList { + values: [RegisterWrite; N], + count: usize, +} + +impl RegisterList { + fn new() -> Self { + Self { + values: [EMPTY_REGISTER; N], + count: 0, + } + } + + fn push(&mut self, number: u32, value: u64) -> Result<(), M5RenderBuildError> { + if self.count == N { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + self.values[self.count] = RegisterWrite { number, value }; + self.count += 1; + Ok(()) + } + + fn finish(self) -> Result<[RegisterWrite; N], M5RenderBuildError> { + if self.count != N { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + Ok(self.values) + } +} + +fn value(registers: &[RegisterWrite], number: u32) -> Result { + registers + .iter() + .find(|r| r.number == number) + .map(|r| r.value) + .ok_or(M5RenderBuildError::SharedRegisterLayout) +} + +const fn ptc_control(maximum: u32) -> u64 { + let maximum = if maximum == 0 { 0x60 } else { maximum }; + ((maximum as u64) << 32) | 0x400020 +} + +fn tile_state(p: &G17pRenderParameters, flags: u64) -> Result { + let tiles_x = (p.width as u64 + 31) / 32; + let tiles_y = (p.height as u64 + 31) / 32; + if !(1..=1024).contains(&tiles_x) || !(1..=1024).contains(&tiles_y) { + return Err(M5RenderBuildError::TileCountOutOfRange); + } + // 25G76 fragment generator: bfi at 89dcb98, 89dcbb8 and 89dcbbc. + let geometry_mask = (0xfffff_u64 << 42) | (0xf_u64 << 38) | (1_u64 << 32); + if flags & geometry_mask != 0 { + return Err(M5RenderBuildError::OverlappingTileFlags); + } + Ok(flags + | ((tiles_x - 1) << 42) + | ((tiles_y - 1) << 52) + | ((p.utile_config & 0xf000) << 26) + | if p.layers > 1 { 1_u64 << 32 } else { 0 }) +} + +fn tile_blocks(pixels: u32) -> u64 { + let tiles = (pixels as u64 + 31) / 32; + let first = (((tiles + 3) / 4) + 3) & !3; + // Native TA host +1208/+120c supplies 0x40200c for the reference: + // boundaries 12, 8, 4 in ten-bit fields, versus G17P's nine-bit fields. + (3 * first) | ((2 * first) << 10) | (first << 20) +} + +pub(crate) fn build_ta_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], M5RenderBuildError> { + // M5's captured register program addresses heap metadata directly. + // G17P's layered path now uses a separate layer table in these slots. + let mut common_parameters = *p; + common_parameters.layermeta = p.heapmeta; + let common = render_desc::build_ta_registers(&common_parameters)?; + if controls.ta_dispatch_control & 1 != 0 { + return Err(M5RenderBuildError::DispatchBitAlreadySet); + } + // Validate the M5 tile fields even when only constructing the TA half. + tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterList::new(); + for mut register in common { + match register.number { + 0x0a309 | 0x0a5a1 | 0x0d419 => continue, + 0x1c8f8 => { + out.push(0x0d499, controls.ta.resource_flags)?; + out.push(0x0d419, controls.ta.resource_control)?; + out.push(0x0a5a1, ptc_control(controls.ta.ptc_max))?; + register.value = controls.ta_tile_control; + } + 0x10171 => { + for (number, source) in [(0x1ca50, 0x101b9), (0x1ca58, 0x10149), (0x1ca60, 0x10141)] + { + out.push(number, value(&common, source)?)?; + } + out.push(0x1ca70, tile_blocks(p.width))?; + out.push(0x1ca78, tile_blocks(p.height))?; + out.push(0x1ca68, value(&common, 0x1c081)?)?; + } + 0x1c069 => register.value = tile_blocks(p.width), + 0x1c071 => register.value = tile_blocks(p.height), + 0x1c898 => { + if register.value > 1 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + register.value |= controls.ta_dispatch_control; + } + 0x10799 => out.push(0x1a149, 0)?, + 0x10209 => { + out.push(0x0a739, 0)?; + register.value = controls.ta.work_key; + } + 0x1c9f0 | 0x14320 => register.value = controls.ta.work_key, + 0x14308 => register.value = p.usc_flist_hardware_buffer_id as i32 as i64 as u64, + _ => {} + } + out.push(register.number, register.value)?; + if register.number == 0x01740 { + out.push(0x0a739, 0)?; + } + } + out.finish() +} + +pub(crate) fn build_fragment_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], M5RenderBuildError> { + // M5's captured register program addresses heap metadata directly. + // G17P's layered path now uses a separate layer table in these slots. + let mut common_parameters = *p; + common_parameters.layermeta = p.heapmeta; + let common = render_desc::build_fragment_registers(&common_parameters)?; + let geometry = tile_state(p, controls.fragment_tile_flags)?; + let mut out = RegisterList::new(); + let mut dispatch_writes = 0; + let mut tile_writes = 0; + for mut register in common { + match register.number { + 0x0a301 | 0x15109 | 0x15101 | 0x160a8 | 0x0a5a9 | 0x0d429 => continue, + 0x100b1 => register.number = 0x16469, + 0x100d9 => register.number = 0x16471, + 0x100b8 => register.number = 0x16138, + 0x10791 => register.value = controls.fragment_raster_control, + 0x15321 => { + out.push(0x0d491, controls.fragment.resource_flags)?; + out.push(0x0d429, controls.fragment.resource_control)?; + out.push(0x0a5a9, ptc_control(controls.fragment.ptc_max))?; + } + 0x16068 => { + tile_writes += 1; + if tile_writes == 1 { + register.value = geometry; + } + } + 0x16020 => { + dispatch_writes += 1; + if dispatch_writes == 3 { + if register.value != 0 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + continue; + } + } + 0x160e0 => out.push(0x1a151, 0)?, + 0x10211 => { + out.push(0x0a741, 0)?; + register.value = controls.fragment.work_key; + } + 0x10420 => register.value = controls.fragment.work_key, + 0x14048 => register.value = p.usc_flist_hardware_buffer_id as i32 as i64 as u64, + _ => {} + } + out.push(register.number, register.value)?; + if register.number == 0x15381 { + out.push(0x153b1, controls.store_pipeline_2_bind)?; + out.push(0x153b9, controls.store_pipeline_2)?; + } + if register.number == 0x01731 { + out.push(0x0a741, 0)?; + } + } + if dispatch_writes != 3 || tile_writes != 2 { + return Err(M5RenderBuildError::SharedRegisterLayout); + } + out.finish() +} + +fn partial_registers( + common: &[RegisterWrite], + insert_after: u32, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; N], M5RenderBuildError> { + let mut out = RegisterList::new(); + for register in common { + let bits = if register.number == 0x10039 { + controls.partial_tile_flags + } else { + 0 + }; + out.push(register.number, register.value | bits)?; + if register.number == insert_after { + out.push(0x153b1, controls.partial_store_pipeline_2_bind)?; + out.push(0x153b9, controls.partial_store_pipeline_2)?; + } + } + out.finish() +} + +pub(crate) fn build_partial_store_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 18], M5RenderBuildError> { + partial_registers(&render_desc::partial_store_registers(p), 0x15381, controls) +} + +pub(crate) fn build_partial_resume_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 25], M5RenderBuildError> { + partial_registers(&render_desc::partial_resume_registers(p), 0x15381, controls) +} + +pub(crate) fn build_partial_load_registers( + p: &G17pRenderParameters, + controls: &M5RenderControls, +) -> Result<[RegisterWrite; 12], M5RenderBuildError> { + partial_registers(&render_desc::partial_load_registers(p), 0x15309, controls) +} + +fn work_key( + p: &G17pRenderParameters, + metadata: RenderDescriptorMetadata, +) -> Result { + if metadata.context_id == 0 || metadata.context_id > 255 { + return Err(M5RenderBuildError::InvalidContext); + } + if metadata.ta_hardware_buffer_id != p.ta_hardware_buffer_id { + return Err(M5RenderBuildError::OwnershipMismatch); + } + metadata + .submission_ordinal + .checked_add(metadata.submission_ordinal / 2) + .ok_or(M5RenderBuildError::Common( + RenderBuildError::ArithmeticOverflow, + ))?; + Ok(((metadata.context_id as u64) << 8) | (metadata.command_buffer_state_word & 0x3f) as u64) +} + +fn alias_offset(alias: u64, offset: u64) -> Result { + alias.checked_add(offset).ok_or(M5RenderBuildError::Common( + RenderBuildError::ArithmeticOverflow, + )) +} + +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Construct the M5 TA record's common layout and all register programs. +/// Runtime ownership/completion fields must be populated before publication. +#[inline(never)] +pub(crate) fn build_ta_descriptor_base( + p: &G17pRenderParameters, + controls: &M5RenderControls, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + low_alias: u64, + raw: &mut [u8], +) -> Result<(), M5RenderBuildError> { + let mut controls = *controls; + controls.ta.work_key = work_key(p, metadata)?; + let program = alias_offset(low_alias, 0x60)?; + alias_offset(low_alias, render_desc::TA_DESCRIPTOR_SIZE as u64)?; + let registers = build_ta_registers(p, &controls)?; + render_desc::build_ta_descriptor_base(p, objects, metadata, ®isters, raw)?; + // Native boundaries: main work ends before the final a739/deflake/dispatch. + for (offset, count) in [ + (0x38, TA_REGISTER_COUNT - 3), + (0x3a, TA_REGISTER_COUNT), + (0x3c, TA_REGISTER_COUNT), + ] { + put_u16(raw, offset, count as u16); + } + put_u64(raw, 0x760, program); + Ok(()) +} + +/// Construct the M5 fragment record, including all three partial programs. +/// The shared encoder is pure and does not apply Neo diagnostic overrides. +#[inline(never)] +pub(crate) fn build_fragment_descriptor_base( + p: &G17pRenderParameters, + controls: &M5RenderControls, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + low_alias: u64, + raw: &mut [u8], +) -> Result<(), M5RenderBuildError> { + let mut controls = *controls; + controls.fragment.work_key = work_key(p, metadata)?; + alias_offset(low_alias, render_desc::FRAGMENT_DESCRIPTOR_SIZE as u64)?; + let main = build_fragment_registers(p, &controls)?; + let store = build_partial_store_registers(p, &controls)?; + let resume = build_partial_resume_registers(p, &controls)?; + let load = build_partial_load_registers(p, &controls)?; + render_desc::build_fragment_descriptor_base( + p, + objects, + metadata, + render_desc::G17pFragmentRegisters { + main: &main, + store: &store, + resume: &resume, + load: &load, + }, + raw, + )?; + put_u32(raw, 0x50, 1); + put_u32(raw, 0x90, 1); + for (offset, count) in [ + (0x80, FRAGMENT_REGISTER_COUNT - 3), + (0x82, FRAGMENT_REGISTER_COUNT - 1), + (0x84, FRAGMENT_REGISTER_COUNT - 1), + (0x88, FRAGMENT_REGISTER_COUNT), + ] { + put_u16(raw, offset, count as u16); + } + for (offset, target) in [ + (0x7a0, 0xa0), + (0xec0, 0x7c0), + (0x15e0, 0xee0), + (0x1d00, 0x1600), + ] { + put_u64(raw, offset, alias_offset(low_alias, target)?); + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/m5/render_desc.rs b/drivers/gpu/drm/asahi/m5/render_desc.rs new file mode 100644 index 0000000000000..42a556acde208 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/render_desc.rs @@ -0,0 +1,1138 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +const TA_REGISTER_COUNT: usize = 73; + +const FRAGMENT_REGISTER_COUNT: usize = 89; + +/// Both embedded register lists match the kext EXACTLY -- same writes, same +/// unique registers, same order -- verified against 2048-byte captures of the +/// native TA and 3D command records (macos-trace recs2048-gfx.txt): +/// +/// TA 73 writes, 71 unique, list ends at +0x3cc +/// 3D 89 writes, 86 unique, list ends at +0x4cc +/// +/// A 1024-byte capture cuts the 3D list at exactly +0x400 and hides its last +/// 17 writes, which briefly looked like 14 registers "Metal never writes" -- +/// they were entries 72..85. Assert the counts so a future truncated capture +/// cannot produce that conclusion again. +const _: () = assert!(TA_REGISTER_COUNT == 73); + +const _: () = assert!(FRAGMENT_REGISTER_COUNT == 89); + +const PARTIAL_STORE_REGISTER_COUNT: usize = 16; + +const PARTIAL_RESUME_REGISTER_COUNT: usize = 23; + +const PARTIAL_LOAD_REGISTER_COUNT: usize = 10; + +const REGISTER_BYTES: usize = 12; + +pub(crate) const TA_DESCRIPTOR_SIZE: usize = 0x9c0; + +pub(crate) const FRAGMENT_DESCRIPTOR_SIZE: usize = 0x2240; + +/// Native render-setup field copied through command descriptor +0xc00 to the +/// final 3D command tail at +0x2110. This is independent of SGX's GPC +/// performance-state map register. +const G17P_FRAGMENT_TAIL_SETUP_WORD: u32 = u32::MAX; + +/// J700 shared-runtime flag snapshot copied by native 3D submit into type-1 +/// command lanes +0x2124/+0x2128. +/// +/// The kext resolves the accelerator's shared event-stamp/HWDSID internal +/// resource at +0x1a130, obtains its CPU mapping, reads byte bit 0 at mapping +/// +0x2ad/+0x2ae, and widens them before per-draw register generation. The +/// working Neo Metal snapshot contains 1/1. +const G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT: [u32; 2] = [1, 1]; + +pub(crate) const G17P_EOT_RESOURCE_SPEC: u64 = 0x8; + +pub(crate) const G17P_LOAD_RESOURCE_SPEC: u64 = 0x0007_8000_0000_0040; + +pub(crate) const G17P_SOURCE_BG_PIPELINE: u64 = 0x0000_0100_0199_0240; + +pub(crate) const G17P_SOURCE_EOT_PIPELINE: u64 = 0x0000_0100_0199_0640; + +const TILE_WIDTH: u64 = 32; + +const TILE_HEIGHT: u64 = 32; + +const MACRO_TILES_X: u64 = 4; + +const MACRO_TILES_Y: u64 = 4; + +const REGION_ENTRY_SIZE: u64 = 5; + +/// Base added by G17P's compact VDM control-stream register (0x1c880). +/// The register retains the legacy 40-bit {TTBR selector, 39-bit offset} +/// shape, so lower-root user addresses are encoded relative to this base. +const G17P_USER_BASE: u64 = 0x10_0000_0000; + +const G17P_VDM_USER_WINDOW_SIZE: u64 = 1 << 39; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RenderBuildError { + ZeroDimensions, + LayerCountOutOfRange, + UtileDimensions, + AddressBeforeContext, + UserAddressOutOfRange, + VdmAddressOutOfRange, + BufferTooSmall, + ArithmeticOverflow, +} + +impl RenderBuildError { +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RegisterWrite { + pub(crate) number: u32, + pub(crate) value: u64, +} + +impl RegisterWrite { + const fn new(number: u32, value: u64) -> Self { + Self { number, value } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct G17pRenderParameters { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) context_base: u64, + /// Allocation epoch of the trusted execution-context lease, shared by TA/3D. + pub(crate) context_generation: u8, + pub(crate) tilemap: u64, + /// Per-layer tiler metadata, which native places between the tilemap and + /// the heap metadata. It is empty, and so equal to `heapmeta`, for a + /// single-layer target. + pub(crate) layermeta: u64, + pub(crate) heapmeta: u64, + pub(crate) tpc: u64, + /// Geometry size of the owned TPC subregion, before capacity page rounding. + pub(crate) tpc_size: u64, + pub(crate) deflake_1: u64, + pub(crate) deflake_2: u64, + pub(crate) deflake_3: u64, + pub(crate) encoder: u64, + pub(crate) ta_status: u64, + pub(crate) fragment_status: u64, + pub(crate) store_pipeline_bind: u64, + pub(crate) store_pipeline: u64, + pub(crate) load_pipeline_bind: u64, + pub(crate) load_pipeline: u64, + pub(crate) partial_store_pipeline_bind: u64, + pub(crate) partial_store_pipeline: u64, + pub(crate) partial_load_pipeline_bind: u64, + pub(crate) partial_load_pipeline: u64, + pub(crate) scissor_array: u64, + pub(crate) depth_bias_array: u64, + pub(crate) aux_fb: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth_buffer: u64, + pub(crate) stencil_buffer: u64, + pub(crate) depth_aux_buffer: u64, + pub(crate) stencil_aux_buffer: u64, + pub(crate) depth_stride: u64, + pub(crate) stencil_stride: u64, + pub(crate) depth_aux_stride: u64, + pub(crate) stencil_aux_stride: u64, + pub(crate) depth_clear_value_bits: u32, + pub(crate) stencil_clear_value: u32, + pub(crate) depth_flags: u64, + /// ZLS control used by every firmware partial-store/load/resume phase. + /// Unlike the pass-boundary control, an attached depth/stencil surface + /// must be both loaded and stored across an internal split. + pub(crate) partial_depth_flags: u64, + pub(crate) depth_dimensions: u64, + pub(crate) merge_upper_x_bits: u32, + pub(crate) merge_upper_y_bits: u32, + pub(crate) layers: u16, + /// Sample count: 1, 2 or 4. Defaults to 1, so every existing parameter + /// vector keeps its exact register program. + pub(crate) samples: u8, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) utile_config: u64, + pub(crate) multisample_control: u64, + pub(crate) ppp_control: u64, + pub(crate) tib_blocks: u64, + pub(crate) tile_config: u64, + pub(crate) aux_fb_flags: u64, + pub(crate) aux_fb_page_count: u64, + pub(crate) cycle: u64, + pub(crate) record_index: u64, + /// TA event correlation written to 0x1ca10/0x014a1/0x0a349. + /// Native sources this independently from the fragment correlation. + pub(crate) lifecycle: u64, + /// Fragment event correlation written to 0x160e0/0x01499/0x0a341. + pub(crate) fragment_lifecycle: u64, + pub(crate) sampler_array: u64, + pub(crate) sampler_count: u32, + pub(crate) process_empty_tiles: bool, + pub(crate) ta_timestamp_start: u64, + pub(crate) ta_timestamp_end: u64, + pub(crate) fragment_timestamp_start: u64, + pub(crate) fragment_timestamp_end: u64, + /// Write the VDM control-stream register (0x1c880) as a full GPU virtual + /// address instead of the native user-process compact form. + /// + /// SCOPE CORRECTED against the macOS oracle. This flag once covered every + /// register `ta_offset` feeds, justified from the legacy G14X path where + /// all of them are whole pointers. A real Metal render pass on THIS chip + /// says otherwise: the captured TA command record writes tilemap, TPC and + /// heapmeta as small context-relative offsets (see `ta_offset`). The legacy + /// encoding does not transfer to G17P, so those six registers are always + /// relative now and this flag applies to the VDM stream alone. Native first + /// lowers the task VA to a task-relative offset, maps the same backing at + /// 0x1000000000 + offset in the GART window, and writes that offset here. + /// Linux userspace supplies the full GART-alias VA; this kernel subtracts + /// the architectural 0x1000000000 base. Hardware adds it back for compact + /// pointers whose bit 39 is clear. + pub(crate) absolute_pointers: bool, + /// + /// Legacy diagnostic OR-mask for the BG resource-specifier high 17 bits. + /// Native calls this field PassControlMaskType and derives it from shader + /// outputs, including extension of the highest active pass bit; it is not + /// an address class or a universal constant. Main and partial shaders can + /// legitimately differ. Default zero preserves the userspace mask; + /// a nonzero value is an explicit diagnostic override only. + pub(crate) bg_resource_prefix: u64, + /// Historical opt-in diagnostic changing byte 2 of the USC private-memory + /// budget qword at TA +0x93d. It must not override +0x94d: that byte is the + /// allocation generation of the execution context, not a PM mode flag. + /// The captured budget is not yet derived from our pool allocation. + pub(crate) native_pm_bytes: bool, + /// Hardware-buffer ID allocated for this TA command. Native allocates it + /// from the accelerator's 0x7f-entry `AGXHardwareBufferIDManager`, stores + /// it in the host-side `AGXTACommandDescriptor` at +0x1248/+0x928, and the + /// register generator emits that value as register 0x1c830. Those two host + /// offsets are not fields in this final 0x9c0-byte firmware command. + pub(crate) ta_hardware_buffer_id: u32, + /// HardwareBufferID of the retained USC private-memory FList shared by TA + /// and 3D. Native allocates this from the USC manager (accelerator + /// +0x133c8), independently of the TA parameter-buffer manager above, and + /// emits the same ID in TA register 0x14308 and fragment register 0x14048. + pub(crate) usc_flist_hardware_buffer_id: u32, + /// Diagnostic capture-value bitmask retained for fields whose proper + /// inputs are not modeled yet. In particular, the 0xa5a* values are + /// assembled from workload-derived USC pool limits. Their captured values + /// are not architectural literals and must not become defaults. + pub(crate) native_ta_registers: u32, + /// Retained GPU-ID diagnostic snapshot, not the fragment parameter-buffer + /// owner. The old 0x1c838 use confused host descriptor +0x928 with a + /// different object's transient raw-command +0x928 field. + pub(crate) gpc_perf_state_map: u32, + /// Low nibble of SGX 0xe01480, copied by native 3D submit to tail +0x210c. + pub(crate) gpc_perf_state_map_low: u32, + /// Bit 0 of SGX 0xe0141c, copied by native 3D submit to tail +0x2104. + pub(crate) gpc_perf_state_control: u32, + /// GPU address of the selected native 0x80 Scene-state entry, or 0 for + /// none. The command-record binding is carried by `RenderDescriptorObjects` + /// in both TA and 3D; this copy is retained for diagnostics. + pub(crate) parameter_buffer: u64, +} + +impl G17pRenderParameters { + + pub(crate) const fn source_rsrc8() -> Self { + Self { + width: 64, + height: 64, + context_base: 0x0000_0010_0000_0000, + context_generation: 1, + tilemap: 0x0000_0010_001b_0000, + layermeta: 0x0000_0010_001b_5000, + heapmeta: 0x0000_0010_001b_5000, + tpc: 0x0000_0010_0024_0000, + tpc_size: 0x800, + deflake_1: 0x0000_0010_0006_82a0, + deflake_2: 0x0000_0010_0006_8020, + deflake_3: 0x0000_0010_0006_8000, + encoder: 0x0000_0010_0001_8000, + ta_status: 0x0000_0010_0007_8000, + fragment_status: 0x0000_0010_001a_8000, + store_pipeline_bind: G17P_EOT_RESOURCE_SPEC, + store_pipeline: G17P_SOURCE_EOT_PIPELINE, + load_pipeline_bind: G17P_LOAD_RESOURCE_SPEC, + load_pipeline: G17P_SOURCE_BG_PIPELINE, + partial_store_pipeline_bind: 0, + partial_store_pipeline: 0, + partial_load_pipeline_bind: 0, + partial_load_pipeline: 0, + scissor_array: 0x0000_0100_019a_0000, + depth_bias_array: 0x0000_0100_01af_8000, + aux_fb: 0x0000_0100_01aa_8000, + occlusion_query_base: 0, + depth_buffer: 0, + stencil_buffer: 0, + depth_aux_buffer: 0, + stencil_aux_buffer: 0, + depth_stride: 0, + stencil_stride: 0, + depth_aux_stride: 0, + stencil_aux_stride: 0, + depth_clear_value_bits: 0x3f80_0000, + stencil_clear_value: 0, + depth_flags: 0, + partial_depth_flags: 0, + depth_dimensions: 0, + merge_upper_x_bits: 0x3cdd_b3d9, + merge_upper_y_bits: 0x3cdd_b3d9, + layers: 1, + samples: 1, + utile_width: 32, + utile_height: 32, + utile_config: 0xa000, + multisample_control: 0x88, + ppp_control: 0x202, + tib_blocks: 8, + tile_config: 0x10280, + aux_fb_flags: 0xc001, + aux_fb_page_count: 0x10_0000, + cycle: 0x17_8020, + record_index: 0x8_0005, + lifecycle: 0, + fragment_lifecycle: 0, + sampler_array: 0, + sampler_count: 0, + process_empty_tiles: true, + ta_timestamp_start: 0xffff_fc20_0002_4c68, + ta_timestamp_end: 0, + fragment_timestamp_start: 0xffff_fc20_0002_4c68, + fragment_timestamp_end: 0xffff_fc20_0002_4c70, + absolute_pointers: false, + bg_resource_prefix: 0, + native_pm_bytes: false, + ta_hardware_buffer_id: 0, + usc_flist_hardware_buffer_id: 0, + native_ta_registers: 0, + gpc_perf_state_map: 0, + gpc_perf_state_map_low: 0, + gpc_perf_state_control: 0, + parameter_buffer: 0, + } + } +} + +#[derive(Debug, Copy, Clone)] +struct Geometry { + tiles_x: u64, + tiles_y: u64, + size1: u64, + size2: u64, + size3: u64, + x_blocks: u64, + y_blocks: u64, + screen: u64, + pixels: u64, + macro_size: u64, +} + +const fn align(value: u64, alignment: u64) -> u64 { + (value + alignment - 1) & !(alignment - 1) +} + +fn geometry(parameters: &G17pRenderParameters) -> Result { + if parameters.width == 0 || parameters.height == 0 { + return Err(RenderBuildError::ZeroDimensions); + } + if parameters.layers == 0 || parameters.layers > 2048 { + return Err(RenderBuildError::LayerCountOutOfRange); + } + + let width = parameters.width as u64; + let height = parameters.height as u64; + let tiles_x = (width + TILE_WIDTH - 1) / TILE_WIDTH; + let tiles_y = (height + TILE_HEIGHT - 1) / TILE_HEIGHT; + let (utile_width, utile_height) = (parameters.utile_width, parameters.utile_height); + if !matches!((utile_width, utile_height), (32, 32) | (32, 16) | (16, 16)) { + return Err(RenderBuildError::UtileDimensions); + } + let utiles_per_tile_x = TILE_WIDTH / utile_width as u64; + let utiles_per_tile_y = TILE_HEIGHT / utile_height as u64; + // Four samples split the utile in Y. Measured against Apple's own + // four-sample render: size1 0x14 -> 0x28, size3 0x20 -> 0x40 and + // macro_size 0x40004 -> 0x40008, where only the y term moves. All three + // follow from doubling this one term; two samples do not split. + let utiles_per_tile_y = if parameters.samples == 4 { + utiles_per_tile_y * 2 + } else { + utiles_per_tile_y + }; + let utiles_per_tile = utiles_per_tile_x * utiles_per_tile_y; + let mtile_x1 = align((tiles_x + MACRO_TILES_X - 1) / MACRO_TILES_X, 4); + let mtile_y1 = align((tiles_y + MACRO_TILES_Y - 1) / MACRO_TILES_Y, 4); + let mtile_x2 = 2 * mtile_x1; + let mtile_x3 = 3 * mtile_x1; + let mtile_y2 = 2 * mtile_y1; + let mtile_y3 = 3 * mtile_y1; + let mtile_stride = mtile_x1 * mtile_y1; + let size1 = (REGION_ENTRY_SIZE * mtile_stride * utiles_per_tile + 3) / 4; + + Ok(Geometry { + tiles_x, + tiles_y, + size1, + size2: mtile_stride, + size3: 2 * mtile_stride * utiles_per_tile, + x_blocks: mtile_x3 | (mtile_x2 << 9) | (mtile_x1 << 18), + y_blocks: mtile_y3 | (mtile_y2 << 9) | (mtile_y1 << 18), + screen: ((tiles_y - 1) << 12) | (tiles_x - 1), + pixels: (width - 1) | ((height - 1) << 16), + macro_size: mtile_y1 * utiles_per_tile_y | ((mtile_x1 * utiles_per_tile_x) << 16), + }) +} + +/// Context-relative offset. ALWAYS relative -- a real Metal render pass on this +/// chip writes small offsets into these registers, not addresses. +/// +/// Oracle: the TA command record captured from `mtlg.m` (macos-trace +/// `recs-gfx.txt`, dumped by `recs.d` at `requestChannelCommandTA`) carries an +/// embedded register list that decodes field-for-field against this function's +/// output -- same registers, same order, 12-byte entries of u32 number plus u64 +/// value -- with these values: +/// +/// 0x01748 = 1 (matches) +/// 0x10141 = 0x200 (matches) +/// 0x1c039 = 0x0119ba00 tilemap <- OFFSET, not a VA +/// 0x1c9c8 = 0x0119ba00 tilemap +/// 0x1c0a1 = 0x01290000 tpc <- OFFSET +/// 0x1c031 = 0x800000000119ca00 heapmeta | bit63 (matches the form) +/// 0x1c9c0 = 0x800000000119ca00 heapmeta +/// 0x1c051 = 0x003a0012006b0003 (matches the constant exactly) +/// +/// Writing full virtual addresses here -- which is what the legacy G14X path +/// does, and what an earlier version of this driver was changed to do -- points +/// the tiler's tilemap, TPC and heapmeta at values the firmware cannot use. +fn ta_resource_offset( + parameters: &G17pRenderParameters, + address: u64, +) -> Result { + address + .checked_sub(parameters.context_base) + .ok_or(RenderBuildError::AddressBeforeContext) +} + +/// Compact user/GART address consumed by the TA register and descriptor +/// fields. Native maps these objects in the architectural 0x1000000000 window +/// and publishes the offset, not the full address. Descriptor +0x7d6 is only +/// 32 bits wide, so reject aliases that cannot be represented consistently. +fn ta_user_offset(address: u64) -> Result { + let offset = address + .checked_sub(G17P_USER_BASE) + .ok_or(RenderBuildError::UserAddressOutOfRange)?; + + if offset > u32::MAX as u64 { + return Err(RenderBuildError::UserAddressOutOfRange); + } + + Ok(offset) +} + +fn vdm_user_offset(address: u64) -> Result { + let offset = address + .checked_sub(G17P_USER_BASE) + .ok_or(RenderBuildError::VdmAddressOutOfRange)?; + + if offset >= G17P_VDM_USER_WINDOW_SIZE { + return Err(RenderBuildError::VdmAddressOutOfRange); + } + + Ok(offset) +} + +pub(crate) fn build_ta_registers( + p: &G17pRenderParameters, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], RenderBuildError> { + build_ta_registers_for_work_key(p, render_work_key(1, 0)) +} + +fn build_ta_registers_for_work_key( + p: &G17pRenderParameters, + work_key: u64, +) -> Result<[RegisterWrite; TA_REGISTER_COUNT], RenderBuildError> { + let g = geometry(p)?; + let tilemap = ta_resource_offset(p, p.tilemap)?; + let layermeta = ta_resource_offset(p, p.layermeta)?; + let heapmeta = ta_resource_offset(p, p.heapmeta)?; + let tpc = ta_resource_offset(p, p.tpc)?; + let deflake_1 = ta_user_offset(p.deflake_1)?; + let deflake_2 = ta_user_offset(p.deflake_2)?; + let deflake_3 = ta_user_offset(p.deflake_3)?; + // The VDM control stream is the one pointer the 192-byte capture window + // does not reach (register 0x1c880 sits past the end of the dump), and it + // is the one that is not relative to the driver-private `context_base`. + // macOS first lowers the task VA to a task-relative offset, then maps the + // backing at 0x1000000000 + offset in the GART window. Linux userspace gives + // us that full alias VA, so this hardware encoder subtracts the GART base. + // Bit 39 selects the upper UAT root; refuse an offset that would silently + // change roots. Keep the full-address escape hatch diagnostic-only. + let encoder = if p.absolute_pointers { + p.encoder + } else { + vdm_user_offset(p.encoder)? + }; + // generateTARegisterListG16Common: the layer count less one with bit 15, + // plus bit 14 and a process-empty-tiles select for a layered target. The + // pipe mirror 0x1c9e8 keeps only the layered bit and the count. + let mut layer_mode = 0x8000 | (p.layers as u64 - 1); + if p.layers > 1 { + layer_mode |= 0x4000 | if p.process_empty_tiles { 0x2000 } else { 0x1000 }; + } + + Ok([ + RegisterWrite::new(0x01748, 1), + // Native N=1 uses 0x200 and N=2 uses 0x0. The one-layer source profile + // selects the N=1 value, matching descriptor byte +0x789 above. + RegisterWrite::new(0x10141, 0x200), + RegisterWrite::new(0x1c039, tilemap), + RegisterWrite::new(0x1c9c8, tilemap), + RegisterWrite::new(0x1c0a1, tpc), + RegisterWrite::new(0x1c031, heapmeta | 0x8000_0000_0000_0000), + RegisterWrite::new(0x1c9c0, heapmeta | 0x8000_0000_0000_0000), + RegisterWrite::new(0x1c051, 0x003a_0012_006b_0003), + RegisterWrite::new(0x1c061, 1), + RegisterWrite::new(0x10149, p.utile_config), + RegisterWrite::new(0x10139, p.multisample_control), + RegisterWrite::new(0x10111, deflake_1), + RegisterWrite::new(0x1c9b0, deflake_1), + RegisterWrite::new(0x10119, deflake_2), + RegisterWrite::new(0x1c9b8, deflake_2), + RegisterWrite::new(0x1c958, 1), + RegisterWrite::new(0x1c950, deflake_3 | 0x0004_0000_0000_0000), + RegisterWrite::new(0x1c930, 0), + RegisterWrite::new(0x1c880, encoder), + RegisterWrite::new(0x1c079, layermeta), + RegisterWrite::new(0x1c9d8, layermeta), + RegisterWrite::new(0x10151, 0), + RegisterWrite::new(0x1c199, 0), + RegisterWrite::new(0x1c1a1, 0), + RegisterWrite::new(0x1c1a9, 0), + RegisterWrite::new(0x1c1b1, 0), + RegisterWrite::new(0x1c1b9, 0), + RegisterWrite::new(0x1c8f8, 0x8860), + RegisterWrite::new(0x1c0b1, g.size1), + RegisterWrite::new(0x1c850, g.size1), + // Silicon holds 0x10131 at 0x88 at ONE, TWO and FOUR samples while + // 0x10139 carries the real sample position table (all five captures in + // logs/m5-msaa-native-hv-20260915-101837). Driving this from + // multisample_control would write the position table here, which + // Apple never does. + RegisterWrite::new(0x10131, 0x88), + RegisterWrite::new(0x10121, p.ppp_control), + RegisterWrite::new(0x10129, g.pixels), + RegisterWrite::new(0x101b9, g.screen), + RegisterWrite::new(0x1c069, g.x_blocks), + RegisterWrite::new(0x1c071, g.y_blocks), + RegisterWrite::new(0x1c081, g.size2), + RegisterWrite::new(0x1c0a9, g.size3), + RegisterWrite::new(0x10171, 0x100), + RegisterWrite::new(0x10169, layer_mode), + RegisterWrite::new(0x0a309, 0), + RegisterWrite::new(0x1c8e0, u64::MAX), + RegisterWrite::new(0x1c8e8, u64::MAX), + RegisterWrite::new(0x1c898, 0), + RegisterWrite::new(0x101e1, 0x1c), + RegisterWrite::new(0x1c9e8, layer_mode & 0x47ff), + RegisterWrite::new(0x1a099, 0), + RegisterWrite::new(0x1a0a1, 0), + RegisterWrite::new(0x1a069, 0), + RegisterWrite::new(0x1a071, 0), + RegisterWrite::new(0x1a0c9, 0), + RegisterWrite::new(0x1a0d1, 0), + RegisterWrite::new(0x101c9, 0), + RegisterWrite::new(0x0d471, 0), + RegisterWrite::new(0x1a0f1, 8), + RegisterWrite::new(0x10799, 0xff_0000), + // Native sources this from AGXHardwareBufferIDManager. It is neither a + // PBDesc ring index nor a capture-stable literal. + RegisterWrite::new(0x1c830, p.ta_hardware_buffer_id as u64), + RegisterWrite::new(0x1ca30, p.cycle), + RegisterWrite::new(0x16c39, p.cycle), + RegisterWrite::new(0x1c910, p.record_index), + // Native derives the high word from the TA AGXUMADataRec PTC/DUPM + // maximum at host descriptor +0x1488, copied through command +0x1158. + // halUpdatePTCMaxCount computes max(0x60, min(pool_input / active_kicks, + // 0x200)); Linux does not yet carry that pool input. Follow the kext's + // exact zero-source fallback 0x60_0040_0020. Bit 1 retains one working + // Metal trace's workload-derived 0xfe value as a diagnostic only. + RegisterWrite::new( + 0x0a5a1, + if p.native_ta_registers & 2 != 0 { + 0x0000_00fe_0040_0020 + } else { + 0x0000_0060_0040_0020 + }, + ), + RegisterWrite::new(0x0d419, 0x0000_0002_0000_0001), + RegisterWrite::new(0x1ca10, p.lifecycle), + RegisterWrite::new(0x014a1, p.lifecycle), + RegisterWrite::new(0x0a349, p.lifecycle), + RegisterWrite::new(0x10209, work_key), + RegisterWrite::new(0x1c9f0, work_key), + RegisterWrite::new(0x14320, work_key), + // AGXTACommandDescriptor::prepare receives the shared USC FList ID + // through descriptor +0x1410. Optional GART/feature payloads are absent + // in Linux, so the retained owner's ID is the complete value. + RegisterWrite::new(0x14308, p.usc_flist_hardware_buffer_id as u64), + RegisterWrite::new(0x14318, p.ta_status | 1), + RegisterWrite::new(0x01740, 1), + RegisterWrite::new(0x1c880, deflake_1), + RegisterWrite::new(0x1c898, 1), + ]) +} + +pub(crate) fn build_fragment_registers( + p: &G17pRenderParameters, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], RenderBuildError> { + build_fragment_registers_for_work_key(p, render_work_key(1, 1)) +} + +fn build_fragment_registers_for_work_key( + p: &G17pRenderParameters, + work_key: u64, +) -> Result<[RegisterWrite; FRAGMENT_REGISTER_COUNT], RenderBuildError> { + let g = geometry(p)?; + let tile_state = 0x3717f + | ((g.tiles_x - 1) << 44) + | ((g.tiles_y - 1) << 53) + | 0x20_0000_0000 + | (if p.layers > 1 { 0x1_0000_0000 } else { 0 }) + | ((p.utile_config & 0xf000) << 28); + + Ok([ + RegisterWrite::new(0x01739, 1), + RegisterWrite::new(0x10009, p.utile_config), + RegisterWrite::new(0x15379, p.store_pipeline_bind), + RegisterWrite::new(0x15381, p.store_pipeline), + RegisterWrite::new(0x15369, p.load_pipeline_bind), + RegisterWrite::new(0x15371, p.load_pipeline), + RegisterWrite::new(0x15131, p.merge_upper_x_bits as u64), + RegisterWrite::new(0x15139, p.merge_upper_y_bits as u64), + RegisterWrite::new(0x100a1, 0), + RegisterWrite::new(0x15069, 0), + RegisterWrite::new(0x15071, 0), + RegisterWrite::new(0x16058, 0), + RegisterWrite::new(0x10019, p.multisample_control), + RegisterWrite::new(0x100b1, g.macro_size), + RegisterWrite::new(0x16030, g.macro_size), + RegisterWrite::new(0x100d9, g.screen), + RegisterWrite::new(0x0a301, 0), + RegisterWrite::new(0x10791, 0xff_0200), + RegisterWrite::new(0x16098, p.heapmeta), + RegisterWrite::new(0x15109, p.scissor_array), + RegisterWrite::new(0x15101, p.depth_bias_array), + RegisterWrite::new(0x15021, p.aux_fb_flags), + RegisterWrite::new(0x15211, ((p.height as u64) << 32) | p.width as u64), + RegisterWrite::new(0x15049, p.aux_fb_page_count), + RegisterWrite::new(0x10051, p.tib_blocks), + RegisterWrite::new(0x15321, p.depth_dimensions), + RegisterWrite::new(0x15301, p.depth_clear_value_bits as u64), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15311, p.occlusion_query_base), + RegisterWrite::new(0x15319, p.depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + RegisterWrite::new(0x15351, 0), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x15231, 0), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15239, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15401, p.depth_stride), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15409, p.stencil_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15411, p.depth_aux_stride), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15419, p.stencil_aux_stride), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x16429, p.tilemap), + RegisterWrite::new(0x16060, p.layermeta), + RegisterWrite::new(0x16431, (4 * g.size1) << 24), + RegisterWrite::new(0x10039, p.tile_config), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16451, 0), + RegisterWrite::new(0x15359, 0), + RegisterWrite::new(0x100b8, 0x8860), + RegisterWrite::new(0x16461, p.aux_fb), + RegisterWrite::new(0x16090, p.aux_fb), + RegisterWrite::new(0x101e9, 0x1c), + RegisterWrite::new(0x160a8, 0), + RegisterWrite::new(0x16068, tile_state), + RegisterWrite::new(0x1a0a9, 0), + RegisterWrite::new(0x1a0b1, 0), + RegisterWrite::new(0x1a079, 0), + RegisterWrite::new(0x1a081, 0), + RegisterWrite::new(0x1a0d9, 0), + RegisterWrite::new(0x1a0e1, 0), + RegisterWrite::new(0x101c1, 0), + RegisterWrite::new(0x0d469, 0), + RegisterWrite::new(0x1a0f9, 8), + // Fragment analog of TA 0x0a5a1. Native derives its high word from the + // 3D AGXUMADataRec PTC/DUPM maximum at host descriptor +0xc00, copied + // through command +0x818. Linux has no owned pool-size input yet, so + // use the exact 0x60 zero-source fallback. Bit 3 retains one captured, + // workload-derived 0xd5 value only as a diagnostic. + RegisterWrite::new( + 0x0a5a9, + if p.native_ta_registers & 8 != 0 { + 0x0000_00d5_0040_0020 + } else { + 0x0000_0060_0040_0020 + }, + ), + RegisterWrite::new(0x0d429, 0x0000_0002_0000_0001), + RegisterWrite::new(0x160e0, p.fragment_lifecycle), + RegisterWrite::new(0x01499, p.fragment_lifecycle), + RegisterWrite::new(0x0a341, p.fragment_lifecycle), + // The fragment must use the same parameter-buffer owner as TA 1c830. + // Native TA::prepare stores its allocated HWPB ID at host +928; the + // same derived host descriptor is submitted through both channels. + // generate3DRegisterList reads that HOST field at 8a5c7cc. Its wrapper + // at 8a515d8 tail-calls with x1=raw command, x2=host descriptor: the + // unused channel argument in the mangled common-function name is NOT + // present in the optimized calling convention. Raw command +928 has + // a different producer and must not be used to infer this register. + RegisterWrite::new(0x1c838, p.ta_hardware_buffer_id as u64), + RegisterWrite::new(0x1ca28, p.cycle), + RegisterWrite::new(0x10211, work_key), + RegisterWrite::new(0x10420, work_key), + // AGX3DCommandDescriptor::prepare receives this ID from the USC pool's + // HardwareBufferIDManager and generate3DRegisterList ORs it with an + // optional GART payload and feature bit. Linux currently has neither + // optional source, so publish the retained USC FList owner's exact ID. + RegisterWrite::new(0x14048, p.usc_flist_hardware_buffer_id as u64), + RegisterWrite::new(0x14080, p.fragment_status | 1), + RegisterWrite::new(0x01731, 1), + RegisterWrite::new(0x16020, 1), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16068, 0x4_0000), + ]) +} + +pub(crate) fn partial_store_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_STORE_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15379, p.partial_store_pipeline_bind), + RegisterWrite::new(0x15381, p.partial_store_pipeline), + RegisterWrite::new(0x10039, p.tile_config), + RegisterWrite::new(0x15359, 0x20), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +pub(crate) fn partial_resume_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_RESUME_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15379, p.partial_store_pipeline_bind), + RegisterWrite::new(0x15381, p.partial_store_pipeline), + RegisterWrite::new(0x15369, p.partial_load_pipeline_bind), + RegisterWrite::new(0x15371, p.partial_load_pipeline), + RegisterWrite::new(0x10039, p.tile_config & 0xffff), + RegisterWrite::new(0x15359, 0x20), + RegisterWrite::new(0x15331, p.depth_buffer), + RegisterWrite::new(0x153c9, p.depth_aux_buffer), + RegisterWrite::new(0x15341, p.stencil_buffer), + RegisterWrite::new(0x153d9, p.stencil_aux_buffer), + RegisterWrite::new(0x15421, p.depth_stride), + RegisterWrite::new(0x15431, p.depth_aux_stride), + RegisterWrite::new(0x15429, p.stencil_stride), + RegisterWrite::new(0x15439, p.stencil_aux_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +pub(crate) fn partial_load_registers( + p: &G17pRenderParameters, +) -> [RegisterWrite; PARTIAL_LOAD_REGISTER_COUNT] { + [ + RegisterWrite::new(0x15369, p.partial_load_pipeline_bind), + RegisterWrite::new(0x15371, p.partial_load_pipeline), + RegisterWrite::new(0x10039, p.tile_config & 0xffff), + RegisterWrite::new(0x15309, (p.stencil_clear_value | 0x300) as u64), + RegisterWrite::new(0x15329, p.depth_buffer), + RegisterWrite::new(0x153c1, p.depth_aux_buffer), + RegisterWrite::new(0x15339, p.stencil_buffer), + RegisterWrite::new(0x153d1, p.stencil_aux_buffer), + RegisterWrite::new(0x15319, p.partial_depth_flags), + RegisterWrite::new(0x15349, 0x0404_0404), + ] +} + +fn put_u8(raw: &mut [u8], offset: usize, value: u8) { + raw[offset] = value; +} + +fn put_u16(raw: &mut [u8], offset: usize, value: u16) { + raw[offset..offset + 2].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u32(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +fn encode_registers(raw: &mut [u8], offset: usize, registers: &[RegisterWrite]) { + for (index, register) in registers.iter().enumerate() { + let entry = offset + index * REGISTER_BYTES; + put_u32(raw, entry, register.number); + put_u64(raw, entry + 4, register.value); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum RenderDescriptorKind { + Tiling, + Fragment, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderDescriptorObjects { + pub(crate) record_a: u64, + pub(crate) shared: u64, + pub(crate) record_b: u64, + pub(crate) zero: u64, +} + +impl RenderDescriptorObjects { + +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct RenderDescriptorMetadata { + pub(crate) context_id: u32, + pub(crate) submission_ordinal: u64, + /// Monotonic state tag allocated by the command buffer's 36-entry state + /// ring. This is neither the selected ring index nor any QID/HWID. Native + /// retains the full word in the selected state object and places only its + /// low six bits in the stage work key. + pub(crate) command_buffer_state_word: u32, + /// TA HardwareBufferID copied to firmware descriptor +0x18. + /// + /// Every working native descriptor corpus couples this field to register + /// 0x1c830 exactly (for example 2/2, 4/4, and 5/5). The same allocator + /// result is also PBState +0x0c and the descriptor-ring index consumed by + /// `halLoadPBDesc`. + pub(crate) ta_hardware_buffer_id: u32, + pub(crate) submit_sequence: u64, +} + +impl RenderDescriptorMetadata { +} + +/// Native's TA/3D register-list producer couples the command-buffer state tag +/// to the descriptor's allocated application-GART context. The physical QID, +/// QoS HardwareBufferID, descriptor-array slot and 36-entry state-ring index +/// are independent namespaces and must not be substituted here. +const fn render_work_key(context_id: u32, command_buffer_state_word: u32) -> u64 { + ((context_id as u64) << 8) | ((command_buffer_state_word & 0x3f) as u64) +} + +fn descriptor_work_ordinal(ordinal: u64) -> u32 { + (ordinal as u32).wrapping_add((ordinal / 2) as u32) +} + +fn write_common_descriptor( + kind: RenderDescriptorKind, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + registers: &[RegisterWrite], + raw: &mut [u8], +) { + let (selector, pointers, register_offset, pointer_gap) = match kind { + RenderDescriptorKind::Tiling => (0u32, 0x10usize, 0x60usize, 8usize), + RenderDescriptorKind::Fragment => (1u32, 0x20usize, 0xa0usize, 0usize), + }; + put_u32(raw, 0, selector); + put_u64(raw, 4, metadata.submit_sequence); + put_u32(raw, 0x0c, metadata.context_id); + + let addresses = [ + objects.record_a, + objects.shared, + objects.record_b, + objects.zero, + ]; + let mut offset = pointers; + for (index, address) in addresses.iter().enumerate() { + put_u64(raw, offset, *address); + offset += 8; + if index == 0 { + offset += pointer_gap; + } + } + encode_registers(raw, register_offset, registers); + + let work = descriptor_work_ordinal(metadata.submission_ordinal); + match kind { + RenderDescriptorKind::Tiling => { + put_u32(raw, 0x18, metadata.ta_hardware_buffer_id); + put_u32(raw, 0x48, work); + } + RenderDescriptorKind::Fragment => {} + } +} + +pub(crate) fn build_ta_descriptor_base( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + registers: &[RegisterWrite], + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < TA_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + if registers.is_empty() || registers.len() > (0x660 - 0x60) / REGISTER_BYTES { + return Err(RenderBuildError::BufferTooSmall); + } + raw[..TA_DESCRIPTOR_SIZE].fill(0); + write_common_descriptor( + RenderDescriptorKind::Tiling, + objects, + metadata, + registers, + raw, + ); + + // Like the fragment boundaries, these are generated for every native TA + // command (25G83 88bdfc4/88be090/88be1a4), not only a queue's first kick. + // The final three entries are 01740, 1c880, 1c898; firmware reads +38 and + // +3a/+3c to select the corresponding embedded RCE boundaries. + put_u16(raw, 0x38, (TA_REGISTER_COUNT - 2) as u16); + put_u16(raw, 0x3a, TA_REGISTER_COUNT as u16); + put_u16(raw, 0x3c, TA_REGISTER_COUNT as u16); + + let deflake = ta_user_offset(parameters.deflake_1)?; + let sampler_max = if parameters.sampler_count == 0 { + 0 + } else { + parameters.sampler_count + 1 + }; + put_u32( + raw, + 0x0768, + (((registers.len() * REGISTER_BYTES) as u32) << 16) | registers.len() as u32, + ); + put_u64(raw, 0x0780, parameters.tpc); + // Native copies RTMemory's geometry-derived TPC size beside its pointer + // on every submission (25G83 88fd970..974), independent of queue lifetime. + put_u64(raw, 0x0788, parameters.tpc_size); + put_u32(raw, 0x07d6, deflake as u32); + put_u32(raw, 0x0876, u32::MAX); + put_u64(raw, 0x087a, parameters.sampler_array); + put_u32(raw, 0x0882, parameters.sampler_count); + put_u32(raw, 0x0886, sampler_max); + // Present in 436/438 valid G17P TA records, including both exact 32x32 + // geometry matches. This belongs to the TA output/activation tail; leaving + // it zero is a structural mismatch before the linked 3D job is generated. + put_u32(raw, 0x0892, 1); + put_u8(raw, 0x0932, 0x44); + put_u8(raw, 0x093c, 1); + put_u8(raw, 0x094d, parameters.context_generation); + put_u64(raw, 0x08fe, parameters.ta_timestamp_start); + put_u64(raw, 0x0906, parameters.ta_timestamp_end); + // +0x090e/+0x0916 are caller-visible timestamp destinations. Firmware + // writes them only inside its timestamp aperture, so they stay zero; the + // completion publishes the pass envelope to userspace instead. + // AGXTAChannelSKSM::submitBuffer mirrors the same dependency/current pair + // consumed by the TA RCE program into the firmware command. These fields + // are deliberately unaligned in the native 0x9c0-byte command layout. + put_u32(raw, 0x086e, (parameters.lifecycle >> 32) as u32); + put_u64(raw, 0x08ce, parameters.lifecycle as u32 as u64); + Ok(()) +} + +fn put_embedded_registers( + raw: &mut [u8], + header_offset: usize, + program_offset: usize, + registers: &[RegisterWrite], +) { + let bytes = registers.len() * REGISTER_BYTES; + put_u32( + raw, + header_offset, + ((bytes as u32) << 16) | registers.len() as u32, + ); + encode_registers(raw, program_offset, registers); +} + +pub(crate) struct G17pFragmentRegisters<'a> { + pub(crate) main: &'a [RegisterWrite], + pub(crate) store: &'a [RegisterWrite], + pub(crate) resume: &'a [RegisterWrite], + pub(crate) load: &'a [RegisterWrite], +} + +pub(crate) fn build_fragment_descriptor_base( + parameters: &G17pRenderParameters, + objects: RenderDescriptorObjects, + metadata: RenderDescriptorMetadata, + programs: G17pFragmentRegisters<'_>, + raw: &mut [u8], +) -> Result<(), RenderBuildError> { + if raw.len() < FRAGMENT_DESCRIPTOR_SIZE { + return Err(RenderBuildError::BufferTooSmall); + } + if programs.main.is_empty() + || programs.main.len() > (0x6a0 - 0xa0) / REGISTER_BYTES + || programs.store.len() > (0xec0 - 0x7c0) / REGISTER_BYTES + || programs.resume.len() > (0x15e0 - 0xee0) / REGISTER_BYTES + || programs.load.len() > (0x1d00 - 0x1600) / REGISTER_BYTES + { + return Err(RenderBuildError::BufferTooSmall); + } + raw[..FRAGMENT_DESCRIPTOR_SIZE].fill(0); + let geometry = geometry(parameters)?; + let register_count = programs.main.len(); + write_common_descriptor( + RenderDescriptorKind::Fragment, + objects, + metadata, + programs.main, + raw, + ); + + put_u64(raw, 0x40, parameters.tilemap); + put_u64(raw, 0x48, parameters.multisample_control); + // Native copies the render target's sample count through host +0x8c8 + // into every fragment descriptor at +0x50 (25G83: 0x8839d80, + // 0x8837af8..0x8837b08). It is not first-work metadata. The UAPI has + // already encoded log2(samples) in UTILE_CONFIG bits [1:0]. + put_u32(raw, 0x50, 1u32 << (parameters.utile_config & 3)); + put_u32(raw, 0x54, geometry.macro_size as u32); + put_u32(raw, 0x68, parameters.merge_upper_x_bits); + put_u32(raw, 0x6c, parameters.merge_upper_y_bits); + put_u64(raw, 0x78, geometry.tiles_x * geometry.tiles_y); + + // RCE boundaries belong to every command, including retained queue kicks. + // Native emits these from the running register count (25G83 88c0740, + // 88c07ac, 88c08c8, 88c0970); firmware consumes +80/+82/+84/+88 when + // selecting an entry in this command's embedded program. The fixed recipe + // ends with 01731=1, 16020=1, 16020=0, 16068=40000. Boundaries are after + // 01731, after the first 16020, that same entry, and the complete program. + // Keeping these only in the cold wrapper left ordinary restages at zero. + put_u16(raw, 0x80, (register_count - 3) as u16); + put_u16(raw, 0x82, (register_count - 2) as u16); + put_u16(raw, 0x84, (register_count - 2) as u16); + put_u16(raw, 0x88, register_count as u16); + + put_u32( + raw, + 0x07a8, + (((register_count * REGISTER_BYTES) as u32) << 16) | register_count as u32, + ); + put_embedded_registers(raw, 0x0ec8, 0x07c0, programs.store); + put_embedded_registers(raw, 0x15e8, 0x0ee0, programs.resume); + put_embedded_registers(raw, 0x1d08, 0x1600, programs.load); + + put_u64(raw, 0x1d20, parameters.depth_bias_array); + put_u64(raw, 0x1d30, parameters.scissor_array); + put_u64(raw, 0x1d40, parameters.occlusion_query_base); + put_u64(raw, 0x1e78, parameters.load_pipeline_bind); + put_u64(raw, 0x1e80, parameters.load_pipeline); + put_u64(raw, 0x1ea8, parameters.partial_load_pipeline_bind); + put_u64(raw, 0x1eb0, parameters.partial_load_pipeline); + put_u32(raw, 0x1ec0, 0x0404_0404); + put_u64(raw, 0x1f38, parameters.tib_blocks); + put_u64(raw, 0x1f40, parameters.aux_fb_flags); + put_u32(raw, 0x1f48, parameters.width); + put_u32(raw, 0x1f4c, parameters.height); + put_u64(raw, 0x1f50, parameters.aux_fb_page_count); + put_u64(raw, 0x1f58, parameters.tile_config); + put_u32(raw, 0x1f78, parameters.store_pipeline_bind as u32); + put_u64(raw, 0x1f7c, parameters.store_pipeline); + put_u32(raw, 0x1f98, parameters.partial_store_pipeline_bind as u32); + put_u64(raw, 0x1f9c, parameters.partial_store_pipeline); + put_u32(raw, 0x1fa8, parameters.depth_clear_value_bits); + put_u64(raw, 0x1fac, (parameters.tib_blocks << 33) | 0x300); + // Native copies isMemorylessRender through the UMD render record +0x240, + // parsed record +0x1b8, and host descriptor +0x8bf into this byte. It is + // not a TIB-size predicate: native's memoryless policy also selects the + // shortened SPM phase programs. The admitted render UAPI represents + // memory-backed resources and has no memoryless-render policy or matching + // SPM-program construction. Keep that policy clear even for >8 TIB + // blocks; supporting memoryless renders requires modeling both together. + // Empty-tile processing remains independent in register 0x10039 bit 16. + put_u8(raw, 0x2100, 0); + // Exact G17P tail copied by AGX3DChannelSKSM::submitBuffer. +0x2104 is the + // accelerator control snapshot and +0x210c its low map nibble. +0x2110 is + // a separate render-setup field: prepare copies descriptor +0xc00 to setup + // +0x818, processRenderSetup carries it to +0x89c, and submitBuffer emits + // it here. Working Metal commands carry the invariant all-ones value. + put_u32(raw, 0x2104, parameters.gpc_perf_state_control); + // AGX3DChannelSKSM::submitBuffer mirrors the same dependency/current pair + // consumed by the fragment RCE program into the firmware command. + put_u32(raw, 0x2108, (parameters.fragment_lifecycle >> 32) as u32); + put_u32(raw, 0x210c, parameters.gpc_perf_state_map_low); + put_u32(raw, 0x2110, G17P_FRAGMENT_TAIL_SETUP_WORD); + // Native submitBuffer zeros +0x2114..+0x2123. Sampler state reaches the + // GPU through its real register/program path, not these command-tail + // lanes. The first two following flag lanes reproduce the working Neo + // shared-runtime snapshot; the final two come from per-command descriptor + // bytes and retain their constructor-zero values until those inputs are + // modeled. + put_u32( + raw, + 0x2124, + G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT[0], + ); + put_u32( + raw, + 0x2128, + G17P_NEO_FRAGMENT_SHARED_RUNTIME_FLAG_SNAPSHOT[1], + ); + put_u64(raw, 0x2168, parameters.fragment_lifecycle as u32 as u64); + put_u64(raw, 0x2198, parameters.fragment_timestamp_start); + put_u64(raw, 0x21a0, parameters.fragment_timestamp_end); + // +0x21a8/+0x21b0: caller-visible destinations, zero as in the TA record. + put_u8(raw, 0x21cc, 0x53); + put_u8(raw, 0x21d6, 1); + put_u8(raw, 0x21e7, parameters.context_generation); + put_u8(raw, 0x2208, 1); + put_u8(raw, 0x220c, 1); + put_u8(raw, 0x222d, g17p_frg_hwdsid_mode()); + Ok(()) +} + +/// FRG +0x222d mode byte: Linux has always written 1; all 160 native light +/// FRG captures hold 0. +fn g17p_frg_hwdsid_mode() -> u8 { + 1 +} + diff --git a/drivers/gpu/drm/asahi/m5/render_target.rs b/drivers/gpu/drm/asahi/m5/render_target.rs new file mode 100644 index 0000000000000..a277377cb5851 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/render_target.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Render-target allocation geometry and M5 viewport state. + +pub(crate) const PAGE_BYTES: usize = 0x4000; +pub(crate) const AUX_BYTES: usize = 0x8000; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct M5RenderTargetLayout { + pub(crate) tilemap_bytes: usize, + pub(crate) heapmeta_offset: usize, + pub(crate) heapmeta_bytes: usize, + pub(crate) target_bytes: usize, + pub(crate) tpc_bytes: usize, +} + +fn align(value: usize, alignment: usize) -> Option { + Some(value.checked_add(alignment - 1)? & !(alignment - 1)) +} + +impl M5RenderTargetLayout { + pub(crate) fn new( + width: u32, height: u32, layers: u16, utile_width: u8, utile_height: u8, + clusters: u32, + ) -> Option { + if !(1..=16384).contains(&width) || !(1..=16384).contains(&height) + || !(1..=2048).contains(&layers) || clusters == 0 + || !matches!((utile_width, utile_height), (32, 32) | (32, 16) | (16, 16)) { + return None; + } + // Same 4x4 macrotile geometry as the register builders. M5 packs its + // boundaries in ten-bit fields; that does not change allocation size. + let tiles_x = (width as usize + 31) / 32; + let tiles_y = (height as usize + 31) / 32; + let per_mtile = align((tiles_x + 3) / 4, 4)? + .checked_mul(align((tiles_y + 3) / 4, 4)?)?; + let utiles = (32 / utile_width as usize) * (32 / utile_height as usize); + let tilemap_bytes = align(5usize.checked_mul(per_mtile)?.checked_mul(utiles)?, 4)? + .checked_mul(16)?.checked_mul(layers as usize)?; + let heapmeta_offset = align(tilemap_bytes, 0x1000)?; + let heapmeta_bytes = if layers > 1 { 0x300 } else { 0x200 }; + let target_bytes = align(heapmeta_offset.checked_add(heapmeta_bytes)?, PAGE_BYTES)?; + let tpc_bytes = align(8usize.checked_mul(per_mtile)?.checked_mul(utiles)? + .checked_mul(16)?.checked_mul(layers as usize)? + .checked_mul(clusters as usize)?, PAGE_BYTES)?; + Some(Self { tilemap_bytes, heapmeta_offset, heapmeta_bytes, target_bytes, tpc_bytes }) + } +} + +fn half_float_bits(value: u32) -> u32 { + // The admitted integer dimensions are exactly representable in binary32. + let highest = 31 - value.leading_zeros(); + ((highest + 126) << 23) | ((value << (23 - highest)) & 0x7fffff) +} + +/// Native M5 deflake page: viewport at +0x9c0, command header 0x3000. +/// G17P uses +0x900/0xc00 and must not supply this page. The eight nonzero +/// dwords match the M5 25G76 resource capture; dimensions are caller-owned. +pub(crate) fn initialize_deflake(page: &mut [u8], width: u32, height: u32) -> Option<()> { + if page.len() != PAGE_BYTES || !(1..=16384).contains(&width) + || !(1..=16384).contains(&height) { return None; } + page.fill(0); + let x = half_float_bits(width); + let y = half_float_bits(height); + for (offset, value) in [(0x9c0, 0x3000), + (0x9c4, 0x80000000 | ((width + 31) / 32 - 1)), + (0x9c8, (height + 31) / 32 - 1), (0x9d0, x), (0x9d4, x), + (0x9d8, y), (0x9dc, y | 0x80000000), (0x9e4, 0x3f800000)] { + page[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + Some(()) +} + diff --git a/drivers/gpu/drm/asahi/m5/resources.rs b/drivers/gpu/drm/asahi/m5/resources.rs new file mode 100644 index 0000000000000..30d94f4d97148 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources.rs @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![allow(dead_code)] + +//! Retained RAM for the J813 G17G/25G76 initialization graph. +//! +//! Construction only: no processor start, mailbox publication, work submission +//! or DRM registration. The eventual boot session must retain this entire +//! owner until BOTH firmware processors are stopped, including on failure. +//! M5 UAT lifecycle admission remains separate from this memory layout. + +use super::mapped::MappedObject; +use crate::{driver::AsahiDrmDevice, m5::initdata as init, mmu}; +use crate::m5::completion::{graphics_pointer_head, MAX_GRAPHICS_GENERATION}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; +use kernel::sync::Arc; + +mod firmware_view; +mod graphics_memory; +mod graphics_usc; +mod parameter_buffer; +mod target_memory; +mod graphics_job; +mod graphics_queues; +mod application_compute; + +const KERNEL_BASE: u64 = 0xffff_fc20_0000_0000; +const PRIVATE_BASE: u64 = KERNEL_BASE + 0x2_0000; +const ROOTS_BASE: u64 = KERNEL_BASE + 0x1a_8000; +const SHARED_BASE: u64 = KERNEL_BASE + 0xc078_8000; +const FWCTL_BASE: u64 = KERNEL_BASE + 0x1c_0000; +const SENTINEL_BASE: u64 = KERNEL_BASE + 0xc07e_0000; +const PBDESC_BASE: u64 = KERNEL_BASE + 0x15e_0000; +const UMA_BASE: u64 = KERNEL_BASE + 0x15e_8000; + +/// Owning mappings are dropped before the UAT. Aliases retain the SAME GEM +/// backing; no firmware pointer refers to a copied snapshot or temporary map. +pub(crate) struct M5FirmwareMemory { + primary_root: MappedObject, + secondary_root: MappedObject, + shared: MappedObject, + private: MappedObject, + region_a: MappedObject, + region_c: MappedObject, + fwctl: MappedObject, + aon: MappedObject, + qos: MappedObject, + pbdesc: MappedObject, + uma: MappedObject, + completions: KVec, + aliases: KVec, + registers: KVec, + graphics: Option<(graphics_job::M5GraphicsJob, graphics_queues::M5GraphicsQueues)>, + application_compute: Option, + application_outer_head: u32, + report_counts: [[u32; 2]; 2], + trace_printed: [u32; 2], + compute_kick_observed: bool, + compute_dispatch_returned: bool, + graphics_stamp_flags: u128, + uat: Arc, +} + +impl M5FirmwareMemory { + /// Internal defaults for the initial M5 render-state envelope. The DRM + /// translator replaces all program/input addresses, and owned target + /// allocation replaces private storage before any queue publication. + pub(crate) fn reference_render_profile() -> + (crate::m5::render_desc::G17pRenderParameters, crate::m5::render::M5RenderControls) { + graphics_job::triangle_parameters() + } + + /// All fallible preparation happens before publication. The caller must + /// pass a prepared M5 UAT while both processors await initdata. The + /// caller retains its own Arc until both CPUs stop, including when this + /// unpublished graph fails construction. There is no Neo tuning fallback. + #[inline(never)] + pub(crate) fn new_with_uat( + dev: &AsahiDrmDevice, + uat: Arc, + boot_timestamp: u64, + ) -> Result> { + let memory = Self::allocate_with_uat(dev, uat, boot_timestamp)?; + fence(Ordering::SeqCst); + Ok(memory) + } + + #[inline(never)] + fn allocate_with_uat( + dev: &AsahiDrmDevice, + uat: Arc, + boot_timestamp: u64, + ) -> Result> { + let mut registers = KVec::new(); + for window in init::M5_25G76_REGISTER_WINDOWS { + let (physical, virtual_address, span) = window.mapped_pages().map_err(|_| EINVAL)?; + let protection = match window.flag { + 0 => mmu::PROT_FW_MMIO_RO, + 2 => mmu::PROT_FW_MMIO_RW, + _ => return Err(EINVAL), + }; + // Page-table writes only. No register contents are accessed. + registers.push(uat.kernel_vm().map_io(dev.as_ref().as_ref(), virtual_address, physical.try_into()?, + span.try_into()?, protection)?, GFP_KERNEL)?; + } + let fixed = |address, size, protection| MappedObject::new_wc_at( + dev, uat.kernel_vm(), address, size, protection); + // Install every fixed address before allocator-selected objects. + let mut private = fixed(PRIVATE_BASE, init::M5_25G76_PRIVATE_CLUSTER_SIZE, mmu::PROT_FW_SHARED_RW)?; + // Separate mappings retain the empty page between the two roots. + let mut primary_root = fixed(ROOTS_BASE, init::ROOT_SIZE_PRIMARY, mmu::PROT_FW_SHARED_RO)?; + let mut secondary_root = fixed(ROOTS_BASE + init::SECONDARY_ROOT_DELTA, + init::ROOT_SIZE_SECONDARY, mmu::PROT_FW_SHARED_RO)?; + let mut shared = fixed(SHARED_BASE, init::NATIVE_SHARED_CLUSTER_SIZE, mmu::PROT_FW_PRIV_RW)?; + let region_a = fixed(KERNEL_BASE + 0x1b_8000, init::REGION_A_SIZE, mmu::PROT_FW_SHARED_RO)?; + let mut region_c = fixed(KERNEL_BASE + 0x1a_0000, init::REGION_C_SIZE, mmu::PROT_FW_SHARED_RO)?; + let fwctl = fixed(FWCTL_BASE, 0x4000, mmu::PROT_FW_SHARED_RW)?; + // These two tables have zero-backed mappings at handoff. Mode6 + // prepares its own UMA slot before any compute publication. + let mut pbdesc = fixed(PBDESC_BASE, 0x4000, mmu::PROT_FW_SHARED_RW)?; + let mut uma = fixed(UMA_BASE, 0x4000, mmu::PROT_FW_SHARED_RW)?; + let mut aliases = KVec::new(); + for (object, address) in [(&mut pbdesc, init::REGION_VIEW_LOW_ADDRS[0]), + (&mut uma, init::REGION_VIEW_LOW_ADDRS[1])] { + aliases.push(object.map_alias_at(uat.kernel_lower_vm(), address, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + if uat.kernel_vm().translate_iova(object.iova())? + != uat.kernel_lower_vm().translate_iova(address)? { + return Err(EFAULT); + } + } + + let mut completions = KVec::new(); + let mut completion_bindings = [init::G17PKsmCompletionAliases { low: 0, high: 0 }; 4]; + for (ordinal, binding) in completion_bindings.iter_mut().enumerate() { + // The fixed alias pairs are measured on M5, including ordinals + // 1 and 3 which are absent from the older Neo initialization. + let low = 0x70_0184_8000 + ordinal as u64 * 0x2_8000; + let high = KERNEL_BASE + 0xc07e_8000 + ordinal as u64 * 0x2_8000; + let mut backing = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, + init::M5_25G76_COMPLETION_BACKING_SIZE, mmu::PROT_FW_PRIV_RW)?; + aliases.push(backing.map_alias_at(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..init::M5_25G76_COMPLETION_BACKING_SIZE).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset as u64)? + != uat.kernel_lower_vm().translate_iova(low + offset as u64)? { + return Err(EFAULT); + } + } + *binding = init::G17PKsmCompletionAliases { low, high }; + completions.push(backing, GFP_KERNEL)?; + } + // Keep allocator-selected objects below the high fixed graph, and + // above the PB/UMA tables. This cannot occupy the deferred blank + // sentinel address or its neighboring guard page. + let dynamic = KERNEL_BASE + 0x168_0000..KERNEL_BASE + 0x1000_0000; + let qos = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), dynamic.clone(), + 0xe40, mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW)?; + let mut aon = MappedObject::new_wc_in_range(dev, uat.kernel_vm(), dynamic, + init::M5_25G76_AON_PTD_SIZE, + mmu::UAT_PGSZ as u64, mmu::PROT_FW_PRIV_RW)?; + aon.with_bytes_mut(|out| init::encode_m5_25g76_aon_ptd(out).map_err(|_| EINVAL))?; + region_c.with_bytes_mut(|out| init::encode_region_c_for_profile( + init::RootProfile::J813G17G25G76, out).map_err(|_| EINVAL))?; + private.with_bytes_mut(|out| init::encode_m5_25g76_private_cluster( + FWCTL_BASE, out).map_err(|_| EINVAL))?; + + let addresses = init::SharedAddresses { + region_a: region_a.iova(), region_c: region_c.iova(), + hw_data_bundle: SHARED_BASE, hw_data_bundle_alloc: init::NATIVE_SHARED_CLUSTER_SIZE, + }; + let primary = init::InstanceAddresses { + root: ROOTS_BASE, main_config: SHARED_BASE + init::NATIVE_PRIMARY_MAIN_BUNDLE_OFFSET as u64, + status_a: PRIVATE_BASE + 0xee40, status_b: PRIVATE_BASE + 0x340, + secondary_extras: [0, 0], + }; + let secondary = init::InstanceAddresses { + root: ROOTS_BASE + init::SECONDARY_ROOT_DELTA, + main_config: SHARED_BASE + init::NATIVE_SECONDARY_MAIN_BUNDLE_OFFSET as u64, + status_a: PRIVATE_BASE + 0xc3100, status_b: 0, + secondary_extras: [primary.status_b + init::PRIMARY_STATUS_B_CONFIG_OFFSET as u64, + PRIVATE_BASE + 0x177400], + }; + let views = init::PrimaryRegionViews { + sentinel_high: SENTINEL_BASE, alias_high: [PBDESC_BASE, UMA_BASE], + }; + let trace_mask = crate::module_parameters::m5_fw_trace.value(); + if trace_mask & !0x203 != 0 { return Err(EINVAL); } + // Exact M5 GFX firmware 0x1fadc retains root+0xa8 (status A), + // then 0x3074..0x3088 copies status-A+0 into its trace flags. + // Bits 0 and 1 gate power/submission records; 0x6260 reads the + // NEXT byte and tests bit 1 (full-word bit 9) for recovery summary + // 0x909. Publish through the host field before either descriptor. + let native_trace = private.read_u32(primary.status_a)?; + private.write_u32(primary.status_a, native_trace | trace_mask)?; + shared.with_bytes_mut(|out| { + init::encode_m5_25g76_hw_data(completion_bindings, qos.iova(), + &mut out[..init::M5_25G76_HW_DATA_SIZE]).map_err(|_| EINVAL)?; + init::encode_m5_25g76_bundle_policy(init::M5PolicyBindings { + state: PRIVATE_BASE + 0x177480, aon_ptd: aon.iova(), boot_timestamp, + }, out).map_err(|_| EINVAL)?; + for (role, instance, grid, views) in [ + (init::InstanceRole::Primary, &primary, PRIVATE_BASE, Some(&views)), + (init::InstanceRole::Secondary, &secondary, PRIVATE_BASE + 0x1770c0, None), + ] { + let channels = init::derive_channel_table(role, grid, instance.status_a, + instance.main_config, SHARED_BASE); + let offset = (instance.main_config - SHARED_BASE) as usize; + init::encode_main_config_for_profile(init::RootProfile::J813G17G25G76, + role, &addresses, instance, &channels, views, + &mut out[offset..offset + init::MAIN_CONFIG_SIZE]).map_err(|_| EINVAL)?; + } + Ok(()) + })?; + primary_root.with_bytes_mut(|out| { + init::encode_root_for_profile(init::RootProfile::J813G17G25G76, + init::InstanceRole::Primary, &addresses, &primary, + out).map_err(|_| EINVAL) + })?; + secondary_root.with_bytes_mut(|out| { + init::encode_root_for_profile(init::RootProfile::J813G17G25G76, + init::InstanceRole::Secondary, &addresses, &secondary, + out).map_err(|_| EINVAL) + })?; + // Unlike PB/UMA, this future blank sentinel is unmapped at native + // handoff. It becomes zero-backed later in macOS initialization. + if uat.kernel_vm().translate_iova(SENTINEL_BASE).is_ok() { + return Err(EINVAL); + } + // Writes to all CPU WC mappings precede any future handoff. Only + // the two measured control openings are present; work rings and + // later compute/render descriptors remain unsubmitted. + fence(Ordering::SeqCst); + // Keep the retained graph off the kernel stack, including + // construction and error-return temporaries in the boot caller. + KBox::init(try_init!(Self { primary_root, secondary_root, shared, private, region_a, region_c, fwctl, aon, qos, + pbdesc, uma, completions, aliases, registers, graphics: None, + report_counts: [[0; 2]; 2], trace_printed: [0; 2], + compute_kick_observed: false, compute_dispatch_returned: false, + graphics_stamp_flags: 0, application_compute: None, + application_outer_head: 0, uat }), GFP_KERNEL) + } + + /// Prepare a first application job while retaining its actual VM. No + /// native program bundle or captured input/output pages are requested. + /// The caller has validated the command and all user mapping coverage. + #[allow(clippy::too_many_arguments)] + pub(crate) fn initialize_application_graphics( + &mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm, + parameters: crate::m5::render_desc::G17pRenderParameters, + controls: crate::m5::render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], + ) -> Result { + if self.graphics.is_some() { return Err(EBUSY); } + let job = graphics_job::M5GraphicsJob::from_vm( + dev, &self.uat, vm, parameters, controls, clusters, usc_required)?; + self.attach_graphics(dev, job) + } + + fn attach_graphics(&mut self, dev: &AsahiDrmDevice, job: graphics_job::M5GraphicsJob) -> Result { + if self.graphics.is_some() { return Err(EBUSY); } + let queues = graphics_queues::M5GraphicsQueues::new(dev, &self.uat, &job)?; + let pb = job.parameter_descriptor()?; + self.pbdesc.with_bytes_mut(|out| { + let base = graphics_job::PB_HARDWARE_ID as usize * 16; + for (index, word) in pb.into_iter().enumerate() { + out[base + index * 4..base + index * 4 + 4].copy_from_slice(&word.to_le_bytes()); + } + Ok(()) + })?; + let usc = job.uma_descriptor(); + self.uma.with_bytes_mut(|out| { + let base = graphics_usc::HARDWARE_ID as usize * 32; + for (index, word) in usc.into_iter().enumerate() { + out[base + index * 8..base + index * 8 + 8].copy_from_slice(&word.to_le_bytes()); + } + Ok(()) + })?; + self.graphics = Some((job, queues)); + Ok(()) + } + + pub(crate) fn quarantine_graphics(&self) { + if let Some((job, _)) = &self.graphics { job.quarantine(); } + } + + /// Drain M5 report channels using the native producer contracts. + /// GFX 0x6ca8..0x6d18 emits 256 x 0x48 event records; 0x17e18 + /// emits 512 x 0x48 trace records. In each cursor object the host owns + /// +0 and firmware owns +0x20. Copy before releasing each slot. + pub(crate) fn drain_reports(&mut self, dev: &AsahiDrmDevice) -> Result { + let mut recovery = false; + for (role, status_offset) in [0xee40u64, 0xc3100].into_iter().enumerate() { + for (channel, (cursor_offset, ring_offset, capacity)) in + [(0x40u64, 0x2c0u64, 256u32), (0x240, 0xa6ac0, 512)] + .into_iter().enumerate() { + let cursor = PRIVATE_BASE + status_offset + cursor_offset; + let ring = (status_offset + ring_offset) as usize; + for _ in 0..128 { + let consumer = self.private.read_u32(cursor)?; + let producer = self.private.read_u32(cursor + 0x20)?; + if consumer >= capacity || producer >= capacity { return Err(EIO); } + if consumer == producer { break; } + fence(Ordering::Acquire); + let offset = ring + consumer as usize * 0x48; + let mut raw = [0u8; 0x48]; + self.private.with_bytes_mut(|bytes| { + raw.copy_from_slice(bytes.get(offset..offset + 0x48).ok_or(EINVAL)?); + Ok(()) + })?; + let word = |off: usize| u32::from_le_bytes(raw[off..off + 4].try_into().unwrap()); + let qword = |off: usize| u64::from_le_bytes(raw[off..off + 8].try_into().unwrap()); + let count = self.report_counts[role][channel]; + if channel == 0 { + // Type 1 (graphics stamps) arrives for every job. + if word(0) == 1 { + dev_dbg!(dev.as_ref(), "M5_EVENT role={} ordinal={} type={} payload={:x?}\n", + role, count, word(0), [qword(4), qword(12), qword(20), qword(28), qword(36)]); + } else { + dev_dbg!(dev.as_ref(), "M5_EVENT role={} ordinal={} type={} payload={:x?}\n", + role, count, word(0), [qword(4), qword(12), qword(20), qword(28), qword(36)]); + } + recovery |= word(0) == 4; + if role == 0 && word(0) == 1 { + self.graphics_stamp_flags |= u128::from(qword(4)) + | (u128::from(qword(12)) << 64); + } + } else { + let args = [qword(12), qword(20), qword(28), qword(36)]; + // This one-shot diagnostic owns qid11/stamp1. The + // exact M5 trace point precedes the hardware kick; + // the admission probe also requires its later callback + // return trace. Firmware-private counters can be cached. + if role == 0 && word(0) == 5 && word(44) == 0x41 + && args == [11, 1, 2, 1] { + self.compute_kick_observed = true; + } + // 0x26aa4 invokes the pending-work callback, which + // dispatches tag14 and writes SGX+0x21018 at0x11f5c. + // 0x26ac4 emits 0x130 only after that callback returns. + if role == 0 && self.compute_kick_observed && word(0) == 5 + && word(44) == 0x130 && args == [4, 0, 0, 0] { + self.compute_dispatch_returned = true; + } + // Exact M5 0x24eb4/0x26754 emit zero-argument + // processing-loop brackets. Count and consume them + // without spending the diagnostic print budget. + let loop_marker = word(0) == 5 && args == [0; 4] + && matches!(word(44) & 0x00ff_ffff, 0x105 | 0x106); + if !loop_marker && self.trace_printed[role] < if role == 0 { 512 } else { 16 } { + dev_dbg!(dev.as_ref(), "M5_TRACE role={} ordinal={} timestamp={} type={} id={:#x} args={:x?}\n", + role, count, qword(4), word(0), word(44), args); + self.trace_printed[role] += 1; + } + } + self.report_counts[role][channel] = count.saturating_add(1); + fence(Ordering::SeqCst); + self.private.write_u32(cursor, (consumer + 1) % capacity)?; + } + } + } + Ok(recovery) + } + + pub(crate) fn graphics_status(&mut self) -> Result<([u32; 3], [[u32; 3]; 2], [u32; 5])> { + let mut outer = [0; 3]; + let base = init::WORK_STATE_GRID_OFFSETS[8]; + for index in 0..3 { + outer[index] = self.private.read_u32(PRIVATE_BASE + + (base + index * init::CHANNEL_STATE_SPACING) as u64)?; + } + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + Ok((outer, queues.counters()?, job.progress()?)) + } + + pub(crate) fn graphics_completion(&mut self) -> Result { + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + let progress = job.progress()?; + let scheduler = job.scheduler_status()?; + Ok(crate::m5::completion::M5GraphicsCompletion { + submitted: progress[0], completed: scheduler[0], linked: scheduler[1], + list: queues.completion_list()?, stamps: [progress[1], progress[3]], + signalled: self.graphics_stamp_flags, + }) + } + + pub(crate) fn graphics_pair_complete( + completion: crate::m5::completion::M5GraphicsCompletion, + generation: u32, + ) -> bool { + (1..=MAX_GRAPHICS_GENERATION).contains(&generation) + && completion.pair_complete_at(graphics_queues::JOB_LIST, generation * 0x100) + } + + pub(crate) fn graphics_installation_status(&mut self) -> Result<([[u64; 5]; 2], [u64; 2], [u64; 2])> { + self.graphics.as_mut().ok_or(EINVAL)?.1.installation_status() + } + + pub(crate) fn graphics_queues_installed(&mut self) -> Result { + self.graphics.as_mut().ok_or(EINVAL)?.1.installed() + } + + pub(crate) fn publish_graphics_install(&mut self) -> Result { + let (outer, inner, progress) = self.graphics_status()?; + if outer != [0; 3] || inner != [[0, 0, 1]; 2] + || progress != [2, 0, 0, 0, 0] { return Err(EBUSY); } + if self.graphics_installation_status()? != ([[0; 5]; 2], [0; 2], [0xffff, 0]) { + return Err(EBUSY); + } + self.publish_graphics_records(0, 1) + } + + pub(crate) fn publish_graphics_job(&mut self) -> Result { + let (outer, inner, progress) = self.graphics_status()?; + if outer != [2; 3] || inner != [[1; 3]; 2] + || progress != [2, 0, 0, 0, 0] { return Err(EBUSY); } + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + if !queues.installed()? { return Err(EBUSY); } + self.graphics_stamp_flags = 0; + job.publish_context()?; + queues.publish_draw()?; + self.publish_graphics_records(2, 4) + } + + /// Reuse only a fully retired pair, replacing its application descriptors + /// while preserving firmware-owned PB/USC state and the installed queues. + #[allow(clippy::too_many_arguments)] + /// A different submitting VM takes over the retired job in place: same + /// context slot, descriptor graph, queues and PB/USC pools. The compute + /// owner shares that context, so its operands follow the job. + fn hand_off_application(&mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let (job, _) = self.graphics.as_mut().ok_or(EINVAL)?; + if job.owns_vm(vm) { return Ok(()); } + job.rebind_vm(dev, vm).inspect_err(|e| pr_err!("M5: rebind_vm failed: {:?}\n", e))?; + job.publish_context().inspect_err(|e| pr_err!("M5: publish_context failed: {:?}\n", e))?; + Ok(()) + } + + pub(crate) fn publish_next_application_job( + &mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm, + parameters: crate::m5::render_desc::G17pRenderParameters, + controls: crate::m5::render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], generation: u32, + ) -> Result { + if !(2..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + let previous = generation - 1; + let (outer, inner, _) = self.graphics_status()?; + if outer != [self.application_outer_head; 3] + || inner != [[graphics_pointer_head(previous); 3]; 2] + || !Self::graphics_pair_complete(self.graphics_completion()?, previous) { + return Err(EBUSY); + } + self.hand_off_application(dev, vm)?; + let (job, queues) = self.graphics.as_mut().ok_or(EINVAL)?; + job.prepare_application_frame(dev, vm, parameters, controls, clusters, + usc_required, generation).inspect_err(|e| pr_err!("M5: prepare_application_frame failed: {:?}\n", e))?; + queues.publish_next_draw(job, generation).inspect_err(|e| pr_err!("M5: publish_next_draw failed: {:?}\n", e))?; + self.graphics_stamp_flags = 0; + self.publish_graphics_records(self.application_outer_head as usize, + graphics_pointer_head(generation) as u16) + } + + fn publish_graphics_records(&mut self, outer_index: usize, inner_head: u16) -> Result { + // Only recycle storage after every outer consumer reaches the old + // producer. The application caller also requires paired retirement. + if outer_index >= 256 || self.graphics_status()?.0 != [outer_index as u32; 3] { + return Err(EBUSY); + } + self.shared.with_bytes_mut(|out| { + for stage in 0..2 { + let offset = init::WORK_RING_BUNDLE_OFFSETS[8] + ((outer_index + stage) % 256) * 24; + let record = &mut out[offset..offset + 24]; + record.fill(0); + let address = graphics_queues::M5GraphicsQueues::queue_address(stage)?; + record[8..16].copy_from_slice(&address.to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&inner_head.to_le_bytes()); + record[22] = graphics_job::QUEUE_IDS[stage]; + record[23] = 1; + } + Ok(()) + })?; + fence(Ordering::SeqCst); + // Both stage-tagged SKSM queues travel through the CL_2 command ring. + let producer = init::WORK_STATE_GRID_OFFSETS[8] + 2 * init::CHANNEL_STATE_SPACING; + self.private.write_u32(PRIVATE_BASE + producer as u64, ((outer_index + 2) % 256) as u32)?; + self.application_outer_head = ((outer_index + 2) % 256) as u32; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn application_outer_head(&self) -> u32 { self.application_outer_head } + + /// Initial admission is deliberately one compute job after retired graphics. + /// No private diagnostic context, queue 11, or replacement UMA is installed. + pub(crate) fn initialize_application_compute(&mut self, dev: &AsahiDrmDevice, + vm: &mmu::Vm, command: &crate::m5::command::TranslatedComputeCommand, + graphics_generation: u32) -> Result { + if self.application_compute.is_some() || self.application_outer_head + 2 > 256 { + return Err(ENOTSUPP); + } + let (outer, inner, _) = self.graphics_status()?; + if graphics_generation == 0 || outer != [self.application_outer_head; 3] + || inner != [[graphics_pointer_head(graphics_generation); 3]; 2] + || !Self::graphics_pair_complete(self.graphics_completion()?, graphics_generation) { + return Err(EBUSY); + } + self.hand_off_application(dev, vm)?; + let job = &self.graphics.as_ref().ok_or(EINVAL)?.0; + self.application_compute = Some(application_compute::M5ApplicationCompute::new( + dev, &self.uat, job, command)?); + Ok(()) + } + + pub(crate) fn application_compute_installed(&mut self) -> Result { + let compute = self.application_compute.as_mut().ok_or(EINVAL)?; + Ok(compute.counters()? == [1; 3] && compute.installed()?) + } + + /// Publish one command-ring record for queue 12. `dispatch` false is the + /// installation record; true is dispatch generation `generation`, whose + /// inner head is the producer that generation's three pointers reach. + pub(crate) fn publish_application_compute(&mut self, dispatch: bool, + generation: u32) -> Result { + let outer_index = self.application_outer_head as usize; + if outer_index >= 256 || self.graphics_status()?.0 != [outer_index as u32; 3] { + return Err(EBUSY); + } + let inner_head = u16::try_from(application_compute::compute_pointer_head( + if dispatch { generation } else { 0 })).map_err(|_| EINVAL)?; + let compute = self.application_compute.as_mut().ok_or(EINVAL)?; + if dispatch { + // Preserve the retired graphics notifications for later render reuse. + self.graphics_stamp_flags &= !(1u128 << application_compute::QUEUE_ID); + compute.publish_dispatch(generation)?; + } else if compute.counters()? != [0, 0, 1] { return Err(EBUSY); } + let offset = init::WORK_RING_BUNDLE_OFFSETS[8] + outer_index * 24; + self.shared.with_bytes_mut(|out| { + let record = &mut out[offset..offset + 24]; + // Every outer consumer reached this slot (checked above), so its + // prior user, graphics or compute, has retired. Graphics records + // wrap this ring after 128 renders, so even the first compute + // record can land on a used slot. + record.fill(0); + record[8..16].copy_from_slice( + &application_compute::M5ApplicationCompute::queue_address().to_le_bytes()); + record[16..20].copy_from_slice(&2u32.to_le_bytes()); + record[20..22].copy_from_slice(&inner_head.to_le_bytes()); + record[22] = application_compute::QUEUE_ID; + record[23] = 1; + Ok(()) + })?; + fence(Ordering::SeqCst); + let head = ((outer_index + 1) % 256) as u32; + let producer = init::WORK_STATE_GRID_OFFSETS[8] + 2 * init::CHANNEL_STATE_SPACING; + self.private.write_u32(PRIVATE_BASE + producer as u64, head)?; + self.application_outer_head = head; + fence(Ordering::SeqCst); + Ok(()) + } + + /// Stage a further dispatch on the already installed compute queue. The + /// installed owner keeps its VM, USC and operand storage; only the command + /// stream and the generation's stamps change. + pub(crate) fn prepare_next_application_compute(&mut self, dev: &AsahiDrmDevice, + vm: &mmu::Vm, command: &crate::m5::command::TranslatedComputeCommand, + generation: u32) -> Result { + if !(2..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + self.hand_off_application(dev, vm)?; + let (job, _) = self.graphics.as_ref().ok_or(EINVAL)?; + self.application_compute.as_mut().ok_or(EINVAL)? + .prepare_next_dispatch(job, command, generation) + } + + pub(crate) fn application_compute_complete(&mut self, generation: u32) -> Result { + let (outer, _, _) = self.graphics_status()?; + Ok(outer == [self.application_outer_head; 3] + && self.application_compute.as_mut().ok_or(EINVAL)? + .complete(self.graphics_stamp_flags, generation)?) + } + + pub(crate) fn application_compute_status(&mut self) + -> Result<([u32; 3], crate::m5::completion::M5ComputeCompletion)> { + let compute = self.application_compute.as_mut().ok_or(EINVAL)?; + Ok((compute.counters()?, compute.completion(self.graphics_stamp_flags)?)) + } + + pub(crate) fn publish_idle_wake(&mut self) -> Result { + if self.control_counters()? != [[1; 3]; 2] { return Err(EBUSY); } + self.shared.with_bytes_mut(|out| { + let record = &mut out[0x1eac0..0x1eb00]; + if record.iter().any(|x| *x != 0) { return Err(EBUSY); } + record[..4].copy_from_slice(&0x0au32.to_le_bytes()); + Ok(()) + })?; + fence(Ordering::SeqCst); + let producer = init::WORK_STATE_GRID_OFFSETS[init::CONTROL_CHANNEL_INDEX] + + 2 * init::CHANNEL_STATE_SPACING; + self.private.write_u32(PRIVATE_BASE + producer as u64, 2)?; + fence(Ordering::SeqCst); + Ok(()) + } + + pub(crate) fn root(&self, secondary: bool) -> u64 { + if secondary { self.secondary_root.iova() } else { self.primary_root.iova() } + } + + /// Read the actual shared control state, not a detached host copy. The + /// offsets and three pointers per role are checked by the M5 capture + /// verifier against both native main objects and the private arena. + pub(crate) fn control_counters(&mut self) -> Result<[[u32; 3]; 2]> { + self.private.with_bytes_mut(|bytes| { + let mut counters = [[0; 3]; 2]; + for (role, grid) in [0usize, 0x1770c0].into_iter().enumerate() { + let base = grid + init::WORK_STATE_GRID_OFFSETS[init::CONTROL_CHANNEL_INDEX]; + for (index, value) in counters[role].iter_mut().enumerate() { + let offset = base + index * init::CHANNEL_STATE_SPACING; + let word = bytes.get(offset..offset + 4).ok_or(EINVAL)?; + // SAFETY: arena and native counter offsets are u32 aligned; + // this bounded WC mapping is retained while firmware runs. + *value = unsafe { core::ptr::read_volatile(word.as_ptr().cast::()) }; + } + } + fence(Ordering::Acquire); + Ok(counters) + }) + } + +} diff --git a/drivers/gpu/drm/asahi/m5/resources/application_compute.rs b/drivers/gpu/drm/asahi/m5/resources/application_compute.rs new file mode 100644 index 0000000000000..4f84edbd0875b --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/application_compute.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! One application compute dispatch sharing the retained graphics VM and USC. +//! +//! Queue 12 is separate from TA/fragment queues 10/11. All publication and +//! completion gates are explicit; constructing this owner does not submit it. + +use super::{graphics_job::M5GraphicsJob, graphics_usc, + firmware_view::FirmwareDataView, MappedObject}; +use crate::{driver::AsahiDrmDevice, m5::submission, m5::command, + m5::channel, m5::completion::{graphics_pointer_head, GRAPHICS_POINTER_COUNT, + MAX_GRAPHICS_GENERATION}, m5::compute, m5::sksm, mmu}; +use kernel::prelude::*; + +pub(super) const QUEUE_ID: u8 = 12; +const GRAPH: u64 = 0xffff_fc20_c008_0000; +const POINTERS: usize = 0x4000; +const ITEMS: usize = 0x8000; +/// Past the 0x500 command pointers the channel state declares, as for the +/// graphics queues, so the producer head wraps like theirs. +const CONFIG: usize = ITEMS + GRAPHICS_POINTER_COUNT as usize * 8; +const CONTROL: usize = 0xc000; +const _: () = assert!(CONFIG + 0x300 <= CONTROL); + +/// Installation publishes one command pointer; every dispatch generation then +/// appends the same three -- descriptor, channel config, kick -- so the +/// producer head runs 1, 4, 7, ... (mod 0x500) exactly as +/// graphics_pointer_head() does for queues 10/11. +pub(super) const fn compute_pointer_head(generation: u32) -> u32 { + graphics_pointer_head(generation) +} + +pub(super) const fn compute_generation_admitted(generation: u32) -> bool { + generation >= 1 && generation <= MAX_GRAPHICS_GENERATION +} + +const LIST: u64 = GRAPH + 0x10000; +const ENTRY: u64 = 0xffff_fc20_003d_0000; +const ENTRY_LOW: u64 = 0x70_0063_0000; +const DESC: u64 = 0xffff_fc20_c040_0000; +const DESC_LOW: u64 = 0x70_0038_0000; +const SCHEDULER: u64 = DESC + 0x4000; +const ZERO: u64 = DESC + 0x8000; +const SIGNALS: u64 = DESC + 0x10000; + +pub(super) struct M5ApplicationCompute { + aliases: KVec, + descriptor: MappedObject, + graph: MappedObject, + entry: MappedObject, + list: MappedObject, + signals: MappedObject, + /// GPU address of the job's compute storage. + storage: u64, + /// The operands mapped into the VM that took the graphics job over. + firmware: FirmwareDataView, +} + +impl M5ApplicationCompute { + pub(super) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, job: &M5GraphicsJob, + command: &command::TranslatedComputeCommand) -> Result { + // The adapter excludes helpers, timestamps, legacy sampler registers + // and the Neo add3 proof ABI before allocating or publishing anything. + if command.sampler_heap != 0 || command.sampler_count != 0 + || command.g17p_add3_buffers.is_some() { return Err(ENOTSUPP); } + // Validate observation support and vacancy before publishing a queue. + let firmware = FirmwareDataView::new(dev)?; + if firmware.queue_registration(QUEUE_ID)? != ([0; 5], 0) { return Err(EBUSY); } + // 128 KiB operands, a preemption page and a zero robustness page. + // Their addresses come from the application's reserved driver arena. + let storage = job.compute_storage(); + let mut descriptor = MappedObject::new_wc_at(dev, uat.kernel_vm(), DESC, + 0x10000, mmu::PROT_FW_PRIV_RW)?; + let mut entry = MappedObject::new_wc_at(dev, uat.kernel_vm(), ENTRY, + 0x20000, mmu::PROT_FW_PRIV_RW)?; + let mut aliases = KVec::new(); + for (object, low, size) in [(&mut descriptor, DESC_LOW, 0x10000), + (&mut entry, ENTRY_LOW, 0x20000)] { + aliases.push(object.map_alias_at(uat.kernel_lower_vm(), low, + mmu::PROT_GPU_SHARED_RW)?, GFP_KERNEL)?; + for offset in (0..size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(object.iova() + offset as u64)? + != uat.kernel_lower_vm().translate_iova(low + offset as u64)? { + return Err(EFAULT); + } + } + } + let mut signals = MappedObject::new_wc_at(dev, uat.kernel_vm(), SIGNALS, + 0x4000, mmu::PROT_FW_SHARED_RW)?; + // The submitted count is the host's and stays put; firmware republishes + // the completed count into the scheduler record each generation. + signals.write_u32(SIGNALS, 1)?; + Self::encode_descriptor(&mut descriptor, job, command, storage, 1)?; + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), GRAPH, + 0x10000, mmu::PROT_FW_PRIV_RW)?; + let mut list = MappedObject::new_wc_at(dev, uat.kernel_vm(), LIST, + 0x4000, mmu::PROT_FW_SHARED_RW)?; + list.with_bytes_mut(|out| { put64(out, 8, LIST); Ok(()) })?; + let prepared = Self::prepare_entry(1)?; + entry.with_bytes_mut(|out| { + let offset = prepared.offset as usize; + out[offset..offset + prepared.bytes.len()].copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let configuration = |install| channel::encode_m5_channel_config( + channel::M5ChannelConfig { + ring_low: ENTRY_LOW, ring_high: ENTRY, queue_id: QUEUE_ID, priority: 2, + install_queue: install, gpu_context: job.context_id(), + event: if install { None } else { Some(channel::M5ChannelEvent { + address: job.usc_control(), timestamp: 1, index: graphics_usc::HARDWARE_ID, + }) }, + channel_control: GRAPH + CONTROL as u64, update_scheduler: install, + scheduler_index: 0, channel_stamp_index: 0, + update_context: !install, owner_context: if install { 0 } else { 1 }, + extended_command: false, + }).map_err(|_| EINVAL); + let install = configuration(true)?; + let config = configuration(false)?; + let kick = Self::prepare_kick(1)?; + graph.with_bytes_mut(|out| { + let reset = submission::encode_m5_25g76_channel_state_reset( + submission::M5ChannelStateResetValues { + uncached_state: GRAPH + POINTERS as u64, + command_pointer_ring: GRAPH + ITEMS as u64, gpu_va_10: LIST, + word_48: 0, gpu_va_9c: GRAPH + CONTROL as u64, + }); + let queue = usize::from(QUEUE_ID) * 0xc0; + out[queue..queue + 0xc0].copy_from_slice(&reset); + for offset in [0x28, 0x2c, 0x40] { put32(out, queue + offset, 2); } + out[queue + 0x36..queue + 0x38].fill(0xff); + put32(out, POINTERS + 0x50, u32::MAX); + put32(out, POINTERS + 0x60, 0x500); + out[CONTROL..CONTROL + 2].fill(0xff); + out[CONTROL + 5] = 1; + out[CONTROL + 0x26] = 2; + out[CONTROL + 0x33] = 0xff; + for (offset, bytes) in [(CONFIG, &install[..]), (CONFIG + 0x100, &config[..]), + (CONFIG + 0x200, &kick[..])] { + out[offset..offset + bytes.len()].copy_from_slice(bytes); + } + for (index, address) in [GRAPH + CONFIG as u64, DESC, + GRAPH + CONFIG as u64 + 0x100, GRAPH + CONFIG as u64 + 0x200] + .into_iter().enumerate() { put64(out, ITEMS + index * 8, address); } + put32(out, POINTERS + 0x40, 1); + Ok(()) + })?; + Ok(Self { aliases, descriptor, graph, entry, list, signals, storage, + firmware }) + } + + pub(super) fn queue_address() -> u64 { GRAPH + QUEUE_ID as u64 * 0xc0 } + + /// Re-encode the whole descriptor for one dispatch generation. The + /// scheduler record is cleared and re-seeded exactly as + /// M5GraphicsJob::rebuild_retired_frame() does for queues 10/11: firmware + /// republishes the completed count into it, so a stale one would satisfy + /// the next generation's completion gate before the dispatch even ran. + fn encode_descriptor(descriptor: &mut MappedObject, job: &M5GraphicsJob, + command: &command::TranslatedComputeCommand, storage: u64, + generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + descriptor.with_bytes_mut(|out| { + out[0x4000..0x4000 + 0xc8].fill(0); + put64(out, 0x4000, SIGNALS); + put32(out, 0x4010, 0x50); + compute::encode_m5_compute_descriptor(compute::M5ComputeDescriptor { + registers: compute::M5ComputeParameters { + preempt_base: storage + 0x20000, + cdm_base: command.control_stream_base, + dispatch_identity: 0x0000_0001_0000_0001, + context_id: job.context_id(), work_ordinal: 0, + robustness: storage + 0x24000, + operand_state_base: storage, + private_pool_hw_id: u64::from(graphics_usc::HARDWARE_ID), + descriptor_control_530: 0x54b, descriptor_control_538: 0xf5, + descriptor_flag_60a: false, descriptor_flag_60b: false, + }, + submit_sequence: 0, scheduler_record: SCHEDULER, + descriptor_low_alias: DESC_LOW, + cdm_terminator: command.control_stream_terminator, + dispatch_stamps: [SIGNALS + 0x40, SIGNALS + 0x80], + event_machine_stamp: generation * 0x100, queue_id: u32::from(QUEUE_ID), + queue_ordinal: 0, has_event: false, submission_index: generation, + status: [SIGNALS + 0xc0, SIGNALS + 0xc8], + shared_control: job.usc_control(), shared_control_enabled: true, + event_metadata: [0; 16], zero_page: ZERO, owner_id: 1, + descriptor_work_ordinal: 0, + }, &mut out[..0x1000]).map_err(|_| EINVAL) + }) + } + + fn prepare_entry(generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + let stamp = u64::from(generation); + sksm::prepare_m5_sksm_entry(sksm::M5SksmQueue { + id: QUEUE_ID, priority: 2, current_stamp: stamp, submitted: 0, + previous_stamp: if generation == 1 { None } else { Some(stamp - 1) }, + }, sksm::M5SksmOperands { + barriers: &[], first_predecessor: 0, implicit_dependency_type: 1, + mcache: None, flag_bit1: false, flag_bit61: false, flag_bit63: false, + payload: [DESC, Self::queue_address()], qos_byte_32: 3, qos_class_40: 8, + qos_word_48: 0xffff, event_mask: [0; 4], rce_kind: 0, + rce: [sksm::M5SksmRce { address: DESC_LOW + 0x40, tag: 37 }, + sksm::M5SksmRce { address: DESC_LOW + 0x760, tag: 4 }, + sksm::M5SksmRce { address: 0, tag: 0 }, + sksm::M5SksmRce { address: 0, tag: 0 }], + auxiliary: 0x3f_ffff_ffff_ffff, + }).map_err(|_| EINVAL) + } + + fn prepare_kick(generation: u32) -> Result<[u8; 21]> { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + sksm::encode_m5_sksm_add_kicks(QUEUE_ID, 2, u64::from(generation), 1) + .map_err(|_| EINVAL) + } + + /// Stage a further dispatch on the installed queue. Nothing here is + /// published: the producer head moves only in publish_dispatch(). + pub(super) fn prepare_next_dispatch(&mut self, job: &M5GraphicsJob, + command: &command::TranslatedComputeCommand, generation: u32) -> Result { + if generation < 2 || !compute_generation_admitted(generation) { return Err(EINVAL); } + if self.counters()? != [compute_pointer_head(generation - 1); 3] + || !self.installed()? { return Err(EBUSY); } + Self::encode_descriptor(&mut self.descriptor, job, command, + self.storage, generation)?; + let prepared = Self::prepare_entry(generation)?; + self.entry.with_bytes_mut(|out| { + let offset = prepared.offset as usize; + let slot = &mut out[offset..offset + prepared.bytes.len()]; + // Every preceding dispatch has retired, including the prior user + // of this 256-entry SKSM slot. Preserve the first-use check. + if generation < 256 && slot.iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + slot.copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let kick = Self::prepare_kick(generation)?; + self.graph.with_bytes_mut(|out| { + out[CONFIG + 0x200..CONFIG + 0x200 + kick.len()].copy_from_slice(&kick); + for (index, address) in [DESC, GRAPH + CONFIG as u64 + 0x100, + GRAPH + CONFIG as u64 + 0x200].into_iter().enumerate() { + let absolute = 1 + (generation as usize - 1) * 3 + index; + let offset = ITEMS + (absolute % GRAPHICS_POINTER_COUNT as usize) * 8; + if absolute < GRAPHICS_POINTER_COUNT as usize + && out[offset..offset + 8].iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + put64(out, offset, address); + } + Ok(()) + }) + } + + pub(super) fn counters(&mut self) -> Result<[u32; 3]> { + let mut out = [0; 3]; + for (index, offset) in [0, 0x30, 0x40].into_iter().enumerate() { + out[index] = self.graph.read_u32(GRAPH + (POINTERS + offset) as u64)?; + } + Ok(out) + } + + pub(super) fn installed(&mut self) -> Result { + let (slot, scheduler) = self.firmware.queue_registration(QUEUE_ID)?; + let header = self.graph.read_u32(GRAPH + CONTROL as u64)?; + let bitmap = self.graph.read_u64(GRAPH + CONTROL as u64 + 0x12)?; + Ok(slot[0] & 1 != 0 && slot[1] == ENTRY && slot[2] == ENTRY_LOW + && slot[3] == 4u64 << 32 && slot[4] as u32 == 2 + && scheduler == GRAPH + CONTROL as u64 + && header & 0xff != 0xff && (header >> 8) & 0xff != 0xff + && (header >> 16) & 0xff == 4 && bitmap == 1u64 << QUEUE_ID) + } + + pub(super) fn publish_dispatch(&mut self, generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + if self.counters()? != [compute_pointer_head(generation - 1); 3] + || !self.installed()? { return Err(EBUSY); } + self.graph.write_u32(GRAPH + POINTERS as u64 + 0x40, + compute_pointer_head(generation)) + } + + pub(super) fn completion(&mut self, signalled: u128) + -> Result { + Ok(crate::m5::completion::M5ComputeCompletion { + submitted: self.signals.read_u32(SIGNALS)?, + completed: self.descriptor.read_u32(SCHEDULER + 0xc)?, + linked: self.descriptor.read_u32(SCHEDULER + 0x94)?, + list: [self.list.read_u64(LIST)?, self.list.read_u64(LIST + 8)?], + stamp: self.signals.read_u32(SIGNALS + 0x40)?, signalled, + }) + } + + pub(super) fn complete(&mut self, signalled: u128, generation: u32) -> Result { + if !compute_generation_admitted(generation) { return Err(EINVAL); } + Ok(self.counters()? == [compute_pointer_head(generation); 3] + && self.completion(signalled)? + .complete_at(LIST, QUEUE_ID, generation * 0x100)) + } +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} diff --git a/drivers/gpu/drm/asahi/m5/resources/firmware_view.rs b/drivers/gpu/drm/asahi/m5/resources/firmware_view.rs new file mode 100644 index 0000000000000..54ed6c966ac76 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/firmware_view.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Read-only view of the GFX firmware's DATA segment (the `gfx-data` reserved region). + +use crate::driver::AsahiDrmDevice; +use core::sync::atomic::{fence, Ordering}; +use kernel::{c_str, io, prelude::*}; + +pub(super) struct FirmwareDataView { + map: io::mem::Mem, + cache_line: usize, +} + +// SAFETY: This owns a fixed mapping of reserved RAM, read only through +// bounded volatile accesses. It neither transfers ownership nor writes DATA. +unsafe impl Send for FirmwareDataView {} +// SAFETY: Concurrent read-only volatile observations do not mutate host state. +unsafe impl Sync for FirmwareDataView {} + +impl FirmwareDataView { + pub(super) fn new(dev: &AsahiDrmDevice) -> Result { + let resource = dev.as_ref().as_ref().of_node().ok_or(EINVAL)? + .reserved_mem_region_to_resource_byname(c_str!("gfx-data"))?; + if resource.size() != 0x134000 || resource.start() < 0x100_0000_0000 + || resource.start() & 0x3fff != 0 + || resource.start().checked_add(resource.size()).ok_or(EOVERFLOW)? > 0x104_0000_0000 + || 0x127430 - 0x4c000 + 13 * 0x28 > resource.size() as usize + || 0x1257e0 - 0x4c000 + 13 * 8 > resource.size() as usize + || 0x178458 - 0x4c000 + 8 > resource.size() as usize { + return Err(EINVAL); + } + let ctr: u64; + // SAFETY: CTR_EL0 is an architectural read-only cache geometry register. + unsafe { core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)) }; + let cache_line = 4usize << ((ctr >> 16) & 0xf); + if cache_line > 0x4000 { return Err(EINVAL); } + // SAFETY: WB matches reserved System RAM. The host never writes this + // extent; observations invalidate covering lines before volatile reads. + let map = unsafe { io::mem::Mem::try_new(resource, io::mem::MemFlag::WB.into()) }?; + fence(Ordering::SeqCst); + Ok(Self { map, cache_line }) + } + + fn words(&self, offset: usize) -> [u64; N] { + // All callers use constructor-checked queue, scheduler or power extents. + let line = self.cache_line; + let start = offset & !(line - 1); + let end = (offset + N * 8 + line - 1) & !(line - 1); + for offset in (start..end).step_by(line) { + // SAFETY: Whole covering lines are inside the retained DATA view. + // The host never dirties them; do not clean stale data to firmware. + unsafe { core::arch::asm!("dc ivac, {addr}", + addr = in(reg) self.map.ptr().add(offset), + options(nostack, preserves_flags)) }; + } + // SAFETY: Complete invalidations before volatile observations. + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; + let mut out = [0; N]; + for (index, value) in out.iter_mut().enumerate() { + // SAFETY: The constructor bounds-checks every supported extent. + *value = unsafe { core::ptr::read_volatile( + self.map.ptr().add(offset + index * 8).cast::()) }; + } + fence(Ordering::Acquire); + out + } + + pub(super) fn queue_registration(&self, queue_id: u8) -> Result<([u64; 5], u64)> { + // TA/fragment own 10/11; application compute owns 12. The constructor + // bounds both complete tables through the last admitted queue. + if !(10..=12).contains(&queue_id) { return Err(EINVAL); } + Ok((self.words::<5>(0x127430 - 0x4c000 + queue_id as usize * 0x28), + self.words::<1>(0x1257e0 - 0x4c000 + queue_id as usize * 8)[0])) + } +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_job.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_job.rs new file mode 100644 index 0000000000000..0245811e9b5db --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_job.rs @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned TA/fragment resource graph for the bounded native-program test. +//! Construction leaves roots and queues unpublished. Submission wiring must +//! retain this owner through completion, faults and firmware shutdown. + +use super::{graphics_memory::M5GraphicsMemory, graphics_usc, MappedObject}; +use super::parameter_buffer::{M5ParameterBuffer, BLOCK_BYTES, PAGE_LIST_BYTES, PAGE_COUNT}; +use super::target_memory::M5TargetMemory; +use crate::{driver::AsahiDrmDevice, m5::render_desc, m5::render, mmu}; +use crate::m5::completion::MAX_GRAPHICS_GENERATION; +use kernel::prelude::*; + +const HIGH: u64 = 0xffff_fc20_c100_0000; +const LOW: u64 = 0x70_0030_0000; +const BYTES: usize = 0x40000; +const TA: usize = 0; +const FRAGMENT: usize = 0x4000; +const SCHEDULER: usize = 0x8000; +const SIGNALS_HIGH: u64 = HIGH + 0x80000; +const SUBMITTED: usize = 0; +const PM: usize = 0xc000; +const PM_SHARED: usize = 0x10000; +const PM_COUNTER: usize = 0x14000; +const PM_BLOCKS: usize = 0x18000; +const USC: usize = 0x1c000; +const USC_INNER: usize = 0x20000; +const USC_STATISTICS: usize = 0x24000; +const SCENES: usize = 0x30000; +const SCENE_ZERO: usize = SCENES + 80 * 0x80; +const TIMESTAMPS: usize = 0x38000; +const STAMPS_A: usize = 0x4000; +const STAMPS_B: usize = 0x8000; +const PAGE_LIST_HIGH: u64 = HIGH + 0x40000; +const METRICS_HIGH: u64 = HIGH + 0x60000; +const USER_BASE: u64 = 0x10_0000_0000; +pub(super) const PB_HARDWARE_ID: u32 = 0; +pub(super) const QUEUE_IDS: [u8; 2] = [10, 11]; +// Match Mesa's framebuffer dimensions; target allocations follow each size. +// Parameter-pool growth for more complex scenes remains separate. +use crate::m5::command::M5_MAX_RENDER_DIMENSION as MAX_APPLICATION_DIMENSION; +/// Retired targets kept for reuse. A video player cycles through a handful of +/// pass sizes every frame and a compositor's frames alternate with it, so +/// allocating per size change put several milliseconds on most jobs. +const TARGET_CACHE_ENTRIES: usize = 8; +/// Compute operands (128 KiB), a preemption page and a zero robustness page. +const COMPUTE_STORAGE_BYTES: usize = 0x28000; + +pub(super) struct M5GraphicsJob { + // The first owner's context retracts its roots before page release. + program: M5GraphicsMemory, + parameter_buffer: M5ParameterBuffer, + /// Storage of application compute, reserved with the parameter buffer so that its + /// address is free in every VM the job is handed to. + compute_storage: MappedObject, + target_memory: M5TargetMemory, + aliases: KVec, + // PB/USC pool views in a VM this job was handed to; empty for the first + // owner, whose pools were allocated in its own VM. + vm_aliases: KVec, + graph_alias: mmu::KernelMapping, + graph: MappedObject, + signals: MappedObject, + descriptor_template: KVec, + scene_template: KVec, + generation: u32, + target_shape: [u32; 5], + /// The VM `target_memory` lives in; a handoff leaves it behind. + target_vm: u64, + /// Retired targets by (VM, shape), least recently used first. + target_cache: KVec<(u64, [u32; 5], M5TargetMemory)>, +} + +impl M5GraphicsJob { + + /// Build a job from the submitting VM and translated application state. + /// No captured program is loaded; the caller supplies stage controls and + /// USC requirements derived for its program. Submission remains explicit. + #[allow(clippy::too_many_arguments)] + pub(super) fn from_vm( + dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, + parameters: render_desc::G17pRenderParameters, controls: render::M5RenderControls, + clusters: u32, usc_required: [u64; 2], + ) -> Result { + let program = M5GraphicsMemory::from_vm(dev, uat, vm)?; + Self::from_memory(dev, uat, program, parameters, controls, clusters, usc_required) + } + + #[allow(clippy::too_many_arguments)] + fn from_memory( + dev: &AsahiDrmDevice, uat: &mmu::Uat, program: M5GraphicsMemory, + mut p: render_desc::G17pRenderParameters, controls: render::M5RenderControls, + clusters: u32, usc_required: [u64; 2], + ) -> Result { + if !(1..=MAX_APPLICATION_DIMENSION).contains(&p.width) + || !(1..=MAX_APPLICATION_DIMENSION).contains(&p.height) + || p.layers != 1 || clusters != 1 { + return Err(ENOTSUPP); + } + let mut parameter_buffer = M5ParameterBuffer::new_reference(dev, &program)?; + let compute_storage = program.allocate_compact( + dev, COMPUTE_STORAGE_BYTES, 0x8000, mmu::PROT_GPU_SHARED_RW)?; + let target_memory = M5TargetMemory::new(dev, &program, &p, clusters)?; + target_memory.apply(&mut p); + p.cycle = parameter_buffer.scene_cycle(); + // Keep the PM register and firmware scene descriptor on the same + // owned metrics slot. The captured address is unmapped in a new VM. + p.record_index = render::parameter_buffer_metrics_record( + parameter_buffer.metrics() + 0x40).map_err(|_| EINVAL)?; + p.ta_hardware_buffer_id = PB_HARDWARE_ID; + p.usc_flist_hardware_buffer_id = graphics_usc::HARDWARE_ID; + p.ta_timestamp_start = HIGH + TIMESTAMPS as u64; + p.ta_timestamp_end = p.ta_timestamp_start + 8; + p.fragment_timestamp_start = p.ta_timestamp_start; + p.fragment_timestamp_end = p.ta_timestamp_end; + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), HIGH, + BYTES, mmu::PROT_FW_PRIV_RW)?; + // Native submitted totals and host-visible stamps are uncached in + // firmware. Keep them separate from the cached descriptor graph. + let mut signals = MappedObject::new_wc_at(dev, uat.kernel_vm(), SIGNALS_HIGH, + 0xc000, mmu::PROT_FW_SHARED_RW)?; + signals.write_u32(SIGNALS_HIGH + SUBMITTED as u64, 2)?; + let graph_alias = graph.map_alias_at(uat.kernel_lower_vm(), LOW, + mmu::PROT_GPU_SHARED_RW)?; + for offset in (0..BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(HIGH + offset as u64)? + != uat.kernel_lower_vm().translate_iova(LOW + offset as u64)? { + return Err(EFAULT); + } + } + let aliases = parameter_buffer.firmware_aliases( + uat, program.vm(), PAGE_LIST_HIGH, METRICS_HIGH, + )?; + let block_page_bases = parameter_buffer.block_page_bases()?; + graph.with_bytes_mut(|out| { + put64(out, SCHEDULER, SIGNALS_HIGH + SUBMITTED as u64); + put32(out, SCHEDULER + 0x10, 0x50); + // Two submitted stage records; firmware compares its completed + // count with this total, then unlinks the scheduler. + for (block, base) in block_page_bases.into_iter().enumerate() { + put32(out, PM_BLOCKS + block * 8, base); + } + put32(out, PM_SHARED, 3); + put32(out, PM_SHARED + 4, 3); + put32(out, PM_SHARED + 0x60, 1); + put32(out, PM_COUNTER, 1); + for (offset, value) in [(0xc, PB_HARDWARE_ID), (0x30, PAGE_LIST_BYTES as u32), + (0x34, PAGE_COUNT), (0x38, 1650), (0x3c, 3), (0x54, PAGE_COUNT - 1), + (0x58, BLOCK_BYTES as u32), (0x7c, 1650 * 16), (0x80, 550 * 16)] { + put32(out, PM + offset, value); + } + for (offset, address) in [(0x20, PAGE_LIST_HIGH), (0x28, parameter_buffer.page_list()), + (0x44, HIGH + PM_BLOCKS as u64), (0x4c, HIGH + PM_SHARED as u64), + (0x64, HIGH + PM_COUNTER as u64), (0x84, parameter_buffer.discard_offset())] { + put64(out, PM + offset, address); + } + // This sole job owns scene 1. M5 metrics use 64-byte spacing, + // unlike Neo dwords. Other reserved scenes remain unassigned; + // do not guess a scratch-ring wrap rule from a single draw. + let selected_scene = SCENES + 0x80; + put64(out, selected_scene, parameter_buffer.metrics() + 0x40); + put64(out, selected_scene + 8, METRICS_HIGH + 0x40); + put64(out, selected_scene + 0x28, parameter_buffer.scene_cycle()); + put64(out, selected_scene + 0x40, HIGH + PM_SHARED as u64 + 0x40); + put64(out, SCENES + 0x80 + 0x4c, 1); + put32(out, USC_INNER, 1); + program.usc().encode_initial_owner(HIGH + USC_INNER as u64, + &mut out[USC..USC + 0x70])?; + + Self::encode_descriptors(out, &p, &controls, program.context_id(), usc_required)?; + Ok(()) + })?; + let mut descriptor_template = KVec::new(); + let mut scene_template = KVec::new(); + graph.with_bytes_mut(|out| { + descriptor_template.extend_from_slice(&out[..SCHEDULER], GFP_KERNEL)?; + scene_template.extend_from_slice(&out[SCENES + 0x80..SCENES + 0x100], GFP_KERNEL)?; + Ok(()) + })?; + let target_vm = program.vm().id(); + Ok(Self { program, parameter_buffer, compute_storage, target_memory, aliases, + vm_aliases: KVec::new(), + graph_alias, graph, signals, descriptor_template, scene_template, generation: 1, + target_shape: [p.width, p.height, u32::from(p.layers), + u32::from(p.utile_width), u32::from(p.utile_height)], + target_vm, target_cache: KVec::new() }) + } + + #[allow(clippy::too_many_arguments)] + fn encode_descriptors( + out: &mut [u8], p: &render_desc::G17pRenderParameters, + controls: &render::M5RenderControls, context_id: u8, + usc_required: [u64; 2], + ) -> Result { + let objects = render_desc::RenderDescriptorObjects { + record_a: HIGH + SCHEDULER as u64, + shared: HIGH + PM as u64, + record_b: HIGH + SCENES as u64 + 0x80, + zero: HIGH + SCENE_ZERO as u64, + }; + let metadata = render_desc::RenderDescriptorMetadata { + context_id: context_id as u32, submission_ordinal: 0, command_buffer_state_word: 0, + ta_hardware_buffer_id: PB_HARDWARE_ID, submit_sequence: 1, + }; + render::build_ta_descriptor_base(p, controls, objects, metadata, + LOW + TA as u64, &mut out[TA..TA + render_desc::TA_DESCRIPTOR_SIZE]) + .map_err(|_| EINVAL)?; + render::build_fragment_descriptor_base(p, controls, objects, + render_desc::RenderDescriptorMetadata { submit_sequence: 0, ..metadata }, + LOW + FRAGMENT as u64, + &mut out[FRAGMENT..FRAGMENT + render_desc::FRAGMENT_DESCRIPTOR_SIZE]) + .map_err(|_| EINVAL)?; + let (ta, fragment) = out.split_at_mut(FRAGMENT); + let ta_usc = render::M5RenderUscState { + control: HIGH + USC as u64, required_bytes: usc_required[0], + allocated_bytes: graphics_usc::CAPACITY_BYTES, + statistics: HIGH + USC_STATISTICS as u64, + }; + render::apply_usc_submission_state(ta, fragment, ta_usc, + render::M5RenderUscState { + required_bytes: usc_required[1], statistics: ta_usc.statistics + 0x40, ..ta_usc + }).map_err(|_| EINVAL)?; + // This bounded first submission owns hardware stamp1 in each + // queue and event-machine timestamp0x100. Do not import native + // allocator history (the captured queues had stamps2/3). + put32(ta, 0x79c, QUEUE_IDS[1] as u32); + put64(ta, 0x7a0, 0x100); + put64(ta, 0x7a8, 1); + ta[0x7b0] = 1; + ta[0x7b1] = 1; + put64(ta, 0x8a6, SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[0] as u64 * 4); + put64(ta, 0x8ae, SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[0] as u64 * 4); + put32(ta, 0x8b6, 0x100); + put32(ta, 0x8ba, QUEUE_IDS[0] as u32); + put64(ta, 0x8c6, (1u64 << 40) | 1); + put64(fragment, 0x2140, SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[1] as u64 * 4); + put64(fragment, 0x2148, SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[1] as u64 * 4); + put32(fragment, 0x2150, 0x100); + put32(fragment, 0x2154, QUEUE_IDS[1] as u32); + put32(fragment, 0x215c, 0); + put64(fragment, 0x2160, (1u64 << 40) | 1); + // Fixed 256x256 reference controls independently observed in + // the successful 25G76 draw; these replace inherited Neo values. + ta[0x789] = 8; + ta[0x85f] = 7; + put32(ta, 0x892, 0); + ta[0x932] = 0; + put32(fragment, 0x1fb0, 8); + fragment[0x21cc] = 0; + fragment[0x222d] = 0; + Ok(()) + } + + pub(super) fn parameter_descriptor(&self) -> Result<[u32; 4]> { + render::initial_parameter_buffer_descriptor(self.parameter_buffer.page_list(), PAGE_COUNT) + .map_err(|_| EINVAL) + } + + pub(super) fn uma_descriptor(&self) -> [u64; 4] { self.program.usc().descriptor() } + + pub(super) fn roots(&self) -> [u64; 2] { self.program.roots() } + + pub(super) fn context_id(&self) -> u8 { self.program.context_id() } + + pub(super) fn owns_vm(&self, vm: &mmu::Vm) -> bool { + self.program.vm().id() == vm.id() + } + + /// Hand this fully retired job to another submitting VM. The hardware + /// context slot, descriptor graph, signals, queues and the PB/USC pools + /// all keep their firmware-visible addresses: the context is re-leased + /// for the new VM and the pools are aliased into it. The previous VM's + /// render targets are not used in the new VM: they stay cached under the + /// previous VM for its next turn, and the next application frame takes or + /// allocates targets in the new VM. The caller publishes the new roots + /// before the next draw. + pub(super) fn rebind_vm(&mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + if self.owns_vm(vm) { return Ok(()); } + if self.scheduler_status()? != [2, 0] { return Err(EBUSY); } + let progress = self.progress()?; + if progress[0] != 2 || progress[1] != self.generation * 0x100 + || progress[3] != self.generation * 0x100 { return Err(EBUSY); } + // Release an earlier handoff's views before reserving the same + // addresses in the incoming VM. + self.vm_aliases = KVec::new(); + let mut aliases = self.program.rebind(dev, vm)?; + for alias in self.parameter_buffer.alias_into(vm)? { + aliases.push(alias, GFP_KERNEL)?; + } + let storage = self.compute_storage.iova(); + if let Some(alias) = self.compute_storage.map_alias_at_with_guards( + vm, storage, mmu::PROT_GPU_SHARED_RW, 2)? { + aliases.push(alias, GFP_KERNEL)?; + } + self.vm_aliases = aliases; + dev_dbg!(dev.as_ref(), "M5_RENDER_VM_HANDOFF context={} generation={} vm={}\n", + self.program.context_id(), self.generation, vm.id()); + Ok(()) + } + + /// Compute shares this application's published roots and USC owner. Only + /// its operand/preemption storage is its own: the GPU address of that storage. + pub(super) fn compute_storage(&self) -> u64 { + self.compute_storage.iova() + } + + pub(super) fn descriptor_addresses(&self, stage: usize) -> Result<[u64; 2]> { + let offset = match stage { 0 => TA, 1 => FRAGMENT, _ => return Err(EINVAL) }; + Ok([HIGH + offset as u64, LOW + offset as u64]) + } + + pub(super) fn usc_control(&self) -> u64 { HIGH + USC as u64 } + + pub(super) fn parameter_state(&self) -> u64 { HIGH + PM as u64 } + + pub(super) fn publish_context(&mut self) -> Result { self.program.publish() } + + pub(super) fn quarantine(&self) { self.program.quarantine(); } + + pub(super) fn progress(&mut self) -> Result<[u32; 5]> { + Ok([self.signals.read_u32(SIGNALS_HIGH + SUBMITTED as u64)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[0] as u64 * 4)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[0] as u64 * 4)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_A as u64 + QUEUE_IDS[1] as u64 * 4)?, + self.signals.read_u32(SIGNALS_HIGH + STAMPS_B as u64 + QUEUE_IDS[1] as u64 * 4)?]) + } + + pub(super) fn scheduler_status(&mut self) -> Result<[u32; 2]> { + // GFX cleans this 0xc8-byte object after unlinking the finished pair. + Ok([self.graph.read_u32(HIGH + SCHEDULER as u64 + 0xc)?, + self.graph.read_u32(HIGH + SCHEDULER as u64 + 0x94)?]) + } + + fn rebuild_retired_frame(&mut self, generation: u32) -> Result { + if generation != self.generation + 1 || generation > MAX_GRAPHICS_GENERATION + || self.scheduler_status()? != [2, 0] { + return Err(EBUSY); + } + let progress = self.progress()?; + if progress[0] != 2 || progress[1] != self.generation * 0x100 + || progress[3] != self.generation * 0x100 { return Err(EBUSY); } + self.graph.with_bytes_mut(|out| { + out[..SCHEDULER].copy_from_slice(&self.descriptor_template); + out[SCHEDULER..SCHEDULER + 0xc8].fill(0); + put64(out, SCHEDULER, SIGNALS_HIGH + SUBMITTED as u64); + put32(out, SCHEDULER + 0x10, 0x50); + out[SCENES + 0x80..SCENES + 0x100].copy_from_slice(&self.scene_template); + out[TIMESTAMPS..TIMESTAMPS + 16].fill(0); + let stamp = generation * 0x100; + put64(out, TA + 0x7a0, stamp as u64); + put64(out, TA + 0x7a8, generation as u64); + put32(out, TA + 0x8b6, stamp); + put64(out, TA + 0x8c6, (1u64 << 40) | generation as u64); + put32(out, FRAGMENT + 0x2150, stamp); + put64(out, FRAGMENT + 0x2160, (1u64 << 40) | generation as u64); + Ok(()) + })?; + // Leave previous stamps intact: they must be superseded by firmware, + // never host-written to the expected new completion value. + self.generation = generation; + Ok(()) + } + + /// Re-encode a retired application's descriptors without replaying its old + /// command pointers. The current bounded queue retains one VM. Target + /// storage can change only after complete retirement; the job + /// stamp keeps increasing and context replacement remains unsupported. + #[allow(clippy::too_many_arguments)] + pub(super) fn prepare_application_frame( + &mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm, mut p: render_desc::G17pRenderParameters, + controls: render::M5RenderControls, clusters: u32, + usc_required: [u64; 2], generation: u32, + ) -> Result { + if vm.id() != self.program.vm().id() || clusters != 1 + || !(1..=MAX_APPLICATION_DIMENSION).contains(&p.width) + || !(1..=MAX_APPLICATION_DIMENSION).contains(&p.height) || p.layers != 1 + || generation != self.generation + 1 || generation > MAX_GRAPHICS_GENERATION { + return Err(ENOTSUPP); + } + // Validate retirement before allocating or releasing any target pages. + // The caller additionally checks both queues, stamps and event bits. + let progress = self.progress()?; + if self.scheduler_status()? != [2, 0] || progress[0] != 2 + || progress[1] != self.generation * 0x100 + || progress[3] != self.generation * 0x100 { return Err(EBUSY); } + let shape = [p.width, p.height, u32::from(p.layers), + u32::from(p.utile_width), u32::from(p.utile_height)]; + let vm_id = self.program.vm().id(); + let replacement = if (vm_id, shape) != (self.target_vm, self.target_shape) { + let cached = self.target_cache.iter() + .position(|(cached_vm, cached_shape, _)| (*cached_vm, *cached_shape) == (vm_id, shape)); + Some(match cached { + Some(index) => self.target_cache.remove(index).map_err(|_| EINVAL)?.2, + None => M5TargetMemory::new(dev, &self.program, &p, clusters).inspect_err(|e| pr_err!("M5: target memory failed: {:?}\n", e))?, + }) + } else { None }; + replacement.as_ref().unwrap_or(&self.target_memory).apply(&mut p); + p.cycle = self.parameter_buffer.scene_cycle(); + p.record_index = render::parameter_buffer_metrics_record( + self.parameter_buffer.metrics() + 0x40).map_err(|_| EINVAL)?; + p.ta_hardware_buffer_id = PB_HARDWARE_ID; + p.usc_flist_hardware_buffer_id = graphics_usc::HARDWARE_ID; + p.ta_timestamp_start = HIGH + TIMESTAMPS as u64; + p.ta_timestamp_end = p.ta_timestamp_start + 8; + p.fragment_timestamp_start = p.ta_timestamp_start; + p.fragment_timestamp_end = p.ta_timestamp_end; + let mut descriptors = KVec::new(); + descriptors.extend_from_slice(&self.descriptor_template, GFP_KERNEL)?; + Self::encode_descriptors(&mut descriptors, &p, &controls, + self.program.context_id(), usc_required).inspect_err(|e| pr_err!("M5: encode_descriptors failed: {:?}\n", e))?; + self.descriptor_template = descriptors; + // The application owns its output. It must never be erased by the + // native diagnostic's sentinel/reset logic. + self.rebuild_retired_frame(generation).inspect_err(|e| pr_err!("M5: rebuild_retired_frame failed: {:?}\n", e))?; + if let Some(target) = replacement { + dev_dbg!(dev.as_ref(), "M5_RENDER_TARGET_RESIZE generation={} old={:?} new={:?}\n", + generation, self.target_shape, shape); + let retired = core::mem::replace(&mut self.target_memory, target); + if self.target_cache.len() == TARGET_CACHE_ENTRIES { + self.target_cache.remove(0).map_err(|_| EINVAL)?; + } + self.target_cache.push((self.target_vm, self.target_shape, retired), GFP_KERNEL)?; + self.target_vm = vm_id; + self.target_shape = shape; + } + Ok(()) + } + +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +pub(super) fn triangle_parameters() -> (render_desc::G17pRenderParameters, render::M5RenderControls) { + let mut p = render_desc::G17pRenderParameters::source_rsrc8(); + p.aux_fb = 0x10000288000; + p.aux_fb_flags = 0x10c001; + p.aux_fb_page_count = 0x100000; + p.context_base = USER_BASE; + p.cycle = 0x220040; + p.deflake_1 = 0x1000068340; + p.deflake_2 = 0x1000068040; + p.deflake_3 = 0x1000068000; + p.depth_aux_buffer = 0; + p.depth_aux_stride = 0; + p.depth_buffer = 0; + p.depth_clear_value_bits = 0x3f800000; + p.depth_dimensions = 0; + p.depth_flags = 0; + p.depth_stride = 0; + p.encoder = 0x1000018000; + p.fragment_lifecycle = 0x4089f4c0508e74c; + p.fragment_status = 0x1000258000; + p.heapmeta = 0x1000261000; + p.width = 256; + p.height = 256; + p.layers = 1; + p.lifecycle = 0x4089f4c0508e74d; + p.load_pipeline = 0x10000170200; + p.load_pipeline_bind = 0xffff800000000040; + p.merge_upper_x_bits = 0x3bddb3d9; + p.merge_upper_y_bits = 0x3bddb3d9; + p.multisample_control = 0x88; + p.occlusion_query_base = 0; + p.ppp_control = 0x202; + p.record_index = 0x80041; + p.stencil_aux_buffer = 0; + p.stencil_aux_stride = 0; + p.stencil_buffer = 0; + p.stencil_clear_value = 0; + p.stencil_stride = 0; + p.store_pipeline = 0x10000170400; + p.store_pipeline_bind = 0; + p.ta_hardware_buffer_id = PB_HARDWARE_ID; + p.ta_status = 0x1000078000; + p.tib_blocks = 0; + p.tile_config = 0x10280; + p.tilemap = 0x1000260000; + p.tpc = 0x1000288000; + p.usc_flist_hardware_buffer_id = graphics_usc::HARDWARE_ID; + p.utile_config = 0xa000; + p.utile_height = 32; + p.utile_width = 32; + p.absolute_pointers = false; + p.partial_store_pipeline_bind = 0; + p.partial_store_pipeline = 0x10000170400; + p.partial_load_pipeline_bind = 0xffff800000000040; + p.partial_load_pipeline = 0x10000170000; + p.depth_bias_array = 0x100002d8000; + p.scissor_array = 0x10000180000; + p.ta_timestamp_start = HIGH + TIMESTAMPS as u64; + p.ta_timestamp_end = p.ta_timestamp_start + 8; + p.fragment_timestamp_start = p.ta_timestamp_start; + p.fragment_timestamp_end = p.ta_timestamp_end; + let controls = render::M5RenderControls { + ta: render::M5RenderStageControls { + resource_flags: 0x4054b, resource_control: 0x200000001, ptc_max: 0xc6, work_key: 0x300, + }, + fragment: render::M5RenderStageControls { + resource_flags: 0x104054b, resource_control: 0x200000001, ptc_max: 0xce, work_key: 0x300, + }, + ta_tile_control: 0x10c08860, ta_dispatch_control: 0x700, + fragment_raster_control: 0x200000ff0200, fragment_tile_flags: 0x200003717f, + store_pipeline_2_bind: 0, store_pipeline_2: 0x10000170400, + partial_store_pipeline_2_bind: 0, partial_store_pipeline_2: 0x10000170400, + partial_tile_flags: 0x200000, + }; + (p, controls) +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_memory.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_memory.rs new file mode 100644 index 0000000000000..cf9a2f93efb03 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_memory.rs @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained M5 render VM and private storage, with an optional native test program. +//! +//! Allocation is separate from root publication and queue submission. A future +//! job owner keeps this graph alive through queue completion. After a fault, +//! ownership remains retained until both firmware CPUs stop. Captured physical +//! backing is never installed. + +use super::MappedObject; +use super::graphics_usc::M5GraphicsUsc; +use crate::{driver::AsahiDrmDevice, mmu}; + +/// Pages of a client VM aliased into the context, at most this many. +const PAGE_BYTES: usize = 0x4000; +const MAX_PAGES: usize = 384; +use kernel::prelude::*; + +pub(super) struct M5GraphicsMemory { + // Retract any published roots before dropping their mapping owners. + context: mmu::M5ExecutionContext, + // Keep user mappings and GEM backing across handle close/VM teardown. + // This guard's bookkeeping is generation-neutral despite its old name. + user_mappings: Option, + pages: KVec, + usc: M5GraphicsUsc, + output_first: Option, +} + +impl M5GraphicsMemory { + + /// Keep the actual submitting VM and allocate only driver-owned storage. + /// Application shader/input/output mappings remain in that VM; no native + /// program firmware or captured pages are requested by this constructor. + pub(super) fn from_vm(dev: &AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm) -> Result { + crate::m5::vm_layout::compact_range(&vm.kernel_range()).ok_or(EINVAL)?; + let user_mappings = vm.retain_job()?; + let context = uat.new_m5_execution_context(vm)?; + let usc = M5GraphicsUsc::new_application(dev, context.vm())?; + Ok(Self { context, user_mappings: Some(user_mappings), pages: KVec::new(), usc, output_first: None }) + } + + /// Hand the retained USC pool and this context slot to another + /// application VM. The previous roots are retracted first (with ASID + /// invalidation), the same hardware context is re-leased for `vm`, and + /// the pool is aliased at unchanged addresses, so every firmware record + /// that names the context or the pool keeps its value. The caller has + /// verified that the previous owner's work is completely retired and + /// publishes the new roots explicitly afterwards. + pub(super) fn rebind(&mut self, dev: &AsahiDrmDevice, vm: &mmu::Vm) + -> Result> { + crate::m5::vm_layout::compact_range(&vm.kernel_range()).ok_or(EINVAL)?; + // Only application owners hand over; the native diagnostic program + // keeps captured pages in its private VM. + if !self.pages.is_empty() || self.output_first.is_some() { return Err(ENOTSUPP); } + let user_mappings = vm.retain_job()?; + self.context.rebind(vm)?; + // Release the previous VM's mapping hold only after its roots are gone. + self.user_mappings = Some(user_mappings); + let aliases = self.usc.alias_into(vm)?; + dev_dbg!(dev.as_ref(), "M5_GRAPHICS_CONTEXT id={} rebound vm={} roots={:x?}; USC pool aliased\n", + self.context.context_id(), vm.id(), self.context.roots()); + Ok(aliases) + } + + /// Return an owning allocation in the compact TA resource arena. The + /// render job retains it until completion or confirmed processor stop. + pub(super) fn allocate_compact( + &self, dev: &AsahiDrmDevice, size: usize, alignment: u64, prot: mmu::Prot, + ) -> Result { + let range = crate::m5::vm_layout::compact_range(&self.vm().kernel_range()).ok_or(EINVAL)?; + MappedObject::new_wc_in_range_with_guards( + dev, self.vm(), range, size, alignment, prot, 2, + ) + } + + /// Job construction can add its own objects before root publication. + pub(super) fn vm(&self) -> &mmu::Vm { + self.context.vm() + } + + pub(super) fn context_id(&self) -> u8 { self.context.context_id() } + + pub(super) fn roots(&self) -> [u64; 2] { + self.context.roots() + } + + pub(super) fn usc(&self) -> &M5GraphicsUsc { + &self.usc + } + + /// A stop request is not proof that hardware has released these mappings. + /// Close VM_BIND admission before any future error fence can wake userspace. + pub(super) fn quarantine(&self) { + if let Some(guard) = &self.user_mappings { + guard.quarantine(); + } + } + + /// Alias existing owned user pages into the firmware VM. PM page lists + /// and scene counters must share backing with their user GPU views. + /// The returned mappings belong to the eventual retained job owner. + pub(super) fn alias_kernel_pages( + &mut self, + uat: &mmu::Uat, + user_address: u64, + kernel_address: u64, + bytes: usize, + ) -> Result> { + if self.context.is_current() { + return Err(EBUSY); + } + if bytes == 0 + || bytes > MAX_PAGES * PAGE_BYTES + || bytes % PAGE_BYTES != 0 + || (user_address | kernel_address) & (PAGE_BYTES as u64 - 1) != 0 + { + return Err(EINVAL); + } + user_address.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + kernel_address.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + let mut indices = KVec::new(); + for offset in (0..bytes).step_by(PAGE_BYTES) { + let address = user_address + offset as u64; + let index = self + .pages + .iter() + .position(|page| page.iova() == address) + .ok_or(EFAULT)?; + indices.push(index, GFP_KERNEL)?; + } + let mut aliases = KVec::new(); + for (ordinal, index) in indices.iter().enumerate() { + let high = kernel_address + (ordinal * PAGE_BYTES) as u64; + let alias = + self.pages[*index].map_alias_at(uat.kernel_vm(), high, mmu::PROT_FW_SHARED_RW)?; + if uat.kernel_vm().translate_iova(high)? + != self + .context + .vm() + .translate_iova(self.pages[*index].iova())? + { + return Err(EFAULT); + } + aliases.push(alias, GFP_KERNEL)?; + } + Ok(aliases) + } + + /// Read a word from a retained reference page while preparing its job. + pub(super) fn read_word(&mut self, address: u64) -> Result { + if address & 3 != 0 { + return Err(EINVAL); + } + let base = address & !(PAGE_BYTES as u64 - 1); + let page = self + .pages + .iter_mut() + .find(|page| page.iova() == base) + .ok_or(EFAULT)?; + page.read_u32(address) + } + + /// Reset owned PM outputs before exposing the context to either firmware. + pub(super) fn clear_pages(&mut self, address: u64, bytes: usize) -> Result { + if self.context.is_current() || bytes == 0 || bytes % PAGE_BYTES != 0 + || address & (PAGE_BYTES as u64 - 1) != 0 { + return Err(EINVAL); + } + let end = address.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + let count = self.pages.iter().filter(|p| p.iova() >= address && p.iova() < end).count(); + if count != bytes / PAGE_BYTES { return Err(EFAULT); } + for page in self.pages.iter_mut().filter(|p| p.iova() >= address && p.iova() < end) { + page.with_bytes_mut(|out| { out.fill(0); Ok(()) })?; + } + Ok(()) + } + + pub(super) fn write_word(&mut self, address: u64, value: u32) -> Result { + if self.context.is_current() || address & 3 != 0 { return Err(EINVAL); } + let base = address & !(PAGE_BYTES as u64 - 1); + let offset = (address - base) as usize; + let page = self.pages.iter_mut().find(|p| p.iova() == base).ok_or(EFAULT)?; + page.with_bytes_mut(|out| { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + Ok(()) + }) + } + + /// Call only after the complete TA/3D resource graph is prepared. + pub(super) fn publish(&mut self) -> Result { + self.context.publish() + } + + pub(super) fn is_current(&self) -> bool { + self.context.is_current() + } + +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_queues.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_queues.rs new file mode 100644 index 0000000000000..02850fbe073df --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_queues.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Paired first-submission queues for the retained M5 graphics experiment. + +use super::{graphics_job::{M5GraphicsJob, PB_HARDWARE_ID, QUEUE_IDS}, graphics_usc, + firmware_view::FirmwareDataView, MappedObject}; +use crate::{driver::AsahiDrmDevice, m5::submission, m5::channel, m5::sksm, mmu}; +use crate::m5::completion::{graphics_pointer_head, GRAPHICS_POINTER_COUNT, + MAX_GRAPHICS_GENERATION}; +use kernel::prelude::*; + +const HIGH: u64 = 0xffff_fc20_c000_0000; +const ENTRY_HIGH: u64 = 0xffff_fc20_0039_0000; +const ENTRY_LOW: u64 = 0x70_005f_0000; +const ENTRY_BYTES: usize = 0x20000; +const CONTROL: usize = 0x18000; +// Native TA and fragment channel+0x10 both point to fffffc2000000048. +// One scheduler belongs to one intrusive list even when two SKSM queues run it. +pub(super) const JOB_LIST: u64 = HIGH + 0x20000; +const POINTERS: [usize; 2] = [0x4000, 0xc000]; +const ITEMS: [usize; 2] = [0x8000, 0x10000]; +const CONFIG: [usize; 2] = [0xa800, 0x12800]; + +pub(super) struct M5GraphicsQueues { + aliases: KVec, + entries: KVec, + graph: MappedObject, + job_list: MappedObject, + firmware: FirmwareDataView, +} + +impl M5GraphicsQueues { + pub(super) fn new(dev: &AsahiDrmDevice, uat: &mmu::Uat, job: &M5GraphicsJob) -> Result { + let mut graph = MappedObject::new_wc_at(dev, uat.kernel_vm(), HIGH, + 0x20000, mmu::PROT_FW_PRIV_RW)?; + let mut job_list = MappedObject::new_wc_at(dev, uat.kernel_vm(), JOB_LIST, + 0x4000, mmu::PROT_FW_SHARED_RW)?; + job_list.with_bytes_mut(|out| { put64(out, 8, JOB_LIST); Ok(()) })?; + let mut entries = KVec::new(); + let mut aliases = KVec::new(); + for stage in 0..2 { + let id = QUEUE_IDS[stage]; + let high = ENTRY_HIGH + (stage * ENTRY_BYTES) as u64; + let low = ENTRY_LOW + (stage * ENTRY_BYTES) as u64; + let mut entry = MappedObject::new_wc_at(dev, uat.kernel_vm(), high, + ENTRY_BYTES, mmu::PROT_FW_PRIV_RW)?; + let alias = entry.map_alias_at(uat.kernel_lower_vm(), low, mmu::PROT_GPU_SHARED_RW)?; + for offset in (0..ENTRY_BYTES).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset as u64)? + != uat.kernel_lower_vm().translate_iova(low + offset as u64)? { + return Err(EFAULT); + } + } + let [descriptor, _] = job.descriptor_addresses(stage)?; + let dm = if stage == 0 { sksm::M5DataMaster::Tiling } + else { sksm::M5DataMaster::Fragment }; + let prepared = Self::prepare_entry(job, stage, 1)?; + entry.with_bytes_mut(|out| { + out[prepared.offset as usize..prepared.offset as usize + prepared.bytes.len()] + .copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let install = channel::encode_m5_channel_config_for(channel::M5ChannelConfig { + ring_low: low, ring_high: high, queue_id: id, priority: 2, + install_queue: true, gpu_context: job.context_id(), event: None, + channel_control: HIGH + CONTROL as u64, update_scheduler: true, + scheduler_index: 0, channel_stamp_index: stage as u32, + update_context: false, owner_context: 0, extended_command: false, + }, stage as u32, 2, None, None).map_err(|_| EINVAL)?; + let config = channel::encode_m5_channel_config_for(channel::M5ChannelConfig { + ring_low: low, ring_high: high, queue_id: id, priority: 2, + install_queue: false, gpu_context: job.context_id(), + event: Some(channel::M5ChannelEvent { address: job.usc_control(), + timestamp: 1, index: graphics_usc::HARDWARE_ID }), + channel_control: HIGH + CONTROL as u64, update_scheduler: false, + scheduler_index: 0, channel_stamp_index: stage as u32, + update_context: true, owner_context: 1, extended_command: false, + }, stage as u32, 2, Some(0), if stage == 0 { + Some(channel::M5ParameterBufferEvent { address: job.parameter_state(), + timestamp: 1, hardware_buffer_id: PB_HARDWARE_ID, manager_id: 0 }) + } else { None }).map_err(|_| EINVAL)?; + let kick = sksm::encode_m5_sksm_add_kicks_for(dm, id, 2, 1, 1) + .map_err(|_| EINVAL)?; + graph.with_bytes_mut(|out| { + let pointers = POINTERS[stage]; + let queue = id as usize * 0xc0; + let reset = submission::encode_m5_25g76_channel_state_reset( + submission::M5ChannelStateResetValues { + uncached_state: HIGH + pointers as u64, + command_pointer_ring: HIGH + ITEMS[stage] as u64, + gpu_va_10: JOB_LIST, word_48: 0, + gpu_va_9c: HIGH + CONTROL as u64, + }); + out[queue..queue + 0xc0].copy_from_slice(&reset); + for offset in [0x28, 0x2c, 0x40] { put32(out, queue + offset, 2); } + out[queue + 0x36..queue + 0x38].fill(0xff); + put32(out, pointers + 0x50, u32::MAX); + put32(out, pointers + 0x60, 0x500); + out[CONFIG[stage]..CONFIG[stage] + install.len()].copy_from_slice(&install); + out[CONFIG[stage] + 0x100..CONFIG[stage] + 0x100 + config.len()].copy_from_slice(&config); + out[CONFIG[stage] + 0x200..CONFIG[stage] + 0x200 + kick.len()].copy_from_slice(&kick); + for (index, address) in [HIGH + CONFIG[stage] as u64, descriptor, + HIGH + CONFIG[stage] as u64 + 0x100, + HIGH + CONFIG[stage] as u64 + 0x200].into_iter().enumerate() { + put64(out, ITEMS[stage] + index * 8, address); + } + // Publish only installation. The draw stays beyond the producer + // until both firmware queue slots and scheduler bindings verify. + put32(out, pointers + 0x40, 1); + Ok(()) + })?; + aliases.push(alias, GFP_KERNEL)?; + entries.push(entry, GFP_KERNEL)?; + } + graph.with_bytes_mut(|out| { + out[CONTROL..CONTROL + 2].fill(0xff); + out[CONTROL + 5] = 1; + out[CONTROL + 0x26] = 2; + out[CONTROL + 0x33] = 0xff; + Ok(()) + })?; + Ok(Self { aliases, entries, graph, job_list, firmware: FirmwareDataView::new(dev)? }) + } + + pub(super) fn queue_address(stage: usize) -> Result { + Ok(HIGH + *QUEUE_IDS.get(stage).ok_or(EINVAL)? as u64 * 0xc0) + } + + fn prepare_entry(job: &M5GraphicsJob, stage: usize, generation: u32) + -> Result { + if !(1..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + let id = *QUEUE_IDS.get(stage).ok_or(EINVAL)?; + let [descriptor, descriptor_low] = job.descriptor_addresses(stage)?; + let stamp = generation as u64; + let barrier = [sksm::M5SksmDependency { queue_id: QUEUE_IDS[0], stamp, kind: 1 }]; + let mut rce = [sksm::M5SksmRce { address: 0, tag: 0 }; 4]; + if stage == 0 { + rce[0] = sksm::M5SksmRce { address: descriptor_low + 0x60, tag: 79 }; + } else { + for (index, (offset, count)) in [(0xa0, 87), (0x7c0, 18), + (0xee0, 25), (0x1600, 12)].into_iter().enumerate() { + rce[index] = sksm::M5SksmRce { address: descriptor_low + offset, tag: count }; + } + } + sksm::prepare_m5_sksm_entry(sksm::M5SksmQueue { + id, priority: 2, current_stamp: stamp, submitted: 0, + previous_stamp: if generation == 1 { None } else { Some(stamp - 1) }, + }, sksm::M5SksmOperands { + barriers: if stage == 0 { &[] } else { &barrier }, first_predecessor: 0, + implicit_dependency_type: 1, mcache: None, flag_bit1: false, + flag_bit61: false, flag_bit63: false, + payload: [descriptor, Self::queue_address(stage)?], + qos_byte_32: 3, qos_class_40: if stage == 0 { 12 } else { 24 }, + qos_word_48: 0xffff, event_mask: [0; 4], rce_kind: 0, rce, + auxiliary: 0x3f_ffff_ffff_ffff, + }).map_err(|_| EINVAL) + } + + pub(super) fn publish_next_draw(&mut self, job: &M5GraphicsJob, generation: u32) -> Result { + if !(2..=MAX_GRAPHICS_GENERATION).contains(&generation) { return Err(EINVAL); } + let previous_head = graphics_pointer_head(generation - 1); + if self.counters()? != [[previous_head; 3]; 2] || !self.installed()? + || self.completion_list()? != [0, JOB_LIST] { return Err(EBUSY); } + for stage in 0..2 { + let prepared = Self::prepare_entry(job, stage, generation)?; + self.entries[stage].with_bytes_mut(|out| { + let offset = prepared.offset as usize; + let slot = &mut out[offset..offset + prepared.bytes.len()]; + // Every preceding job has retired, including the prior user + // of this 256-entry SKSM slot. Preserve the first-use check. + if generation < 256 && slot.iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + slot.copy_from_slice(&prepared.bytes); + Ok(()) + })?; + let dm = if stage == 0 { sksm::M5DataMaster::Tiling } + else { sksm::M5DataMaster::Fragment }; + let kick = sksm::encode_m5_sksm_add_kicks_for(dm, QUEUE_IDS[stage], + 2, generation as u64, 1).map_err(|_| EINVAL)?; + self.graph.with_bytes_mut(|out| { + out[CONFIG[stage] + 0x200..CONFIG[stage] + 0x200 + kick.len()] + .copy_from_slice(&kick); + let addresses = [job.descriptor_addresses(stage)?[0], + HIGH + CONFIG[stage] as u64 + 0x100, + HIGH + CONFIG[stage] as u64 + 0x200]; + for (index, address) in addresses.into_iter().enumerate() { + let absolute = 1 + (generation as usize - 1) * 3 + index; + let offset = ITEMS[stage] + (absolute % GRAPHICS_POINTER_COUNT as usize) * 8; + if absolute < GRAPHICS_POINTER_COUNT as usize + && out[offset..offset + 8].iter().any(|byte| *byte != 0) { + return Err(EBUSY); + } + put64(out, offset, address); + } + Ok(()) + })?; + } + for pointers in POINTERS { + self.graph.write_u32(HIGH + pointers as u64 + 0x40, + graphics_pointer_head(generation))?; + } + Ok(()) + } + + pub(super) fn counters(&mut self) -> Result<[[u32; 3]; 2]> { + let mut result = [[0; 3]; 2]; + for stage in 0..2 { + for (index, offset) in [0, 0x30, 0x40].into_iter().enumerate() { + result[stage][index] = self.graph.read_u32(HIGH + (POINTERS[stage] + offset) as u64)?; + } + } + Ok(result) + } + + pub(super) fn completion_list(&mut self) -> Result<[u64; 2]> { + Ok([self.job_list.read_u64(JOB_LIST)?, self.job_list.read_u64(JOB_LIST + 8)?]) + } + + pub(super) fn installation_status(&mut self) -> Result<([[u64; 5]; 2], [u64; 2], [u64; 2])> { + let mut slots = [[0; 5]; 2]; + let mut pointers = [0; 2]; + for stage in 0..2 { + (slots[stage], pointers[stage]) = self.firmware.queue_registration(QUEUE_IDS[stage])?; + } + Ok((slots, pointers, [u64::from(self.graph.read_u32(HIGH + CONTROL as u64)?), + self.graph.read_u64(HIGH + CONTROL as u64 + 0x12)?])) + } + + pub(super) fn installed(&mut self) -> Result { + let (slots, pointers, control) = self.installation_status()?; + for stage in 0..2 { + let slot = slots[stage]; + let offset = (stage * ENTRY_BYTES) as u64; + // GFX 0x12240/0x1241c copies tag15 +0x22 to slot +0x20: + // this word is the TA/fragment stage, not the outer channel. + if slot[0] & 1 == 0 || slot[1] != ENTRY_HIGH + offset + || slot[2] != ENTRY_LOW + offset || slot[3] != 4u64 << 32 + || slot[4] as u32 != stage as u32 || pointers[stage] != HIGH + CONTROL as u64 { + return Ok(false); + } + } + Ok(control[0] & 0xff != 0xff && (control[0] >> 8) & 0xff != 0xff + && (control[0] >> 16) & 0xff == 4 + && control[1] == ((1u64 << QUEUE_IDS[0]) | (1u64 << QUEUE_IDS[1]))) + } + + pub(super) fn publish_draw(&mut self) -> Result { + if self.counters()? != [[1; 3]; 2] || !self.installed()? { return Err(EBUSY); } + for pointers in POINTERS { + self.graph.write_u32(HIGH + pointers as u64 + 0x40, 4)?; + } + Ok(()) + } +} + +fn put32(out: &mut [u8], offset: usize, value: u32) { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} +fn put64(out: &mut [u8], offset: usize, value: u64) { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} diff --git a/drivers/gpu/drm/asahi/m5/resources/graphics_usc.rs b/drivers/gpu/drm/asahi/m5/resources/graphics_usc.rs new file mode 100644 index 0000000000000..1d5bd727ee719 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/graphics_usc.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Retained USC backing in an M5 application's reserved driver VA region. +//! +//! Firmware records are encoded from the owned mappings, never fixed test +//! addresses. Guard pages belong to each mapping's allocator reservation. +//! No root or firmware table is published here. + +use super::MappedObject; +use crate::{driver::AsahiDrmDevice, mmu}; +use kernel::prelude::*; + +pub(super) const HARDWARE_ID: u32 = 1; +const DIRECTORY_ENTRIES: usize = 0x80000; +const BLOCK_BYTES: usize = 0x20_0000; +const BLOCK_COUNT: usize = 19; +const UNIT_BYTES: usize = 0x1000; +const UNIT_COUNT: usize = BLOCK_COUNT * BLOCK_BYTES / UNIT_BYTES; +pub(super) const CAPACITY_BYTES: u64 = (BLOCK_COUNT * BLOCK_BYTES) as u64; + +pub(super) struct M5GraphicsUsc { + directory: MappedObject, + freelist: MappedObject, + blocks: KVec, +} + +impl M5GraphicsUsc { + pub(super) fn new(dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + Self::new_in_range(dev, vm, vm.kernel_range()) + } + + pub(super) fn new_application(dev: &AsahiDrmDevice, vm: &mmu::Vm) -> Result { + let range = crate::m5::vm_layout::auxiliary_range(&vm.kernel_range()).ok_or(EINVAL)?; + Self::new_in_range(dev, vm, range) + } + + fn new_in_range( + dev: &AsahiDrmDevice, vm: &mmu::Vm, range: core::ops::Range, + ) -> Result { + let allocate = |size, guard_pages| MappedObject::new_wc_in_range_with_guards( + dev, vm, range.clone(), size, 0x8000, mmu::PROT_GPU_SHARED_RW, guard_pages, + ); + let mut directory = allocate(DIRECTORY_ENTRIES * 8, 2)?; + let mut freelist = allocate(mmu::UAT_PGSZ, 1)?; + let mut blocks = KVec::new(); + for _ in 0..BLOCK_COUNT { + blocks.push(allocate(BLOCK_BYTES, 2)?, GFP_KERNEL)?; + } + directory.with_bytes_mut(|out| { + for unit in 0..UNIT_COUNT { + let address = Self::unit_address(&blocks, unit).ok_or(EINVAL)?; + out[unit * 8..unit * 8 + 8].copy_from_slice(&address.to_le_bytes()); + } + Ok(()) + })?; + freelist.with_bytes_mut(|out| { + // Same initial free-block format as the qualified M5 compute + // pool. Every allocation here is free; no native growth history + // or in-use allocator contents are imported. + for block in 0..BLOCK_COUNT { + let address = blocks[block].iova(); + out[block * 0x40..block * 0x40 + 8] + .copy_from_slice(&(address | (1u64 << 61)).to_le_bytes()); + } + Ok(()) + })?; + let mut result = Self { directory, freelist, blocks }; + result.validate(vm)?; + dev_dbg!(dev.as_ref(), + "M5_GRAPHICS_USC reserved={:#x}..{:#x} directory={:#x} freelist={:#x} blocks={} capacity={:#x}; guard pages reserved\n", + range.start, range.end, result.directory.iova(), result.freelist.iova(), + BLOCK_COUNT, CAPACITY_BYTES); + Ok(result) + } + + fn unit_address(blocks: &[MappedObject], unit: usize) -> Option { + if unit >= UNIT_COUNT { return None; } + let units_per_block = BLOCK_BYTES / UNIT_BYTES; + Some(blocks.get(unit / units_per_block)?.iova() + + (unit % units_per_block * UNIT_BYTES) as u64) + } + + pub(super) fn descriptor(&self) -> [u64; 4] { + // M5 halLoadUMADesc: 4-KiB population, not 16-KiB CPU page count. + [self.directory.iova() >> 7, (UNIT_COUNT as u64) << 32, + (DIRECTORY_ENTRIES as u64) << 32 | UNIT_COUNT as u64, 0] + } + + /// Initialize a fresh owner using the qualified M5 initial-state recipe. + /// Native graphics snapshots include later growth generations; copying + /// their seeds/disabled flag would disagree with these all-free blocks. + pub(super) fn encode_initial_owner(&self, inner: u64, out: &mut [u8]) -> Result { + if out.len() < 0x70 || inner < 0xffff_fc00_0000_0000 || inner & 7 != 0 { + return Err(EINVAL); + } + out[..0x70].fill(0); + for (offset, value) in [(0, 1u32), (8, HARDWARE_ID), (0x10, 2), + (0x1c, DIRECTORY_ENTRIES as u32), (0x24, UNIT_COUNT as u32), + (0x2c, UNIT_COUNT as u32), (0x40, 4), (0x48, (BLOCK_COUNT * 8) as u32), + (0x5c, 1), (0x60, 2)] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, address) in [(0x14, self.directory.iova()), + (0x30, self.freelist.iova()), (0x4c, inner)] { + out[offset..offset + 8].copy_from_slice(&address.to_le_bytes()); + } + Ok(()) + } + + /// Map the retained pool into another application VM at the same + /// addresses, reproducing each allocation's guard reservation, so every + /// firmware record naming the directory, free list or blocks stays valid + /// across a render client handoff. The returned aliases must be kept for + /// as long as that VM may submit. + pub(super) fn alias_into(&mut self, vm: &mmu::Vm) -> Result> { + let range = crate::m5::vm_layout::auxiliary_range(&vm.kernel_range()).ok_or(EINVAL)?; + let mut aliases = KVec::new(); + let alias = |object: &mut MappedObject, size: usize, guards: usize, + aliases: &mut KVec| -> Result { + let end = object.iova().checked_add((size + guards * mmu::UAT_PGSZ) as u64) + .ok_or(EOVERFLOW)?; + if object.iova() < range.start || end > range.end { return Err(ERANGE); } + if let Some(mapping) = object.map_alias_at_with_guards( + vm, object.iova(), mmu::PROT_GPU_SHARED_RW, guards)? { + aliases.push(mapping, GFP_KERNEL)?; + } + Ok(()) + }; + alias(&mut self.directory, DIRECTORY_ENTRIES * 8, 2, &mut aliases)?; + alias(&mut self.freelist, mmu::UAT_PGSZ, 1, &mut aliases)?; + for block in self.blocks.iter_mut() { + alias(block, BLOCK_BYTES, 2, &mut aliases)?; + } + // Only the mappings are checked here: the firmware has been + // advancing the free-ring cursors and returning directory units + // while the pool was in use, so the construction-time contents no + // longer hold and must not be re-asserted on handoff. + self.validate_mappings(vm)?; + Ok(aliases) + } + + /// Verify every CPU page named by the populated 4-KiB directory and the + /// free-block heads. The allocator keeps the underlying mappings alive. + pub(super) fn validate(&mut self, vm: &mmu::Vm) -> Result { + self.validate_mappings(vm)?; + self.validate_contents() + } + + /// Range, alignment, guard-gap and translation checks for every pool + /// object in `vm`; valid before and after firmware use of the pool. + fn validate_mappings(&self, vm: &mmu::Vm) -> Result { + if self.blocks.len() != BLOCK_COUNT { return Err(EFAULT); } + let range = vm.kernel_range(); + let validate_guards = |object: &MappedObject, size: usize, guards: usize| -> Result { + let end = object.iova().checked_add((size + guards * mmu::UAT_PGSZ) as u64) + .ok_or(EOVERFLOW)?; + if object.iova() < range.start || end > range.end + || object.iova() & 0x7fff != 0 { return Err(EFAULT); } + // Check after every allocation has completed: a later neighbor + // must not have consumed either reserved 16-KiB guard page. + for guard in 0..guards { + if vm.translate_iova(object.iova() + (size + guard * mmu::UAT_PGSZ) as u64).is_ok() { + return Err(EFAULT); + } + } + Ok(()) + }; + validate_guards(&self.directory, DIRECTORY_ENTRIES * 8, 2)?; + validate_guards(&self.freelist, mmu::UAT_PGSZ, 1)?; + for block in &self.blocks { validate_guards(block, BLOCK_BYTES, 2)?; } + let translate_all = |object: &MappedObject| -> Result { + for offset in (0..object.logical_size).step_by(mmu::UAT_PGSZ) { + vm.translate_iova(object.iova() + offset as u64)?; + } + Ok(()) + }; + translate_all(&self.directory)?; + translate_all(&self.freelist)?; + for block in &self.blocks { translate_all(block)?; } + Ok(()) + } + + /// Construction-time contents: an all-free block list and a directory + /// that names every unit in order. Firmware use changes both. + fn validate_contents(&mut self) -> Result { + for block in 0..BLOCK_COUNT { + let base = self.blocks[block].iova(); + if self.freelist.read_u64(self.freelist.iova() + (block * 0x40) as u64)? + != (base | (1u64 << 61)) { + return Err(EFAULT); + } + } + for unit in 0..UNIT_COUNT { + if self.directory.read_u64(self.directory.iova() + (unit * 8) as u64)? + != Self::unit_address(&self.blocks, unit).ok_or(EFAULT)? { + return Err(EFAULT); + } + } + if self.directory.read_u64(self.directory.iova() + (UNIT_COUNT * 8) as u64)? != 0 { + return Err(EFAULT); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m5/resources/parameter_buffer.rs b/drivers/gpu/drm/asahi/m5/resources/parameter_buffer.rs new file mode 100644 index 0000000000000..3c6b955b6c799 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/parameter_buffer.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! VM-owned parameter buffer for the qualified 256x256 M5 render profile. +//! +//! The allocation addresses are dynamic. Capacity and scene layout remain +//! the witnessed native profile; larger renders and pool growth are separate. + +use super::{graphics_memory::M5GraphicsMemory, MappedObject}; +use crate::{driver::AsahiDrmDevice, mmu, m5::vm_layout}; +use kernel::prelude::*; + +pub(super) const BLOCK_BYTES: usize = 0x80000; +pub(super) const PAGE_LIST_BYTES: usize = 0x1c000; +pub(super) const PAGE_COUNT: u32 = 48; + +pub(super) struct M5ParameterBuffer { + blocks: [MappedObject; 3], + page_list: MappedObject, + metrics: MappedObject, + scene_scratch: MappedObject, + discard: MappedObject, +} + +impl M5ParameterBuffer { + pub(super) fn new_reference(dev: &AsahiDrmDevice, memory: &M5GraphicsMemory) -> Result { + let allocate = |size| memory.allocate_compact( + dev, size, 0x8000, mmu::PROT_GPU_SHARED_RW, + ); + let mut owner = Self { + blocks: [allocate(BLOCK_BYTES)?, allocate(BLOCK_BYTES)?, allocate(BLOCK_BYTES)?], + page_list: allocate(PAGE_LIST_BYTES)?, + metrics: allocate(0x4000)?, + scene_scratch: allocate(0x4000)?, + discard: allocate(0x8000)?, + }; + let block_bases = owner.block_page_bases()?; + owner.page_list.with_bytes_mut(|out| { + for (block, first) in block_bases.into_iter().enumerate() { + for page in 0..16 { + let offset = (block * 16 + page) * 4; + out[offset..offset + 4].copy_from_slice(&(first + page as u32).to_le_bytes()); + } + } + Ok(()) + })?; + owner.validate(memory.vm())?; + dev_dbg!(dev.as_ref(), + "M5_PARAMETER_BUFFER blocks={:x?} page_list={:#x} metrics={:#x} scene={:#x} discard={:#x}; compact mappings owned\n", + owner.blocks.each_ref().map(|b| b.iova()), owner.page_list(), owner.metrics(), + owner.scene_scratch.iova(), owner.discard.iova()); + Ok(owner) + } + + pub(super) fn block_page_bases(&self) -> Result<[u32; 3]> { + let mut bases = [0; 3]; + for (base, block) in bases.iter_mut().zip(self.blocks.iter()) { + let offset = block.iova().checked_sub(vm_layout::DRIVER_BASE).ok_or(ERANGE)?; + if offset & 0x7fff != 0 { return Err(EINVAL); } + *base = u32::try_from(offset >> 15).map_err(|_| ERANGE)?; + } + Ok(bases) + } + + pub(super) fn page_list(&self) -> u64 { self.page_list.iova() } + pub(super) fn metrics(&self) -> u64 { self.metrics.iova() } + pub(super) fn scene_cycle(&self) -> u64 { + self.scene_scratch.iova() - vm_layout::DRIVER_BASE + 0x40 + } + pub(super) fn discard_offset(&self) -> u64 { + self.discard.iova() - vm_layout::DRIVER_BASE + } + + /// Firmware and application views must reference the same backing. + pub(super) fn firmware_aliases( + &mut self, uat: &mmu::Uat, vm: &mmu::Vm, page_list_high: u64, metrics_high: u64, + ) -> Result> { + let mut aliases = KVec::new(); + for (object, high) in [(&mut self.page_list, page_list_high), (&mut self.metrics, metrics_high)] { + let alias = object.map_alias_at(uat.kernel_vm(), high, mmu::PROT_FW_SHARED_RW)?; + for offset in (0..object.logical_size).step_by(mmu::UAT_PGSZ) { + if uat.kernel_vm().translate_iova(high + offset as u64)? + != vm.translate_iova(object.iova() + offset as u64)? { + return Err(EFAULT); + } + } + aliases.push(alias, GFP_KERNEL)?; + } + Ok(aliases) + } + + /// Map the retained pool into another application VM at the same compact + /// addresses with the original two-page guard reservations, so the + /// firmware's parameter-manager records keep naming valid pages after a + /// render client handoff. + pub(super) fn alias_into(&mut self, vm: &mmu::Vm) -> Result> { + let range = vm_layout::compact_range(&vm.kernel_range()).ok_or(EINVAL)?; + let mut aliases = KVec::new(); + let alias = |object: &mut MappedObject, aliases: &mut KVec| -> Result { + let end = object.iova() + .checked_add((object.logical_size + 2 * mmu::UAT_PGSZ) as u64) + .ok_or(EOVERFLOW)?; + if object.iova() < range.start || end > range.end { return Err(ERANGE); } + if let Some(mapping) = object.map_alias_at_with_guards( + vm, object.iova(), mmu::PROT_GPU_SHARED_RW, 2)? { + aliases.push(mapping, GFP_KERNEL)?; + } + Ok(()) + }; + for block in self.blocks.iter_mut() { alias(block, &mut aliases)?; } + alias(&mut self.page_list, &mut aliases)?; + alias(&mut self.metrics, &mut aliases)?; + alias(&mut self.scene_scratch, &mut aliases)?; + alias(&mut self.discard, &mut aliases)?; + self.validate(vm)?; + Ok(aliases) + } + + fn validate(&self, vm: &mmu::Vm) -> Result { + let check = |object: &MappedObject| -> Result { + let start = object.iova(); + let end = start.checked_add(object.logical_size as u64).ok_or(EOVERFLOW)?; + if start < vm_layout::DRIVER_BASE || end > vm_layout::COMPACT_END + || start & 0x7fff != 0 { return Err(ERANGE); } + for offset in (0..object.logical_size).step_by(mmu::UAT_PGSZ) { + vm.translate_iova(start + offset as u64)?; + } + for page in 0..2 { + if vm.translate_iova(end + page * mmu::UAT_PGSZ as u64).is_ok() { + return Err(EFAULT); + } + } + Ok(()) + }; + for block in &self.blocks { check(block)?; } + for object in [&self.page_list, &self.metrics, &self.scene_scratch, &self.discard] { + check(object)?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m5/resources/target_memory.rs b/drivers/gpu/drm/asahi/m5/resources/target_memory.rs new file mode 100644 index 0000000000000..f0f2c5176018d --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/resources/target_memory.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Driver-owned render targets in the actual submitting VM. + +use super::{graphics_memory::M5GraphicsMemory, MappedObject}; +use crate::{driver::AsahiDrmDevice, m5::render_desc::G17pRenderParameters, mmu, m5::vm_layout, + m5::render_target::{self, M5RenderTargetLayout, PAGE_BYTES, AUX_BYTES}}; +use kernel::prelude::*; + +pub(super) struct M5TargetMemory { + target: MappedObject, + tpc: MappedObject, + deflake: MappedObject, + ta_status: MappedObject, + fragment_status: MappedObject, + auxiliary: MappedObject, + layout: M5RenderTargetLayout, +} + +impl M5TargetMemory { + pub(super) fn new( + dev: &AsahiDrmDevice, memory: &M5GraphicsMemory, p: &G17pRenderParameters, + clusters: u32, + ) -> Result { + let layout = M5RenderTargetLayout::new(p.width, p.height, p.layers, + p.utile_width, p.utile_height, clusters).ok_or(EINVAL)?; + let allocate = |size| memory.allocate_compact( + dev, size, 0x8000, mmu::PROT_GPU_SHARED_RW, + ); + let mut owner = Self { + target: allocate(layout.target_bytes)?, tpc: allocate(layout.tpc_bytes)?, + deflake: allocate(PAGE_BYTES)?, ta_status: allocate(PAGE_BYTES)?, + fragment_status: allocate(PAGE_BYTES)?, auxiliary: allocate(AUX_BYTES)?, layout, + }; + owner.deflake.with_bytes_mut(|bytes| { + render_target::initialize_deflake(bytes, p.width, p.height).ok_or(EINVAL) + })?; + owner.validate(memory.vm())?; + dev_dbg!(dev.as_ref(), + "M5_RENDER_TARGET target={:#x}/{:#x} heapmeta={:#x} tpc={:#x}/{:#x} deflake={:#x} status={:x?} auxiliary={:#x}; VM-owned\n", + owner.target.iova(), layout.target_bytes, owner.target.iova() + layout.heapmeta_offset as u64, + owner.tpc.iova(), layout.tpc_bytes, owner.deflake.iova(), + [owner.ta_status.iova(), owner.fragment_status.iova()], owner.auxiliary.iova()); + Ok(owner) + } + + pub(super) fn apply(&self, p: &mut G17pRenderParameters) { + p.context_base = vm_layout::DRIVER_BASE; + p.tilemap = self.target.iova(); + p.heapmeta = self.target.iova() + self.layout.heapmeta_offset as u64; + p.tpc = self.tpc.iova(); + p.deflake_3 = self.deflake.iova(); + p.deflake_2 = self.deflake.iova() + 0x40; + p.deflake_1 = self.deflake.iova() + 0x340; + p.ta_status = self.ta_status.iova(); + p.fragment_status = self.fragment_status.iova(); + p.aux_fb = self.auxiliary.iova(); + } + + fn validate(&self, vm: &mmu::Vm) -> Result { + for object in [&self.target, &self.tpc, &self.deflake, &self.ta_status, + &self.fragment_status, &self.auxiliary] { + let start = object.iova(); + let end = start.checked_add(object.logical_size as u64).ok_or(EOVERFLOW)?; + if start < vm_layout::DRIVER_BASE || end > vm_layout::COMPACT_END + || start & 0x7fff != 0 { return Err(ERANGE); } + for offset in (0..object.logical_size).step_by(PAGE_BYTES) { + vm.translate_iova(start + offset as u64)?; + } + for page in 0..2 { + if vm.translate_iova(end + page * PAGE_BYTES as u64).is_ok() { return Err(EFAULT); } + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m5/sksm.rs b/drivers/gpu/drm/asahi/m5/sksm.rs new file mode 100644 index 0000000000000..826f1def2c9b4 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/sksm.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! T8142/25G76 HAL300 SKSM entry and tag-14 serialization. +//! +//! The M5 host clears 0x200 bytes and writes two-bit dependency types at +//! +0x180. Neo's 0x180-byte entry cannot represent this data. These pure +//! builders do not register queues, publish producers or ring a doorbell. +//! Callers must retain every referenced object and implement the M5 lifetime +//! and completion protocol before any generated bytes reach firmware. + +pub(crate) const M5_SKSM_ENTRY_BYTES: usize = 0x200; +pub(crate) const M5_SKSM_ENTRY_COUNT: u32 = 256; + +/// Logical channel selector measured in native TA, 3D and CL tag-14 records. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum M5DataMaster { + Tiling = 0, + Fragment = 1, + Compute = 2, +} +const STAMP_MASK: u64 = (1 << 40) - 1; +const PACKED_ADDRESS_MASK: u64 = (1 << 38) - 1; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum M5SksmError { + QueueId, + Priority, + Stamp, + QueueFull, + Dependencies, + DependencyType, + Address, + Mcache, + Rce, + Qos, + AddCount, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmDependency { + pub(crate) queue_id: u8, + /// Already converted to the hardware's 32-bit generation / 8-bit index. + pub(crate) stamp: u64, + pub(crate) kind: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmRce { + pub(crate) address: u64, + pub(crate) tag: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmMcache { + pub(crate) address: u64, + pub(crate) index: u8, + pub(crate) count: u8, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmQueue { + pub(crate) id: u8, + /// Host queue +0x1c, not the outer firmware channel's priority. + pub(crate) priority: u8, + pub(crate) current_stamp: u64, + pub(crate) submitted: u32, + pub(crate) previous_stamp: Option, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5SksmOperands<'a> { + /// Explicit dependencies already resolved by their owning event machine. + pub(crate) barriers: &'a [M5SksmDependency], + /// Native convertEventMachineTSToHWTS(command) - 1, for first publication. + /// This must be grounded in the queue's initialized completion state. + pub(crate) first_predecessor: u64, + pub(crate) implicit_dependency_type: u8, + pub(crate) mcache: Option, + pub(crate) flag_bit1: bool, + pub(crate) flag_bit61: bool, + pub(crate) flag_bit63: bool, + pub(crate) payload: [u64; 2], + pub(crate) qos_byte_32: u8, + pub(crate) qos_class_40: u8, + pub(crate) qos_word_48: u16, + pub(crate) event_mask: [u64; 4], + pub(crate) rce_kind: u8, + pub(crate) rce: [M5SksmRce; 4], + /// Native encoder's scalar x5 argument, copied unmodified to +0x178. + pub(crate) auxiliary: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PreparedM5SksmEntry { + pub(crate) bytes: [u8; M5_SKSM_ENTRY_BYTES], + pub(crate) offset: u32, + /// May only replace host state after entry storage and a dmb ish. + pub(crate) queue_after: M5SksmQueue, +} + +fn check_queue(id: u8, priority: u8, stamp: u64) -> Result<(), M5SksmError> { + if id > 0x7f { + return Err(M5SksmError::QueueId); + } + if priority > 3 { + return Err(M5SksmError::Priority); + } + if stamp & !STAMP_MASK != 0 { + return Err(M5SksmError::Stamp); + } + Ok(()) +} + +fn packed_address(address: u64) -> Result { + if address & 31 != 0 || address >> 5 > PACKED_ADDRESS_MASK { + return Err(M5SksmError::Address); + } + Ok(address >> 5) +} + +fn put(bytes: &mut [u8], offset: usize, value: u64) { + bytes[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// Recovered from M5 encodeKickEntry at 25G76 0xfffffe00089e750c and its +/// encodeDependencyList at 0xfffffe00089e781c. Geometry is deliberately fixed +/// to the independently captured 256 x 512-byte M5 queues. +pub(crate) fn prepare_m5_sksm_entry( + queue: M5SksmQueue, + operands: M5SksmOperands<'_>, +) -> Result { + check_queue(queue.id, queue.priority, queue.current_stamp)?; + if queue.submitted >= M5_SKSM_ENTRY_COUNT { + return Err(M5SksmError::QueueFull); + } + // Native always appends one implicit predecessor after explicit barriers. + if operands.barriers.len() >= 32 { + return Err(M5SksmError::Dependencies); + } + if operands.qos_class_40 > 31 { + return Err(M5SksmError::Qos); + } + if operands.rce_kind > 3 || operands.rce.iter().any(|r| r.tag > 0x7f) { + return Err(M5SksmError::Rce); + } + let mut bytes = [0; M5_SKSM_ENTRY_BYTES]; + let mut header = (queue.current_stamp << 2) + | ((queue.id as u64) << 42) + | ((queue.priority as u64) << 59) + | ((operands.flag_bit1 as u64) << 1) + | ((operands.flag_bit61 as u64) << 61) + | ((operands.flag_bit63 as u64) << 63); + if let Some(mcache) = operands.mcache { + if mcache.index > 63 || mcache.count == 0 || mcache.count > 64 { + return Err(M5SksmError::Mcache); + } + let packed = packed_address(mcache.address)?; + put( + &mut bytes, + 8, + (mcache.index as u64) | (packed << 6) | ((mcache.count as u64 - 1) << 54), + ); + header |= 1 << 58; + } + put(&mut bytes, 0, header); + put(&mut bytes, 0x10, operands.payload[0]); + put(&mut bytes, 0x18, operands.payload[1]); + let predecessor = M5SksmDependency { + queue_id: queue.id, + stamp: queue.previous_stamp.unwrap_or(operands.first_predecessor), + kind: operands.implicit_dependency_type, + }; + let mut kinds = 0; + for (i, dependency) in operands + .barriers + .iter() + .chain(core::iter::once(&predecessor)) + .enumerate() + { + check_queue(dependency.queue_id, 0, dependency.stamp)?; + if dependency.kind > 3 { + return Err(M5SksmError::DependencyType); + } + put( + &mut bytes, + 0x28 + i * 8, + dependency.stamp | ((dependency.queue_id as u64) << 40), + ); + kinds |= (dependency.kind as u64) << (i * 2); + } + let count = operands.barriers.len() + 1; + let valid = if count == 32 { + u32::MAX + } else { + (1u32 << count) - 1 + }; + put( + &mut bytes, + 0x20, + valid as u64 + | ((operands.qos_byte_32 as u64) << 32) + | ((operands.qos_class_40 as u64) << 40) + | ((operands.qos_word_48 as u64) << 48), + ); + put(&mut bytes, 0x180, kinds); + for (i, mask) in operands.event_mask.into_iter().enumerate() { + put(&mut bytes, 0x128 + i * 8, mask); + } + bytes[0x148] = operands.rce_kind; + for (i, rce) in operands.rce.into_iter().enumerate() { + put( + &mut bytes, + 0x150 + i * 8, + packed_address(rce.address)? | ((rce.tag as u64) << 43), + ); + } + put(&mut bytes, 0x178, operands.auxiliary); + Ok(PreparedM5SksmEntry { + bytes, + offset: (queue.current_stamp as u32 & 255) * M5_SKSM_ENTRY_BYTES as u32, + queue_after: M5SksmQueue { + current_stamp: (queue.current_stamp + 1) & STAMP_MASK, + submitted: queue.submitted + 1, + previous_stamp: Some(queue.current_stamp), + ..queue + }, + }) +} + +/// Native CL add-kicks record at the captured submitBuffer publication site. +/// This is a packed 21-byte command, unrelated to the outer 24-byte entry. +pub(crate) fn encode_m5_sksm_add_kicks( + queue_id: u8, + priority: u8, + first_stamp: u64, + count: u16, +) -> Result<[u8; 21], M5SksmError> { + encode_m5_sksm_add_kicks_for( + M5DataMaster::Compute, + queue_id, + priority, + first_stamp, + count, + ) +} + +/// Encode a retained queue's stage explicitly; graphics never uses CL's DM2. +pub(crate) fn encode_m5_sksm_add_kicks_for( + data_master: M5DataMaster, + queue_id: u8, + priority: u8, + first_stamp: u64, + count: u16, +) -> Result<[u8; 21], M5SksmError> { + check_queue(queue_id, priority, first_stamp)?; + if count == 0 || count > 256 { + return Err(M5SksmError::AddCount); + } + let mut bytes = [0; 21]; + bytes[..4].copy_from_slice(&14u32.to_le_bytes()); + bytes[4..6].copy_from_slice(&(queue_id as u16).to_le_bytes()); + bytes[6..8].copy_from_slice(&count.to_le_bytes()); + bytes[8] = priority; + bytes[9..17].copy_from_slice(&first_stamp.to_le_bytes()); + bytes[17..21].copy_from_slice(&(data_master as u32).to_le_bytes()); + Ok(bytes) +} + diff --git a/drivers/gpu/drm/asahi/m5/submission.rs b/drivers/gpu/drm/asahi/m5/submission.rs new file mode 100644 index 0000000000000..c5414cea1ae2d --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/submission.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +fn put_u32_le(raw: &mut [u8], offset: usize, value: u32) { + raw[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put_u64_le(raw: &mut [u8], offset: usize, value: u64) { + raw[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// M5 G17G / macOS 25G76 resets six pairs of 128-bit vectors, covering 0xc0 +/// bytes. Its firmware consumes through +0xa7. The common 0x80-byte reset +/// would omit the moved fields and the firmware-visible tail. +pub(crate) const M5_25G76_CHANNEL_STATE_RESET_SIZE: usize = 0xc0; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct M5ChannelStateResetValues { + pub(crate) uncached_state: u64, + pub(crate) command_pointer_ring: u64, + pub(crate) gpu_va_10: u64, + pub(crate) word_48: u32, + /// Opaque host field +0xf8 copied to the unaligned firmware field +0x9c. + pub(crate) gpu_va_9c: u64, +} + +/// Exact reset stores from M5 AGXChannel::resetChannelState at +/// 0xfffffe000897d36c..0xfffffe000897d410 (AGXG17G 353.14, 25G76). +/// Uncached stores are separately shared with apply_g17_uncached_channel_reset. +/// This codec alone does not admit M5 submission or a Linux GPU runtime. +pub(crate) fn encode_m5_25g76_channel_state_reset( + values: M5ChannelStateResetValues, +) -> [u8; M5_25G76_CHANNEL_STATE_RESET_SIZE] { + let mut raw = [0u8; M5_25G76_CHANNEL_STATE_RESET_SIZE]; + put_u64_le(&mut raw, 0x00, values.uncached_state); + put_u64_le(&mut raw, 0x08, values.command_pointer_ring); + put_u64_le(&mut raw, 0x10, values.gpu_va_10); + put_u32_le(&mut raw, 0x24, u32::MAX); + put_u32_le(&mut raw, 0x28, 4); + put_u32_le(&mut raw, 0x44, u32::MAX); + put_u32_le(&mut raw, 0x48, values.word_48); + put_u64_le(&mut raw, 0x9c, values.gpu_va_9c); + raw +} + +/// Logical render prefix, resolved at ioctl admission before asynchronous work. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] +pub(crate) struct G17PRenderOrder { + pub(crate) sequence: u64, + pub(crate) wait_through: Option, +} + diff --git a/drivers/gpu/drm/asahi/m5/vm_layout.rs b/drivers/gpu/drm/asahi/m5/vm_layout.rs new file mode 100644 index 0000000000000..19a6593736112 --- /dev/null +++ b/drivers/gpu/drm/asahi/m5/vm_layout.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Application VA contract shared with Mesa's agx_m5_va.h. + +use core::ops::Range; + +pub(crate) const DRIVER_BASE: u64 = 0x10_0000_0000; +pub(crate) const COMPACT_END: u64 = DRIVER_BASE + (1u64 << 32); +pub(crate) const DATA_START: u64 = 0x40_0000_0000; + +/// TA deflake fields encode an unsigned 32-bit offset from DRIVER_BASE. +/// Keep the private render arena inside that aperture and the USC backing +/// after it, within the same VM_CREATE reservation. A client may leave a +/// prefix of the compact aperture for its own command data. Mesa's default +/// reservation still begins at DRIVER_BASE and data begins at DATA_START. +pub(crate) fn compact_range(reserved: &Range) -> Option> { + if reserved.start < DRIVER_BASE || reserved.start >= COMPACT_END + || reserved.start & 0x3fff != 0 || reserved.end <= COMPACT_END + || reserved.end > DATA_START || reserved.end & 0x3fff != 0 { + return None; + } + Some(reserved.start..COMPACT_END) +} + +pub(crate) fn auxiliary_range(reserved: &Range) -> Option> { + compact_range(reserved)?; + Some(COMPACT_END..reserved.end) +} + +/// Command pointers must not reach driver-owned mappings, including when +/// the client leaves application data below its private arena reservation. +pub(crate) fn user_range_contains( + reserved: &Range, address: u64, size: u64, user_end: u64, +) -> bool { + let Some(end) = address.checked_add(size) else { return false; }; + size != 0 && address >= DRIVER_BASE && end <= user_end + && (end <= reserved.start || address >= reserved.end) +} + diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 5641384b51bfe..75c4d684b219f 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -209,7 +209,8 @@ struct drm_asahi_params_global { * * G13-G15 on 14.x report DRM_ASAHI_GPU_HAL_LEGACY. * The exact J613 25G83 runtime and the G17P runtime report - * DRM_ASAHI_GPU_HAL_200. + * DRM_ASAHI_GPU_HAL_200, the M5 (T8142) runtime + * DRM_ASAHI_GPU_HAL_300. */ __u32 gpu_hal_generation; @@ -241,7 +242,7 @@ enum drm_asahi_gpu_hal_generation { */ DRM_ASAHI_GPU_HAL_200 = 200, - /** @DRM_ASAHI_GPU_HAL_300: HAL300 command stream. Never reported by this kernel. */ + /** @DRM_ASAHI_GPU_HAL_300: HAL300 command stream, reported by the M5 (T8142) runtime. */ DRM_ASAHI_GPU_HAL_300 = 300, }; From f4dadb0fdbe066053bb441a761aa5c04b341fbbd Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Thu, 5 Nov 2026 20:12:36 +0100 Subject: [PATCH 108/117] drm/asahi: m5: Take the device binder and the queue's usage client integration/aurora added a bound-device owner to the platform driver data and a usage client to Gpu::new_queue. Create the owner in the M5 probe path like the other backends and accept the usage client on the M5 queue. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/asahi/driver.rs | 4 ++++ drivers/gpu/drm/asahi/m5/drm.rs | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 554a82285c865..003ec58b2f5da 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -464,6 +464,9 @@ impl AsahiPlatformDriver { let resources = regs::Resources::new(pdev)?; let res = regs::G17Resources::new(pdev)?; let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + // SAFETY: Every adapter calls close_binding in unbind before devres; + // probe errors drop this local owner while the bound callback is live. + let binding = unsafe { crate::binding::Owner::new(pdev.as_ref(), &drm)? }; let gpu = crate::m5::probe(pdev, &drm, res)?; let data = try_pin_init!(AsahiDrmRegistrationData { gpu: gpu as Arc, @@ -473,6 +476,7 @@ impl AsahiPlatformDriver { // The M5 firmware session retains its firmware-visible state until reboot. let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; Ok(AsahiPlatformDriverData { + binding, _device: reg.device().into(), _reg: reg, g17: None, diff --git a/drivers/gpu/drm/asahi/m5/drm.rs b/drivers/gpu/drm/asahi/m5/drm.rs index a2a7b9d2679ac..981a8531b71a5 100644 --- a/drivers/gpu/drm/asahi/m5/drm.rs +++ b/drivers/gpu/drm/asahi/m5/drm.rs @@ -167,7 +167,8 @@ impl gpu::Gpu for M5DrmGpu { fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, _ualloc_priv: Arc>, - priority: u32, usc_exec_base: u64) -> Result> { + priority: u32, usc_exec_base: u64, + _usage: Arc) -> Result> { if priority > 3 { return Err(EINVAL); } let window = command::QueueUscWindow { base: usc_exec_base, user_start: vm_layout::DRIVER_BASE, user_end: USER_END, From fb8436308454f967116359b10ae66fd151f01289 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Fri, 30 Oct 2026 11:33:39 +0100 Subject: [PATCH 109/117] dt-bindings: media: apple,avd: Add t8142 compatible The M5 (T8142) has a Hibiscus generation AVD with the T8140 register layout but its own firmware tunables and H.264 buffer layout. Signed-off-by: Thomas Glanzmann --- Documentation/devicetree/bindings/media/apple,avd.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/media/apple,avd.yaml b/Documentation/devicetree/bindings/media/apple,avd.yaml index de4101dfb50f7..1187255b0f474 100644 --- a/Documentation/devicetree/bindings/media/apple,avd.yaml +++ b/Documentation/devicetree/bindings/media/apple,avd.yaml @@ -35,6 +35,7 @@ properties: - apple,t8122-avd - apple,t8132-avd - apple,t8140-avd + - apple,t8142-avd reg: items: From 6e1e57f9a0e549402535f4075b9bbd9f0c6a2977 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Fri, 30 Oct 2026 11:33:39 +0100 Subject: [PATCH 110/117] media: apple: avd: Add T8142 (M5) support The M5 AVD (macOS CAvdMcpuHibiscus, avd-version 4) uses T8140's VP slot layout and v5 HEVC header with the avd-fw v5 tier 2 tunables. Its H.264 co-located motion vector buffer takes 128 bytes per macroblock instead of 64; with 64 inter frames decode with corrupted motion. Add AVD_QUIRK_H264_MV_128 and the t8142 variant. Tested on the J813 (MacBook Air M5): H.264, HEVC Main/Main10, VP9 and AV1 decode bit-exact against the software decoders through GStreamer v4l2codecs and FFmpeg's V4L2 request hwaccels. Also mark the device runtime-suspended before enabling runtime PM in probe, so a binding after a failed CM3 boot (e.g. missing firmware) starts from a clean state instead of an inherited runtime PM error. Signed-off-by: Thomas Glanzmann --- drivers/media/platform/apple/avd/avd-drv.c | 37 +++++++++++++++++++++ drivers/media/platform/apple/avd/avd-h264.c | 12 ++++--- drivers/media/platform/apple/avd/avd.h | 2 ++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 7d2e408e34486..2cba04cd6f443 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -551,6 +551,38 @@ static const struct avd_variant avd_t6041_variant = { .quirks = AVD_QUIRK_HEVC_V5_HEADER, }; +/* + * M5 (Hibiscus, macOS CAvdMcpuHibiscus): T8140's VP block layout and HEVC + * header; avd-fw's v5 tier 2 tunables match macOS's Hibiscus table. + */ +static const struct avd_variant avd_t8142_variant = { + .vp_slots = { + [AVD_CODEC_HEVC] = 2, + [AVD_CODEC_H264] = 1, + [AVD_CODEC_VP9] = 1, + [AVD_CODEC_AV1] = 1, + }, + .vp_slot_base = { + [AVD_CODEC_HEVC] = 0, + [AVD_CODEC_H264] = 4, + [AVD_CODEC_VP9] = 8, + [AVD_CODEC_AV1] = 9, + }, + .fifo_slots = 7, + .configure_stream = t8122_configure_stream, + .capabilities = AVD_CAPABILITY_HEVC | + AVD_CAPABILITY_H264 | + AVD_CAPABILITY_VP9 | + AVD_CAPABILITY_AV1, + .fw_name = "apple/avd-fw-v5-t2.bin", + .revision = 4, + .vp_slot_offset = 0xc, + .submit_offset = 0x40, + .submit_queue_max_offset = 0x44, + .submit_queue_status_offset = 0x7c, + .quirks = AVD_QUIRK_HEVC_V5_HEADER | AVD_QUIRK_H264_MV_128, +}; + /* can also be derived from a version register */ static const struct of_device_id avd_of_match[] = { { @@ -589,6 +621,10 @@ static const struct of_device_id avd_of_match[] = { .compatible = "apple,t8140-avd", .data = &avd_t8140_variant }, + { + .compatible = "apple,t8142-avd", + .data = &avd_t8142_variant + }, {}, }; @@ -770,3 +806,4 @@ MODULE_FIRMWARE("apple/avd-fw-v3-t1.bin"); MODULE_FIRMWARE("apple/avd-fw-v4-t0.bin"); MODULE_FIRMWARE("apple/avd-fw-v5-t0.bin"); MODULE_FIRMWARE("apple/avd-fw-v5-t1.bin"); +MODULE_FIRMWARE("apple/avd-fw-v5-t2.bin"); diff --git a/drivers/media/platform/apple/avd/avd-h264.c b/drivers/media/platform/apple/avd/avd-h264.c index fd1d4d53025ae..df80d61acdef6 100644 --- a/drivers/media/platform/apple/avd/avd-h264.c +++ b/drivers/media/platform/apple/avd/avd-h264.c @@ -120,9 +120,11 @@ static const u32 default_8x8_inter[] = { 0x191b1c1e, 0x1b1c1e20, 0x1c1e2021, 0x1e202123, }; -static inline u32 mv_color_size(u32 w, u32 h) +static inline u32 mv_color_size(struct avd_ctx *ctx, u32 w, u32 h) { - return (DIV_ROUND_UP(w, 16) + 1) * (DIV_ROUND_UP(h, 16) + 1) * 64; + u32 per_mb = ctx->dev->variant->quirks & AVD_QUIRK_H264_MV_128 ? 128 : 64; + + return (DIV_ROUND_UP(w, 16) + 1) * (DIV_ROUND_UP(h, 16) + 1) * per_mb; } /* sorry for the formatting */ @@ -562,7 +564,7 @@ static void stream_slice(struct avd_ctx *ctx, struct avd_h264_run *run) dma_addr_t mv_color_addr = vb ? vb2_dma_contig_plane_dma_addr(vb, 0) + (vb->planes[0].length - - mv_color_size(fmt_width(ctx), + mv_color_size(ctx, fmt_width(ctx), fmt_height(ctx))) : run->addresses.mv_color; @@ -782,7 +784,7 @@ static void avd_h264_run_preamble(struct avd_ctx *ctx, struct avd_h264_run *run) dst_len = run->base.bufs.dst->vb2_buf.planes[0].length; - mv_color_len = mv_color_size(fmt_width(ctx), fmt_height(ctx)); + mv_color_len = mv_color_size(ctx, fmt_width(ctx), fmt_height(ctx)); run->addresses.mv_color = run->base.y_out + (dst_len - mv_color_len); } @@ -1157,7 +1159,7 @@ static void avd_h264_adjust_decoded_fmt(struct avd_ctx *ctx, struct v4l2_pix_format_mplane *pix_mp) { pix_mp->plane_fmt[0].sizeimage += - mv_color_size(pix_mp->width, pix_mp->height); + mv_color_size(ctx, pix_mp->width, pix_mp->height); } static int avd_h264_try_ctrl(struct avd_ctx *ctx, struct v4l2_ctrl *ctrl) diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index a1825a1029ec5..1313a07e00389 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -41,6 +41,8 @@ #define AVD_QUIRK_HEVC_V5_HEADER BIT(2) /* T8140: pace only HEVC submissions, with a shorter back-off. */ #define AVD_QUIRK_V5_QUEUE_PACING BIT(3) +/* H.264 co-located motion vectors take 128 bytes per macroblock, not 64. */ +#define AVD_QUIRK_H264_MV_128 BIT(4) #define VP_SLOT_NONE 255 #define INST_FIFO_SLOT_NONE 255 From 4aa33e1a3568b77e3a575ad2fff08aeb7fc58d84 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Fri, 30 Oct 2026 11:33:40 +0100 Subject: [PATCH 111/117] media: apple: avd: Report the CM3 state when it fails to start avd_boot() returned -ETIMEDOUT without a message when the CM3 did not raise FLAG0, which made a wrong firmware image or a missing power domain look like a silent probe failure. Log the flag value. Signed-off-by: Thomas Glanzmann --- drivers/media/platform/apple/avd/avd-hw.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/avd/avd-hw.c b/drivers/media/platform/apple/avd/avd-hw.c index b64c0bf4d78ea..e973ba9d25b0f 100644 --- a/drivers/media/platform/apple/avd/avd-hw.c +++ b/drivers/media/platform/apple/avd/avd-hw.c @@ -61,8 +61,10 @@ int avd_boot(struct avd_dev *avd) /* wait for cm3 to boot */ ret = readl_poll_timeout(avd->mbox + AVD_REG_FLAG0_SET, val, val == 1, 10, 10000); - if (ret) + if (ret) { + dev_err(avd->dev, "cm3 did not start: flag0 %#x\n", val); return ret; + } return 0; } From 723ba5c4c0eb277128bd97a9ee48c11d2639f383 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:35:02 +0100 Subject: [PATCH 112/117] dt-bindings: media: apple,t8140-isp: Add the T8142 (M5) ISP The M5's ISP is the A18 Pro's ISP17a at another base address. Signed-off-by: Thomas Glanzmann --- .../devicetree/bindings/media/apple,t8140-isp.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml index 59c6c91fb19d7..1671fa01bec96 100644 --- a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml +++ b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml @@ -19,14 +19,18 @@ select: properties: compatible: contains: - const: apple,t8140-isp + enum: + - apple,t8140-isp + - apple,t8142-isp required: - compatible properties: compatible: items: - - const: apple,t8140-isp + - enum: + - apple,t8140-isp + - apple,t8142-isp - const: apple,isp reg: From 38c26d24c9d6f9cad42799b5400ab99638fa0285 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 1 Nov 2026 21:35:02 +0100 Subject: [PATCH 113/117] media: apple: isp: Add T8142 (M5) The J813's ISP is the A18 Pro's ISP17a: the T8140 generation with the H17 firmware ABI, its own MMIO windows (the J813 ADT's dart-isp dapf-instance-0 ranges, including a DSID window and two new ranges), the PMP clock and bandwidth scratch registers, the T6031 mailbox interrupt enable and the SCL1 output path (T8142 has no BES scaler). Its IMX958 setfile (isp_2224_01XX.dat, the one macOS's appleh16camerad selects for sensor 0x958) keeps the lens shading row strides where H17 reads the offset's high half, as the T6040's do: setfile_stride_fixup zeroes them on T8142 too. The camera streams at 1280x720 and 1920x1080 (NV12). White balance stays at the setfile's default gains: the H17 firmware's AWB is semantic AWB, which needs the ANE. Signed-off-by: Thomas Glanzmann --- drivers/media/platform/apple/isp/isp-cam.c | 2 +- drivers/media/platform/apple/isp/isp-drv.c | 59 ++++++++++++++++++++++ drivers/media/platform/apple/isp/isp-drv.h | 3 ++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 15b1ca2816fbc..4b096aa29dddf 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -696,7 +696,7 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) memset(isp->data_surf->virt + len, 0, setfile->size - len); release_firmware(fw); /* Seen on T6040 only; T8140 runs its setfiles as they are. */ - if (isp->hw->gen == ISP_GEN_T6040) + if (isp->hw->gen == ISP_GEN_T6040 || isp->hw->setfile_stride_fixup) isp_setfile_fixup_h17(isp->data_surf->virt, len); err = isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index a2390964a13d8..627a9bdb9e94d 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -1083,6 +1083,64 @@ static const struct apple_isp_hw apple_isp_hw_t8132 = { .deferred_start = true, }; +/* + * T8142 (M5, J813): the Neo's ISP17a. The J813 ADT's dart-isp + * dapf-instance-0 ranges, rounded out to 16 KiB pages; mostly the T8140 + * windows at +0x80000000, with one DSID window and two new ranges. + */ +static const struct isp_mmio_window apple_isp_fw_mmio_t8142[] = { + { 0x220104000, 0x14000 }, /* DSID broadcast-clear window */ + { 0x3803c0000, 0x28000 }, /* PMGR, with the PMP clock scratch */ + { 0x380704000, 0x4000 }, /* PMGR, ISP power states */ + { 0x380730000, 0x4000 }, + { 0x380e3c000, 0x4000 }, + { 0x382724000, 0x4000 }, /* PMP bandwidth scratch */ + { 0x39062c000, 0x30000 }, + { 0x3a1340000, 0x4000 }, + { 0x3a1358000, 0xc000 }, + { 0x421660000, 0x4000 }, +}; + +static const struct apple_isp_hw apple_isp_hw_t8142 = { + .gen = ISP_GEN_T8140, + .fw_abi = ISP_FW_ABI_H17, + .pmu_base = 0x0, + + .dsid_count = 1, + .dsid_clr_base0 = 0x220116000, + .dsid_clr_range0 = 0x2fc, + + .clock_scratch = 0x3803d0ca0, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x382724000, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + .mbox_irq_enable = ISP_MBOX_IRQ_ENABLE_T6031, + + /* + * No BES scaler: the firmware asserts on CH_CROP_SET ("BesProc is + * removed/not present. Must config AuxSclPr") and takes the SCL1 set. + */ + .scl1 = true, + .lpdp = true, + .meta_size = ISP_META_SIZE_T8140, + + .fw_iova_mask = GENMASK_ULL(35, 0), + .mbox2_offset = 0x410, + .mbox_irq_route = true, + .coproc_control = ISP_COPROC_CONTROL_T8140, + .fw_mmio = apple_isp_fw_mmio_t8142, + .num_fw_mmio = ARRAY_SIZE(apple_isp_fw_mmio_t8142), + .boot_mode = 1, + .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, + .resident_fw = true, + /* The J813 IMX958 runs the xx24 setfile 2224_01, as T6040 does. */ + .setfile_stride_fixup = true, +}; + static const struct of_device_id apple_isp_of_match[] = { { .compatible = "apple,t8103-isp", .data = &apple_isp_hw_t8103 }, { .compatible = "apple,t8112-isp", .data = &apple_isp_hw_t8112 }, @@ -1092,6 +1150,7 @@ static const struct of_device_id apple_isp_of_match[] = { { .compatible = "apple,t6030-isp", .data = &apple_isp_hw_t6030 }, { .compatible = "apple,t6031-isp", .data = &apple_isp_hw_t6031 }, { .compatible = "apple,t8140-isp", .data = &apple_isp_hw_t8140 }, + { .compatible = "apple,t8142-isp", .data = &apple_isp_hw_t8142 }, { .compatible = "apple,t6041-isp", .data = &apple_isp_hw_t6040 }, { .compatible = "apple,t8132-isp", .data = &apple_isp_hw_t8132 }, {}, diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 738590fec3f35..9056c3d7af143 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -251,6 +251,9 @@ struct apple_isp_hw { * detected only once. */ bool deferred_start; + + /* The xx24 setfiles' lens shading strides need zeroing (T8142). */ + bool setfile_stride_fixup; }; enum isp_sensor_id { From 13e8c1bb9acb457f31d01f3317070d227679b8b6 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 6 Nov 2026 06:19:35 -0500 Subject: [PATCH 114/117] media: avd: declare J813 MV quirk in existing program harness --- drivers/media/platform/apple/avd/tests/program-test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/media/platform/apple/avd/tests/program-test.py b/drivers/media/platform/apple/avd/tests/program-test.py index 2778a416d3a93..ab84291ea3c7c 100755 --- a/drivers/media/platform/apple/avd/tests/program-test.py +++ b/drivers/media/platform/apple/avd/tests/program-test.py @@ -47,6 +47,7 @@ #define VISIBLE_IF_KUNIT #define AVD_QUIRK_LSR 1 #define AVD_QUIRK_NO_PIPE_STATE 2 +#define AVD_QUIRK_H264_MV_128 (1U << 4) #define AVD_CODEC_HEVC 0 #define AVD_CODEC_H264 1 #define AVD_CODEC_VP9 2 From 20d9fd50f3ab22f4ac6487ba682abc797a8aa5e8 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 6 Nov 2026 06:43:50 -0500 Subject: [PATCH 115/117] drm/asahi: describe bounded M5 stencil and multisample renders Validate complete depth and stencil allocations in sample space, discard both single-layer strides, and derive the captured fragment controls for two- and four-sample RGBA8 tiles. Use the UAPI utile dimensions once when sizing tile lists. Keep unknown formats, layouts and render modes outside admission. Hardware qualification is pending. --- drivers/gpu/drm/asahi/m5/command.rs | 168 ++++++++++-------------- drivers/gpu/drm/asahi/m5/drm.rs | 33 +++-- drivers/gpu/drm/asahi/m5/render_desc.rs | 14 +- 3 files changed, 92 insertions(+), 123 deletions(-) diff --git a/drivers/gpu/drm/asahi/m5/command.rs b/drivers/gpu/drm/asahi/m5/command.rs index c4555602c781e..58e0ca0eb6670 100644 --- a/drivers/gpu/drm/asahi/m5/command.rs +++ b/drivers/gpu/drm/asahi/m5/command.rs @@ -754,116 +754,86 @@ fn validate_zls( Ok(()) } -/// Bounded M5 D32 clear/store envelope, alongside the original color-only -/// profile. Native 25G76 supplies ff00ff dimensions and 80044 with compression; -/// bits2/6 select compression, while bit19 stores the depth plane. Geometry, -/// stencil absence, and application programs are checked by the M5 caller. +/// Validate the complete single-layer M5 Z/S allocations before their unused +/// array strides are discarded. AIL stores samples in an expanded 2D surface: +/// 2x doubles Y, and 4x doubles both axes. The dimensions register stays in pixels. pub(crate) fn validate_m5_reference_depth( command: &UapiRenderCommand, address_space: &A, ) -> Result<(), UapiTranslateError> { - let depth = command.depth; - if depth.base == 0 { - // No depth plane. The surface may still carry an inert stencil one, - // which contributes nothing but its compression pair and the shared - // pixel dimensions; a render that transfers it is refused with the - // rest of the stencil profile. - let stencil_only = command.stencil.base != 0 - && command.zls_control - == if command.stencil.compression_base != 0 { 0x110 } else { 0 } - && command.depth_dimensions - == ((u32::from(command.width) - 1) - | ((u32::from(command.height) - 1) << 16)); - return if depth.compression_base == 0 && depth.stride == 0 - && depth.compression_stride == 0 - && (stencil_only - || (command.zls_control == 0 && command.depth_dimensions == 0)) { - Ok(()) - } else { - Err(UapiTranslateError::ZlsFields) - }; - } - let compressed = depth.compression_base != 0; - // The control word's load and store bits are the only part a render - // chooses: bit15 loads the depth plane and bit19 stores it, and a render - // that neither tests nor writes depth sets neither, which is how a - // conformance surface with an unused depth buffer arrives. Everything - // else -- the compression pair in bits 2 and 6, the GPU tiling, the 32F - // format -- is the witnessed profile and stays exact. depth_flags reaches - // register 0x15319 unchanged either way, so the admitted envelope is the - // set of load/store combinations over that fixed remainder. - let compression_bits = if compressed { 0x44 } else { 0 }; - // Bits 25-26 select the depth format: 0 is 32-bit float, 2 is 16-bit - // unorm. The narrow plane was captured on 25G76 alongside the wide one - // and moves nothing else in the register program except the clear value - // and one ISP flag, both of which the caller derives. Its compression - // metadata has no witness, so a compressed 16-bit plane stays out. - let z16 = command.zls_control & (3 << 25) == (2 << 25); - if z16 && compressed { - return Err(UapiTranslateError::ZlsFields); - } - let format_bits = if z16 { 2 << 25 } else { 0 }; - // A stencil plane may be attached without being transferred: a surface - // with a depth-stencil buffer carries one for every render, whether or - // not the render uses it. Its compression pair in bits 4 and 8 describes - // the attached layout and is the only stencil state admitted here -- - // loading or storing the plane (bits 14 and 18) needs its own witnesses, - // as does the stencil unit in the graphics record. - let stencil_compression_bits = - if command.stencil.compression_base != 0 { 0x110 } else { 0 }; - let control_transfers = command.zls_control & 0x88000; - let expected_control = - compression_bits | stencil_compression_bits | format_bits | control_transfers; let width = u32::from(command.width); let height = u32::from(command.height); if !(1..=M5_MAX_RENDER_DIMENSION).contains(&width) || !(1..=M5_MAX_RENDER_DIMENSION).contains(&height) - || command.layers != 1 || (compressed && (width < 16 || height < 16)) { + || command.layers != 1 || !matches!(command.samples, 1 | 2 | 4) { return Err(UapiTranslateError::ZlsFields); } - // AIL tiles are 16 KiB whatever the format: 64x64 at four bytes a pixel, - // 128x64 at two. While both axes fit one tile the level is a whole - // number of them; if either axis is smaller, the level uses a - // power-of-two miptail aligned to a 128-byte cacheline. Single-layer M5 - // surfaces are not page-padded. - let (tile_width, bytes_per_pixel) = if z16 { (128u32, 2u64) } else { (64u32, 4u64) }; - let data_bytes = if width >= tile_width && height >= 64 { - u64::from(width.div_ceil(tile_width)) * u64::from(height.div_ceil(64)) * 0x4000 - } else { - (u64::from(width.next_power_of_two()) - * u64::from(height.next_power_of_two()) * bytes_per_pixel) - .div_ceil(128) * 128 - }; - let metadata_bytes = (u64::from(width.next_power_of_two().div_ceil(16)) - * u64::from(height.next_power_of_two().div_ceil(16)) * 8).div_ceil(128) * 128; - let dimensions = (width - 1) | ((height - 1) << 16); - if command.depth_dimensions != dimensions - || command.zls_control != expected_control - // The clear value is whatever the application asked for; it reaches - // register 0x15301 verbatim and is only consumed when the render - // actually clears the plane. - || (command.depth_clear & 0x7fffffff) > 0x7f800000 - // Image suballocations and small miptails use cacheline alignment; - // the backing VM pages remain independently validated below. - || depth.base & 0x7f != 0 || depth.compression_base & 0x7f != 0 - // Both strides are array strides, and the M5 caller zeroes them - // because native single-layer ZLS does not encode one. Every render - // admitted here is single-layer -- layers != 1 is refused above -- so - // the value that arrives is whatever the userspace packer computed - // and the hardware never sees it: for a surface smaller than a page - // Mesa's (pages - 1) << 14 | 1 underflows to 0xffffc001. Comparing it - // would refuse a valid render over a field that is discarded. The - // backing store is still validated in full below, from the - // dimensions rather than from either stride. - || (!compressed && depth.compression_stride != 0) { - return Err(UapiTranslateError::ZlsFields); + let sample_width = width * if command.samples == 4 { 2 } else { 1 }; + let sample_height = height * if command.samples > 1 { 2 } else { 1 }; + let depth = command.depth; + let stencil = command.stencil; + let z16 = command.zls_control & (3 << 25) == (2 << 25); + let mut expected_control = 0; + for (plane, bytes_per_sample, transfers, compression) in [ + (depth, if z16 { 2u64 } else { 4u64 }, 0x88000u64, 0x44u64), + (stencil, 1u64, 0x44000u64, 0x110u64), + ] { + if plane.base == 0 { + if plane.compression_base != 0 || plane.stride != 0 + || plane.compression_stride != 0 { + return Err(UapiTranslateError::ZlsFields); + } + continue; + } + let compressed = plane.compression_base != 0; + // Compressed Z16 remains outside the captured layout. Both Z32 and + // S8 compression use 8-byte metadata records for 16x16 sample tiles. + if plane.base & 0x7f != 0 || plane.compression_base & 0x7f != 0 + || (!compressed && plane.compression_stride != 0) + || (compressed && (bytes_per_sample == 2 + || sample_width < 16 || sample_height < 16)) { + return Err(UapiTranslateError::ZlsFields); + } + expected_control |= command.zls_control & transfers; + if compressed { expected_control |= compression; } + // AIL tiles contain 16 KiB: 64x64 Z32, 128x64 Z16, 128x128 S8. + // A miptail smaller than either tile axis is power-of-two padded and + // cacheline aligned, rather than rounded to a whole 16-KiB page. + let (tile_width, tile_height) = match bytes_per_sample { + 4 => (64u32, 64u32), + 2 => (128, 64), + _ => (128, 128), + }; + let data_bytes = if sample_width >= tile_width && sample_height >= tile_height { + u64::from(sample_width.div_ceil(tile_width)) + * u64::from(sample_height.div_ceil(tile_height)) * 0x4000 + } else { + (u64::from(sample_width.next_power_of_two()) + * u64::from(sample_height.next_power_of_two()) * bytes_per_sample) + .div_ceil(128) * 128 + }; + require_range(address_space, plane.base, data_bytes, GpuAccess::ReadWrite)?; + if compressed { + let metadata_bytes = (u64::from(sample_width.next_power_of_two().div_ceil(16)) + * u64::from(sample_height.next_power_of_two().div_ceil(16)) * 8) + .div_ceil(128) * 128; + require_range(address_space, plane.compression_base, metadata_bytes, + GpuAccess::ReadWrite)?; + } } - // Validate the entire resized surface and metadata before the - // M5 caller removes the unused single-layer array stride fields. - require_range(address_space, depth.base, data_bytes, GpuAccess::ReadWrite)?; - if compressed { - require_range(address_space, depth.compression_base, metadata_bytes, - GpuAccess::ReadWrite)?; + if depth.base != 0 { + if z16 { expected_control |= 2 << 25; } + if (command.depth_clear & 0x7fffffff) > 0x7f800000 { + return Err(UapiTranslateError::ZlsFields); + } + } + let dimensions = if depth.base != 0 || stencil.base != 0 { + (width - 1) | ((height - 1) << 16) + } else { 0 }; + // Exact matching rejects unknown format bits and transfer requests for + // an absent plane, as well as mismatched compression/base-address pairs. + if command.zls_control != expected_control || command.depth_dimensions != dimensions { + return Err(UapiTranslateError::ZlsFields); } Ok(()) } diff --git a/drivers/gpu/drm/asahi/m5/drm.rs b/drivers/gpu/drm/asahi/m5/drm.rs index 981a8531b71a5..bbaba829a945e 100644 --- a/drivers/gpu/drm/asahi/m5/drm.rs +++ b/drivers/gpu/drm/asahi/m5/drm.rs @@ -313,16 +313,6 @@ impl queue::Queue for M5Queue { payload.utile_width, payload.utile_height, payload.samples); error })?; - // An attached stencil plane is admitted only while it is inert: - // its buffers are range-checked with the depth ones and its - // registers are written from the same parameter block, but a - // render that loads or stores it, or that runs the stencil unit, - // is outside the witnessed profile. - if payload.zls_control & 0x44000 != 0 { - pr_err!("M5 render admission rejected stencil transfer control={:#x}\n", - payload.zls_control); - return Err(ENOTSUPP); - } command::validate_m5_reference_depth(&payload, &address_space) .map_err(|error| { pr_err!("M5 depth admission rejected {:?}: size={}x{} dimensions={:#x} control={:#x} stride={:#x} meta_stride={:#x} base={:#x} meta={:#x} clear={:#x}\n", @@ -339,6 +329,16 @@ impl queue::Queue for M5Queue { // The full data/metadata ranges were validated above. translated.parameters.depth_stride = 0; translated.parameters.depth_aux_stride = 0; + translated.parameters.stencil_stride = 0; + translated.parameters.stencil_aux_stride = 0; + if payload.samples > 1 { + // 25G76 2x/4x captures: 10051 carries samples << 16, + // d491 enables multisampling, and PTC gains eight entries. + // The UAPI already supplies sample positions and utile size. + translated.parameters.tib_blocks = u64::from(payload.samples) << 16; + controls.fragment.resource_flags |= 1 << 32; + controls.fragment.ptc_max = 0xd6; + } // Native 25G76 indexed draws use 15021=0x10c000. The reference // triangle's low bit causes fragment recovery with larger // indexed overdraw, even after the tiling stage completes. @@ -368,16 +368,23 @@ impl queue::Queue for M5Queue { } } -/// Until the M5 controls are derived for other render modes, admit only the -/// reference render-state envelope. This is an experimental first-job ABI +/// Admit the captured M5 single-layer render-state envelope. This is an experimental first-job ABI /// path, not an assertion that arbitrary Vulkan programs are supported. fn admit_reference_render(command: &command::UapiRenderCommand) -> Result { + let sample_layout = match command.samples { + 1 => command.utile_width == 32 && command.utile_height == 32, + 2 => command.utile_width == 32 && command.utile_height == 32 + && command.sample_size == 4 && command.multisample_control == 0x44cc, + 4 => command.utile_width == 32 && command.utile_height == 16 + && command.sample_size == 4 && command.multisample_control == 0xeaa2_6e26, + _ => false, + }; let timestamps = [command.vertex_timestamps.start, command.vertex_timestamps.end, command.fragment_timestamps.start, command.fragment_timestamps.end]; if !(1..=command::M5_MAX_RENDER_DIMENSION).contains(&u32::from(command.width)) || !(1..=command::M5_MAX_RENDER_DIMENSION).contains(&u32::from(command.height)) || command.layers != 1 - || command.utile_width != 32 || command.utile_height != 32 || command.samples != 1 + || !sample_layout || command.sampler_heap != 0 || command.sampler_count != 0 || command.occlusion_query_base != 0 || timestamps.iter().any(|timestamp| timestamp.handle != 0 || timestamp.offset != 0) { diff --git a/drivers/gpu/drm/asahi/m5/render_desc.rs b/drivers/gpu/drm/asahi/m5/render_desc.rs index 42a556acde208..c2dd099e01a9d 100644 --- a/drivers/gpu/drm/asahi/m5/render_desc.rs +++ b/drivers/gpu/drm/asahi/m5/render_desc.rs @@ -152,7 +152,6 @@ pub(crate) struct G17pRenderParameters { pub(crate) layers: u16, /// Sample count: 1, 2 or 4. Defaults to 1, so every existing parameter /// vector keeps its exact register program. - pub(crate) samples: u8, pub(crate) utile_width: u8, pub(crate) utile_height: u8, pub(crate) utile_config: u64, @@ -282,7 +281,6 @@ impl G17pRenderParameters { merge_upper_x_bits: 0x3cdd_b3d9, merge_upper_y_bits: 0x3cdd_b3d9, layers: 1, - samples: 1, utile_width: 32, utile_height: 32, utile_config: 0xa000, @@ -353,15 +351,9 @@ fn geometry(parameters: &G17pRenderParameters) -> Result 0x28, size3 0x20 -> 0x40 and - // macro_size 0x40004 -> 0x40008, where only the y term moves. All three - // follow from doubling this one term; two samples do not split. - let utiles_per_tile_y = if parameters.samples == 4 { - utiles_per_tile_y * 2 - } else { - utiles_per_tile_y - }; + // The UAPI's utile height already halves at four samples. Use the actual + // dimensions once; a second sample-count factor overallocates tile lists + // and publishes a macro size that disagrees with the native register set. let utiles_per_tile = utiles_per_tile_x * utiles_per_tile_y; let mtile_x1 = align((tiles_x + MACRO_TILES_X - 1) / MACRO_TILES_X, 4); let mtile_y1 = align((tiles_y + MACRO_TILES_Y - 1) / MACRO_TILES_Y, 4); From 93e3760751cf77bd99b95f9e3c3fd57c566bb089 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 6 Nov 2026 06:51:24 -0500 Subject: [PATCH 116/117] drm/asahi: admit the padded M5 multisample tile stride Mesa aligns a 32-bit render target sample stride to eight bytes. Preserve the bounded native sample layouts while accepting that real UAPI value, and include stride and positions in rejection diagnostics. The first physical stencil/depth probe passed; the initial multisample probe was safely refused by the four-byte stride check. --- drivers/gpu/drm/asahi/m5/drm.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/m5/drm.rs b/drivers/gpu/drm/asahi/m5/drm.rs index bbaba829a945e..7252236924f91 100644 --- a/drivers/gpu/drm/asahi/m5/drm.rs +++ b/drivers/gpu/drm/asahi/m5/drm.rs @@ -308,9 +308,10 @@ impl queue::Queue for M5Queue { return Err(ENOTSUPP); }; admit_reference_render(&payload).map_err(|error| { - pr_err!("M5 render admission rejected size={}x{} layers={} utiles={}x{} samples={}\n", + pr_err!("M5 render admission rejected size={}x{} layers={} utiles={}x{} samples={} sample_bytes={} positions={:#x}\n", payload.width, payload.height, payload.layers, - payload.utile_width, payload.utile_height, payload.samples); + payload.utile_width, payload.utile_height, payload.samples, + payload.sample_size, payload.multisample_control); error })?; command::validate_m5_reference_depth(&payload, &address_space) @@ -371,12 +372,14 @@ impl queue::Queue for M5Queue { /// Admit the captured M5 single-layer render-state envelope. This is an experimental first-job ABI /// path, not an assertion that arbitrary Vulkan programs are supported. fn admit_reference_render(command: &command::UapiRenderCommand) -> Result { + // A single 32-bit color target has an eight-byte aligned sample stride + // in Mesa's tilebuffer layout; sample_size includes that padding. let sample_layout = match command.samples { 1 => command.utile_width == 32 && command.utile_height == 32, 2 => command.utile_width == 32 && command.utile_height == 32 - && command.sample_size == 4 && command.multisample_control == 0x44cc, + && command.sample_size == 8 && command.multisample_control == 0x44cc, 4 => command.utile_width == 32 && command.utile_height == 16 - && command.sample_size == 4 && command.multisample_control == 0xeaa2_6e26, + && command.sample_size == 8 && command.multisample_control == 0xeaa2_6e26, _ => false, }; let timestamps = [command.vertex_timestamps.start, command.vertex_timestamps.end, From e50ba38b24140fc0751b811ea093c8f843bcec76 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 6 Nov 2026 07:09:26 -0500 Subject: [PATCH 117/117] drm/asahi: clarify M5 utile dimension documentation --- drivers/gpu/drm/asahi/m5/render_desc.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/m5/render_desc.rs b/drivers/gpu/drm/asahi/m5/render_desc.rs index c2dd099e01a9d..8e73ee27c5749 100644 --- a/drivers/gpu/drm/asahi/m5/render_desc.rs +++ b/drivers/gpu/drm/asahi/m5/render_desc.rs @@ -150,8 +150,7 @@ pub(crate) struct G17pRenderParameters { pub(crate) merge_upper_x_bits: u32, pub(crate) merge_upper_y_bits: u32, pub(crate) layers: u16, - /// Sample count: 1, 2 or 4. Defaults to 1, so every existing parameter - /// vector keeps its exact register program. + /// Utile dimensions in pixels, including the sample-layout adjustment. pub(crate) utile_width: u8, pub(crate) utile_height: u8, pub(crate) utile_config: u64,